@engine9/schemas 1.11.0 → 1.11.3

package/channels/email/segmentDefinitionClauses.js

@@ -60,8 +60,9 @@ export const emailEngagement = {
     type: 'object',
     properties: {
       pluginId: {
-        title: 'Plugin ID',
-        type: 'string'
+        title: 'Plugin',
+        type: 'string',
+        resolveOptionsUsing: 'plugin'
       },
       windowDays: {
         title: 'Window (days)',

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@engine9/schemas",
-  "version": "1.11.0",
+  "version": "1.11.3",
   "type": "module",
   "description": "Engine9 standard schemas: shared tables, transforms, searches, and segments",
   "main": "index.js",
   "scripts": {
-    "test": "node --test person/upsert.test.js person_address/normalize.test.js person_email/upsert.test.js person_hash/hash.test.js person_name/upsert.test.js person_phone/upsert.test.js stacks/stacks.test.js transaction/core/attribute.test.js"
+    "test": "node --test person/upsert.test.js person_address/normalize.test.js person_address/segmentDefinitionClauses.test.js person_email/upsert.test.js person_hash/hash.test.js person_name/upsert.test.js person_phone/upsert.test.js stacks/stacks.test.js timeline/segmentDefinitionClauses.test.js transaction/core/attribute.test.js transaction/core/segmentDefinitionClauses.test.js"
   },
   "engines": {
     "node": ">=18"

package/person/index.js

@@ -22,9 +22,9 @@ const transforms = {
   appendName
 };
 const person = {
-    title: 'Primary Info',
+    title: 'Ids',
     form: {
-      title: 'Primary Info',
+      title: 'Ids',
       type: 'object',
       properties: {
         ids: {

package/person_address/index.js

@@ -2,6 +2,7 @@ import upsertPersonAddress from './transforms/inbound/upsert_tables.js';
 import normalize from './transforms/inbound/normalize.js';
 import appendAddress from './transforms/outbound/appendAddress.js';
 import schema from './schema.js';
+import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
 
 const metadata = {
   name: '@engine9/schemas/person_address',
@@ -23,8 +24,10 @@ export const transforms = {
 
 export { metadata };
 export { schema };
+export { segmentDefinitionClauses };
 export default {
   metadata,
   schema,
-  transforms
+  transforms,
+  segmentDefinitionClauses
 };

package/person_address/segmentDefinitionClauses.js

@@ -0,0 +1,98 @@
+import { normalizeCountry, normalizePostalCode, normalizeRegion } from './normalize.js';
+
+function blank(value) {
+  return value == null || String(value).trim() === '';
+}
+
+/* Stored regions are normalized per country; without a country, accept US then Canadian names. */
+function regionFor(region, country) {
+  if (country) return normalizeRegion(region, country);
+  const us = normalizeRegion(region, 'US');
+  const upper = String(region).toUpperCase().replace(/\./g, '').trim();
+  return us !== upper ? us : normalizeRegion(region, 'CA');
+}
+
+/* Without a country, read US ZIPs and Canadian postal codes by shape; leave anything else as typed. */
+function postalCountryFor(postalCode, country) {
+  if (country) return country;
+  if (/^\d{5}(-?\d{4})?$/.test(postalCode)) return 'US';
+  if (/^[A-Z]\d[A-Z] ?\d[A-Z]\d$/.test(postalCode)) return 'CA';
+  return null;
+}
+
+function equals(column, value) {
+  return {
+    type: 'EQUALS',
+    values: [{ ref: { column } }, { value: { value } }]
+  };
+}
+
+export const addresses = {
+  title: 'Addresses',
+  description:
+    'People with an address in a region (state / province) and/or postal code. One value per field; combine several with an or group.',
+  form: {
+    title: 'Addresses',
+    type: 'object',
+    properties: {
+      region: {
+        title: 'Region',
+        description: "State or province code or name, e.g. 'VA' or 'Virginia'",
+        type: 'string'
+      },
+      postalCode: {
+        title: 'Postal code',
+        description: "Exact postal code, or a LIKE pattern when it contains %, e.g. '220%'",
+        type: 'string'
+      },
+      country: {
+        title: 'Country',
+        description: "ISO country code or name, e.g. 'US'. Optional; narrows the match and how region and postal code are read.",
+        type: 'string'
+      }
+    },
+    required: []
+  },
+  optionsToEQL(options = {}) {
+    const { region, postalCode, country: countryOption } = options;
+    const conditions = [];
+    const text = [];
+    const country = blank(countryOption) ? null : normalizeCountry(countryOption);
+    if (!blank(region)) {
+      const value = regionFor(region, country);
+      conditions.push(equals('region', value));
+      text.push(`in ${value}`);
+    }
+    if (!blank(postalCode)) {
+      const raw = String(postalCode).trim().toUpperCase();
+      if (raw.includes('%')) {
+        conditions.push({
+          type: 'LIKE',
+          values: [{ ref: { column: 'postal_code' } }, { value: { value: raw } }]
+        });
+        text.push(`with postal code matching ${raw}`);
+      } else {
+        const value = normalizePostalCode(raw, postalCountryFor(raw, country));
+        conditions.push(equals('postal_code', value));
+        text.push(`with postal code ${value}`);
+      }
+    }
+    if (country) {
+      conditions.push(equals('country', country));
+      text.push(`in country ${country}`);
+    }
+    return {
+      text: ['Has an address', ...text].join(' '),
+      eql: {
+        table: 'person_address',
+        columns: ['person_id'],
+        conditions
+      }
+    };
+  }
+};
+
+export const segmentDefinitionClauses = { addresses };
+export default {
+  segmentDefinitionClauses
+};

package/person_address/segmentDefinitionClauses.test.js

@@ -0,0 +1,46 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { addresses } from './segmentDefinitionClauses.js';
+
+const valuesByColumn = (eql) =>
+  Object.fromEntries(eql.conditions.map((c) => [c.values[0].ref.column, [c.type, c.values[1].value.value]]));
+
+test('region names normalize to the stored code', () => {
+  const { text, eql } = addresses.optionsToEQL({ region: 'Virginia' });
+  assert.equal(eql.table, 'person_address');
+  assert.deepEqual(eql.columns, ['person_id']);
+  assert.deepEqual(valuesByColumn(eql), { region: ['EQUALS', 'VA'] });
+  assert.equal(text, 'Has an address in VA');
+});
+
+test('Canadian province names normalize without a country', () => {
+  const { eql } = addresses.optionsToEQL({ region: 'Ontario' });
+  assert.deepEqual(valuesByColumn(eql), { region: ['EQUALS', 'ON'] });
+});
+
+test('postal codes normalize by shape, or match a LIKE pattern', () => {
+  assert.deepEqual(valuesByColumn(addresses.optionsToEQL({ postalCode: '22201-1234' }).eql), {
+    postal_code: ['EQUALS', '22201']
+  });
+  assert.deepEqual(valuesByColumn(addresses.optionsToEQL({ postalCode: 'k1a0b1' }).eql), {
+    postal_code: ['EQUALS', 'K1A 0B1']
+  });
+  assert.deepEqual(valuesByColumn(addresses.optionsToEQL({ postalCode: 'SW1A 1AA' }).eql), {
+    postal_code: ['EQUALS', 'SW1A 1AA']
+  });
+  assert.deepEqual(valuesByColumn(addresses.optionsToEQL({ postalCode: '220%' }).eql), {
+    postal_code: ['LIKE', '220%']
+  });
+});
+
+test('country narrows the match and drives region normalization', () => {
+  const { text, eql } = addresses.optionsToEQL({ region: 'virginia', country: 'United States' });
+  assert.deepEqual(valuesByColumn(eql), { region: ['EQUALS', 'VA'], country: ['EQUALS', 'US'] });
+  assert.equal(text, 'Has an address in VA in country US');
+});
+
+test('no options means any address', () => {
+  const { text, eql } = addresses.optionsToEQL({});
+  assert.deepEqual(eql.conditions, []);
+  assert.equal(text, 'Has an address');
+});

package/person_remote/index.js

@@ -15,15 +15,16 @@ const metadata = {
   }
 };
 const all = {
-    title: 'Remote people',
-    name: 'All remote people',
+    title: 'Remote Ids',
+    name: 'All remote ids',
     form: {
-      title: 'Remote people',
+      title: 'Remote Ids',
       type: 'object',
       properties: {
         pluginId: {
-          title: 'Plugin ID',
-          type: 'string'
+          title: 'Plugin',
+          type: 'string',
+          resolveOptionsUsing: 'plugin'
         }
       },
       required: []

package/plugin/schema.js

@@ -35,7 +35,7 @@ export const tables = [
       id: 'id_uuid',
       plugin_id: 'foreign_uuid',
       name: 'string',
-      value: 'string',
+      value: 'text',
       created_at: 'created_at',
       modified_at: 'modified_at'
     },

package/segment/segmentDefinitionClauses.js

@@ -6,8 +6,8 @@ export const segment = {
     properties: {
       segmentId: {
         title: 'Segment',
-        description: '',
-        type: 'string'
+        type: 'string',
+        resolveOptionsUsing: 'segment'
       }
     },
     required: []

package/stacks/limited-pii/stack.json

@@ -8,8 +8,8 @@
     "@engine9/schemas/person",
     "@engine9/schemas/person_remote",
     "@engine9/schemas/segment",
-    "@engine9/schemas/timeline",
     "@engine9/schemas/source_code",
+    "@engine9/schemas/timeline",
     "@engine9/schemas/transaction/core",
     "@engine9/schemas/person_hash"
   ],

package/stacks/stacks.test.js

@@ -12,6 +12,11 @@ test('standard stack includes plaintext contact plugins and does not exclude the
   assert.ok(standard.include.includes('@engine9/schemas/person_address'));
   assert.ok(standard.include.includes('@engine9/schemas/timeline'));
   assert.ok(standard.include.includes('@engine9/schemas/source_code'));
+  assert.ok(
+    standard.include.indexOf('@engine9/schemas/source_code') <
+      standard.include.indexOf('@engine9/schemas/timeline'),
+    'source_code must install before timeline (timeline depends on it)'
+  );
   assert.ok(standard.include.includes('@engine9/schemas/transaction/core'));
   assert.ok(!standard.include.includes('@engine9/plugins/reports/people'));
   assert.deepEqual(standard.exclude, []);

package/stacks/standard/stack.json

@@ -12,8 +12,8 @@
     "@engine9/schemas/person_email",
     "@engine9/schemas/person_phone",
     "@engine9/schemas/person_address",
-    "@engine9/schemas/timeline",
     "@engine9/schemas/source_code",
+    "@engine9/schemas/timeline",
     "@engine9/schemas/transaction/core",
     "@engine9/schemas/transaction/contact_details"
   ],

package/timeline/index.js

@@ -1,9 +1,11 @@
 import schema from './schema.js';
 import upsert from './transforms/inbound/upsert_tables.js';
+import { segmentDefinitionClauses } from './segmentDefinitionClauses.js';
 const metadata = {
   name: '@engine9/schemas/timeline',
   dependencies: {
-    '@engine9/schemas/person': '>=1.9.0'
+    '@engine9/schemas/person': '>=1.9.0',
+    '@engine9/schemas/source_code': '>=1.9.0'
   }
 };
 export const transforms = {
@@ -11,8 +13,10 @@ export const transforms = {
 };
 export { metadata };
 export { schema };
+export { segmentDefinitionClauses };
 export default {
   metadata,
   schema,
-  transforms
+  transforms,
+  segmentDefinitionClauses
 };

package/timeline/segmentDefinitionClauses.js

@@ -0,0 +1,56 @@
+function sourceCodeCondition(sourceCode) {
+  const value = String(sourceCode);
+  const type = value.includes('%') ? 'LIKE' : 'EQUALS';
+  return {
+    type,
+    values: [{ ref: { column: 'source_code' } }, { value: { value } }]
+  };
+}
+
+/**
+ * People with at least one timeline entry whose source_code matches.
+ *
+ * Options:
+ * - sourceCode: exact code, or LIKE pattern when it contains %
+ */
+export const sourceCode = {
+  title: 'Timeline source code',
+  description:
+    'People with a timeline entry credited to a source code. Exact match, or a LIKE pattern when the value contains %.',
+  form: {
+    title: 'Timeline source code',
+    type: 'object',
+    properties: {
+      sourceCode: {
+        title: 'Source code',
+        description: 'Exact source code, or a LIKE pattern when it contains %',
+        type: 'string'
+      }
+    },
+    required: ['sourceCode']
+  },
+  optionsToEQL(options = {}) {
+    const { sourceCode: code } = options;
+    if (!code) throw new Error('sourceCode is required');
+    const matchLabel = String(code).includes('%') ? 'matching ' : '';
+    return {
+      text: `Has a timeline entry with source code ${matchLabel}${code}`,
+      eql: {
+        table: 'timeline',
+        columns: ['person_id'],
+        joins: [
+          {
+            table: 'source_code_dictionary',
+            join_eql: 'timeline.source_code_id=source_code_dictionary.source_code_id'
+          }
+        ],
+        conditions: [sourceCodeCondition(code)]
+      }
+    };
+  }
+};
+
+export const segmentDefinitionClauses = { sourceCode };
+export default {
+  segmentDefinitionClauses
+};

package/timeline/segmentDefinitionClauses.test.js

@@ -0,0 +1,37 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { sourceCode, segmentDefinitionClauses } from './segmentDefinitionClauses.js';
+
+test('timeline sourceCode is the only segment definition clause', () => {
+  assert.deepEqual(Object.keys(segmentDefinitionClauses), ['sourceCode']);
+});
+
+test('timeline sourceCode requires sourceCode', () => {
+  assert.throws(() => sourceCode.optionsToEQL({}), /sourceCode/);
+});
+
+test('timeline sourceCode exact match joins the dictionary', () => {
+  const { text, eql } = sourceCode.optionsToEQL({ sourceCode: 'AU_TEST' });
+  assert.equal(text, 'Has a timeline entry with source code AU_TEST');
+  assert.equal(eql.table, 'timeline');
+  assert.deepEqual(eql.columns, ['person_id']);
+  assert.deepEqual(eql.joins, [
+    {
+      table: 'source_code_dictionary',
+      join_eql: 'timeline.source_code_id=source_code_dictionary.source_code_id'
+    }
+  ]);
+  assert.deepEqual(eql.conditions, [
+    {
+      type: 'EQUALS',
+      values: [{ ref: { column: 'source_code' } }, { value: { value: 'AU_TEST' } }]
+    }
+  ]);
+});
+
+test('timeline sourceCode uses LIKE when the value contains %', () => {
+  const { text, eql } = sourceCode.optionsToEQL({ sourceCode: 'AUTH_%' });
+  assert.equal(text, 'Has a timeline entry with source code matching AUTH_%');
+  assert.equal(eql.conditions[0].type, 'LIKE');
+  assert.equal(eql.conditions[0].values[1].value.value, 'AUTH_%');
+});

package/transaction/core/README.md

@@ -55,8 +55,7 @@ The inbound `upsert` transform writes rows to `transaction`.
 
 ## Segment definition clauses
 
-- `all` selects people who have any transaction, optionally filtered by `plugin_id` and a `start` / `end` window on `ts` (ISO datetimes or relative expressions such as `-30d`).
-- `minimum` selects people who have a transaction of at least a given `amount`.
+- `all` selects people who have any transaction, optionally filtered by `plugin_id`, a `start` / `end` window on `ts` (ISO datetimes or relative expressions such as `-30d`), and `minAmount` / `maxAmount` (inclusive) on `amount`. Every filter applies to the same transaction, so `{ start: '2026-01-01', minAmount: 100 }` is one gift of at least 100 this year.
 
 ## Metrics
 

package/transaction/core/segmentDefinitionClauses.js

@@ -1,5 +1,15 @@
 import { relativeDate, isValidDate } from '@engine9/input-tools';
 
+function blank(value) {
+  return value == null || String(value).trim() === '';
+}
+
+function amountOption(name, value) {
+  const n = Number(value);
+  if (!Number.isFinite(n)) throw new Error(`Invalid ${name}: ${value}`);
+  return n;
+}
+
 export const all = {
   title: 'Transactions',
   form: {
@@ -7,7 +17,9 @@ export const all = {
     type: 'object',
     properties: {
       plugin_id: {
-        type: 'string'
+        title: 'Plugin',
+        type: 'string',
+        resolveOptionsUsing: 'plugin'
       },
       start: {
         title: 'Start',
@@ -20,12 +32,22 @@ export const all = {
         type: 'string',
         description:
           'Optional; only include rows with ts < end (exclusive upper bound). Parsed with relativeDate; supports ISO datetimes or relative expressions.'
+      },
+      minAmount: {
+        title: 'Minimum amount',
+        type: 'number',
+        description: 'Optional; only include rows with amount >= minAmount.'
+      },
+      maxAmount: {
+        title: 'Maximum amount',
+        type: 'number',
+        description: 'Optional; only include rows with amount <= maxAmount.'
       }
     },
     required: []
   },
   optionsToEQL: (options) => {
-    const { plugin_id, start, end } = options;
+    const { plugin_id, start, end, minAmount, maxAmount } = options;
     const conditions = [];
     let text = 'Has any transactions';
     if (plugin_id) {
@@ -44,6 +66,16 @@ export const all = {
       conditions.push(`transaction.ts<'${e.toISOString()}'`);
       text += ` with ts < ${e.toISOString()}`;
     }
+    if (!blank(minAmount)) {
+      const min = amountOption('minAmount', minAmount);
+      conditions.push(`transaction.amount>=${min}`);
+      text += ` with amount >= ${min}`;
+    }
+    if (!blank(maxAmount)) {
+      const max = amountOption('maxAmount', maxAmount);
+      conditions.push(`transaction.amount<=${max}`);
+      text += ` with amount <= ${max}`;
+    }
     return {
       text,
       eql: {
@@ -60,28 +92,7 @@ export const all = {
     };
   }
 };
-export const minimum = {
-  title: 'Minimum transaction amount',
-  form: {
-    title: 'Minimum transaction amount',
-    type: 'object',
-    properties: {
-      amount: {
-        type: 'number'
-      },
-      filterType: {
-        type: 'string'
-      }
-    },
-    required: []
-  },
-  optionsToEQL: (options) => ({
-    table: 'transaction',
-    columns: ['person_id'],
-    conditions: [`amount>=${parseFloat(options.amount)}`]
-  })
-};
-export const segmentDefinitionClauses = { all, minimum };
+export const segmentDefinitionClauses = { all };
 export default {
   segmentDefinitionClauses
 };

package/transaction/core/segmentDefinitionClauses.test.js

@@ -0,0 +1,23 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { all, segmentDefinitionClauses } from './segmentDefinitionClauses.js';
+
+test('all is the only transaction clause', () => {
+  assert.deepEqual(Object.keys(segmentDefinitionClauses), ['all']);
+});
+
+test('amount bounds apply to the same transaction as the date window', () => {
+  const { text, eql } = all.optionsToEQL({ start: '2026-01-01', minAmount: 100, maxAmount: '500' });
+  assert.deepEqual(eql.conditions, [
+    "transaction.ts>='2026-01-01T00:00:00.000Z'",
+    'transaction.amount>=100',
+    'transaction.amount<=500'
+  ]);
+  assert.equal(text, 'Has any transactions with ts >= 2026-01-01T00:00:00.000Z with amount >= 100 with amount <= 500');
+});
+
+test('blank amounts are ignored and invalid amounts throw', () => {
+  assert.deepEqual(all.optionsToEQL({ minAmount: '', maxAmount: null }).eql.conditions, []);
+  assert.throws(() => all.optionsToEQL({ minAmount: 'abc' }), /Invalid minAmount: abc/);
+  assert.throws(() => all.optionsToEQL({ maxAmount: '1; drop table person' }), /Invalid maxAmount/);
+});