@engine9/schemas 1.9.0

package/person_hash/search.js

@@ -0,0 +1,86 @@
+function hashCondition(column, value) {
+  return {
+    type: 'EQUALS',
+    values: [
+      { ref: { column } },
+      { value: { value } }
+    ]
+  };
+}
+
+function hashColumnForValue(value, shaColumn, md5Column) {
+  const hex = String(value || '').trim().toLowerCase();
+  if (hex.length === 32) return md5Column;
+  return shaColumn;
+}
+
+export const emailHashes = {
+  title: 'Email hashes',
+  form: {
+    title: 'Email hashes',
+    type: 'object',
+    properties: {
+      emailHash: {
+        title: 'Email hash (SHA-256 or MD5 hex)',
+        type: 'string'
+      }
+    },
+    required: []
+  },
+  optionsToEQL(options) {
+    const { emailHash } = options;
+    const conditions = [];
+    let text = 'Has an email hash';
+    if (emailHash) {
+      const column = hashColumnForValue(emailHash, 'email_hash_v1', 'email_hash_md5');
+      conditions.push(hashCondition(column, String(emailHash).trim().toLowerCase()));
+      text = `Has email hash ${emailHash}`;
+    }
+    return {
+      text,
+      eql: {
+        table: 'person_hash_email',
+        columns: ['person_id'],
+        conditions
+      }
+    };
+  }
+};
+
+export const phoneHashes = {
+  title: 'Phone hashes',
+  form: {
+    title: 'Phone hashes',
+    type: 'object',
+    properties: {
+      phoneHash: {
+        title: 'Phone hash (SHA-256 or MD5 hex)',
+        type: 'string'
+      }
+    },
+    required: []
+  },
+  optionsToEQL(options) {
+    const { phoneHash } = options;
+    const conditions = [];
+    let text = 'Has a phone hash';
+    if (phoneHash) {
+      const column = hashColumnForValue(phoneHash, 'phone_hash_v1', 'phone_hash_md5');
+      conditions.push(hashCondition(column, String(phoneHash).trim().toLowerCase()));
+      text = `Has phone hash ${phoneHash}`;
+    }
+    return {
+      text,
+      eql: {
+        table: 'person_hash_phone',
+        columns: ['person_id'],
+        conditions
+      }
+    };
+  }
+};
+
+export default {
+  emailHashes,
+  phoneHashes
+};

package/person_hash/transforms/inbound/extract_identifiers.js

@@ -0,0 +1,70 @@
+import {
+  hashEmail,
+  hashPhone,
+  isBlankSha256,
+  resolvePhoneFromRow
+} from '../../hash.js';
+
+export const type = 'id';
+export const description =
+  'Extract email_hash_v1 / phone_hash_v1 identifiers from plaintext or existing hashes. Does not persist email or phone.';
+
+function pushIdentifier(row, path, typeName, value) {
+  if (!value || isBlankSha256(value)) return;
+  row.identifiers = row.identifiers || [];
+  if (row.identifiers.some((id) => id.type === typeName && id.value === value)) return;
+  row.identifiers.push({ path, type: typeName, value });
+}
+
+function applyEmailHashes(row) {
+  const fromEmail = hashEmail(row.email);
+  if (fromEmail) {
+    row.email = String(row.email).trim();
+    row.email_hash_v1 = fromEmail.email_hash_v1;
+    row.email_hash_md5 = fromEmail.email_hash_md5;
+    pushIdentifier(row, 'person_hash_email', 'email_hash_v1', fromEmail.email_hash_v1);
+    return;
+  }
+  if (row.email_hash_v1 && !isBlankSha256(row.email_hash_v1)) {
+    pushIdentifier(row, 'person_hash_email', 'email_hash_v1', row.email_hash_v1);
+  }
+}
+
+function applyPhoneHashes(row) {
+  const rawPhone = resolvePhoneFromRow(row);
+  if (!row.phone && rawPhone) {
+    row.phone = rawPhone;
+    row.phone_type = row.phone_type || 'Cell';
+  }
+  const fromPhone = hashPhone(row.phone);
+  if (fromPhone) {
+    row.phone = fromPhone.phone;
+    row.phone_hash_v1 = fromPhone.phone_hash_v1;
+    row.phone_hash_md5 = fromPhone.phone_hash_md5;
+    pushIdentifier(row, 'person_hash_phone', 'phone_hash_v1', fromPhone.phone_hash_v1);
+    return;
+  }
+  if (row.phone_hash_v1 && !isBlankSha256(row.phone_hash_v1)) {
+    pushIdentifier(row, 'person_hash_phone', 'phone_hash_v1', row.phone_hash_v1);
+    return;
+  }
+  if (row.phone && String(row.phone).replace(/[^0-9+]*/g, '').trim().length > 0) {
+    // Too short to hash — drop plaintext so it cannot be stored downstream.
+    row.phone = null;
+    row.phone_hash_v1 = null;
+    row.phone_hash_md5 = null;
+  }
+}
+
+export async function transform({ batch }) {
+  batch.forEach((row) => {
+    applyEmailHashes(row);
+    applyPhoneHashes(row);
+  });
+}
+
+export default {
+  type,
+  description,
+  transform
+};

package/person_hash/transforms/inbound/upsert_tables.js

@@ -0,0 +1,105 @@
+import { mergeIntoQueue } from '@engine9/input-tools';
+import { isBlankMd5, isBlankSha256 } from '../../hash.js';
+
+export const type = 'upsert';
+export const description =
+  'Upsert person_hash_email / person_hash_phone from hashes on the batch. Never writes email or phone plaintext.';
+
+export const bindings = {
+  databaseEmailHashes: {
+    path: 'sql.query',
+    options: { table: 'person_hash_email', lookup: ['email_hash_v1'] }
+  },
+  databasePhoneHashes: {
+    path: 'sql.query',
+    options: { table: 'person_hash_phone', lookup: ['phone_hash_v1'] }
+  },
+  tablesToUpsert: { path: 'sql.tables.upsert' }
+};
+
+function usableSha256(value) {
+  return Boolean(value) && !isBlankSha256(value);
+}
+
+function usableMd5(value) {
+  if (value == null || value === '') return '';
+  if (isBlankMd5(value)) return '';
+  return value;
+}
+
+function findExisting(rows, personId, hashColumn, hashValue) {
+  const matching = rows.filter((d) => d[hashColumn] === hashValue);
+  const forPerson = matching.filter((d) => personId && d.person_id === personId);
+  if (forPerson.length > 1) {
+    throw new Error(
+      `Cannot update ${hashColumn}, there are ${forPerson.length} database entries for person_id ${personId}`
+    );
+  }
+  return forPerson[0] || null;
+}
+
+function mergeHashRow(existing, incoming, md5Column) {
+  return {
+    ...existing,
+    ...incoming,
+    [md5Column]: incoming[md5Column] || existing[md5Column] || '',
+    source_input_id: existing.source_input_id ?? incoming.source_input_id
+  };
+}
+
+function pushHashRow(tablesToUpsert, table, existing, { person_id, hashColumn, hashValue, md5Column, md5Value, input_id }) {
+  const md5 = md5Value || existing?.[md5Column] || '';
+  tablesToUpsert[table] = tablesToUpsert[table] || [];
+  mergeIntoQueue(
+    tablesToUpsert[table],
+    {
+      person_id: existing?.person_id || person_id,
+      [hashColumn]: hashValue,
+      [md5Column]: md5,
+      source_input_id: existing?.source_input_id || input_id
+    },
+    {
+      keyFields: ['person_id', hashColumn],
+      merge: (a, b) => mergeHashRow(a, b, md5Column),
+      label: table
+    }
+  );
+}
+
+export async function transform({ batch, databaseEmailHashes, databasePhoneHashes, tablesToUpsert }) {
+  if (batch.length === 0) return;
+  const emailRows = databaseEmailHashes || [];
+  const phoneRows = databasePhoneHashes || [];
+  batch.forEach((row) => {
+    if (!row.person_id) return;
+    if (usableSha256(row.email_hash_v1)) {
+      const existing = findExisting(emailRows, row.person_id, 'email_hash_v1', row.email_hash_v1);
+      pushHashRow(tablesToUpsert, 'person_hash_email', existing, {
+        person_id: row.person_id,
+        hashColumn: 'email_hash_v1',
+        hashValue: row.email_hash_v1,
+        md5Column: 'email_hash_md5',
+        md5Value: usableMd5(row.email_hash_md5),
+        input_id: row.input_id
+      });
+    }
+    if (usableSha256(row.phone_hash_v1)) {
+      const existing = findExisting(phoneRows, row.person_id, 'phone_hash_v1', row.phone_hash_v1);
+      pushHashRow(tablesToUpsert, 'person_hash_phone', existing, {
+        person_id: row.person_id,
+        hashColumn: 'phone_hash_v1',
+        hashValue: row.phone_hash_v1,
+        md5Column: 'phone_hash_md5',
+        md5Value: usableMd5(row.phone_hash_md5),
+        input_id: row.input_id
+      });
+    }
+  });
+}
+
+export default {
+  bindings,
+  type,
+  description,
+  transform
+};

package/person_hash/transforms/outbound/appendEmailHash.js

@@ -0,0 +1,42 @@
+import { hashEmail } from '../../hash.js';
+
+export const description =
+  'Append email_hash_v1 / email_hash_md5 from plaintext email or person_hash_email. Does not append email.';
+
+export const bindings = {
+  hashes: {
+    path: 'sql.query',
+    options: {
+      table: 'person_hash_email',
+      columns: ['person_id', 'email_hash_v1', 'email_hash_md5'],
+      lookup: ['person_id']
+    }
+  }
+};
+
+export const transform = ({ batch, hashes = [] }) => {
+  const byPerson = {};
+  hashes.forEach((row) => {
+    if (!byPerson[row.person_id]) byPerson[row.person_id] = row;
+  });
+  batch.forEach((row) => {
+    if (!row.email_hash_v1) {
+      const fromEmail = hashEmail(row.email);
+      if (fromEmail) {
+        row.email_hash_v1 = fromEmail.email_hash_v1;
+        row.email_hash_md5 = row.email_hash_md5 || fromEmail.email_hash_md5;
+      }
+    }
+    const stored = byPerson[row.person_id];
+    if (stored) {
+      row.email_hash_v1 = row.email_hash_v1 || stored.email_hash_v1 || null;
+      row.email_hash_md5 = row.email_hash_md5 || stored.email_hash_md5 || null;
+    }
+  });
+};
+
+export default {
+  description,
+  bindings,
+  transform
+};

package/person_hash/transforms/outbound/appendPhoneHash.js

@@ -0,0 +1,42 @@
+import { hashPhone } from '../../hash.js';
+
+export const description =
+  'Append phone_hash_v1 / phone_hash_md5 from plaintext phone or person_hash_phone. Does not append phone.';
+
+export const bindings = {
+  hashes: {
+    path: 'sql.query',
+    options: {
+      table: 'person_hash_phone',
+      columns: ['person_id', 'phone_hash_v1', 'phone_hash_md5'],
+      lookup: ['person_id']
+    }
+  }
+};
+
+export const transform = ({ batch, hashes = [] }) => {
+  const byPerson = {};
+  hashes.forEach((row) => {
+    if (!byPerson[row.person_id]) byPerson[row.person_id] = row;
+  });
+  batch.forEach((row) => {
+    if (!row.phone_hash_v1) {
+      const fromPhone = hashPhone(row.phone);
+      if (fromPhone) {
+        row.phone_hash_v1 = fromPhone.phone_hash_v1;
+        row.phone_hash_md5 = row.phone_hash_md5 || fromPhone.phone_hash_md5;
+      }
+    }
+    const stored = byPerson[row.person_id];
+    if (stored) {
+      row.phone_hash_v1 = row.phone_hash_v1 || stored.phone_hash_v1 || null;
+      row.phone_hash_md5 = row.phone_hash_md5 || stored.phone_hash_md5 || null;
+    }
+  });
+};
+
+export default {
+  description,
+  bindings,
+  transform
+};

package/person_hash/ui.console.json5

@@ -0,0 +1,80 @@
+{
+  routes: {
+    'person/:id/*': {
+      components: {
+        sidebar: {
+          items: [
+            {
+              component: 'Link',
+              properties: {
+                label: 'Email hashes',
+                to: 'email_hashes'
+              }
+            },
+            {
+              component: 'Link',
+              properties: {
+                label: 'Phone hashes',
+                to: 'phone_hashes'
+              }
+            }
+          ]
+        },
+        main: [
+          {
+            path: 'email_hashes',
+            label: 'Email hashes',
+            component: 'RecordTable',
+            properties: {
+              table: 'person_hash_email',
+              conditions: [{ eql: 'person_id={{id}}' }],
+              columns: [
+                {
+                  title: 'SHA-256',
+                  template: '{{email_hash_v1}}',
+                  width: '45%'
+                },
+                {
+                  title: 'MD5',
+                  template: '{{email_hash_md5}}',
+                  width: '35%'
+                },
+                {
+                  title: 'Modified',
+                  template: '{{date modified_at}}',
+                  width: '20%'
+                }
+              ]
+            }
+          },
+          {
+            path: 'phone_hashes',
+            label: 'Phone hashes',
+            component: 'RecordTable',
+            properties: {
+              table: 'person_hash_phone',
+              conditions: [{ eql: 'person_id={{id}}' }],
+              columns: [
+                {
+                  title: 'SHA-256',
+                  template: '{{phone_hash_v1}}',
+                  width: '45%'
+                },
+                {
+                  title: 'MD5',
+                  template: '{{phone_hash_md5}}',
+                  width: '35%'
+                },
+                {
+                  title: 'Modified',
+                  template: '{{date modified_at}}',
+                  width: '20%'
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  }
+}

package/person_phone/README.md

@@ -0,0 +1,79 @@
+# Person Phone Schema Plugin
+
+The `@engine9/schemas/person_phone` plugin stores phone numbers for people and provides the common phone-related behavior used by imports, matching, searches, segments, and exports.
+
+It depends on `@engine9/schemas/person` because every phone row belongs to a person through `person_id`.
+
+## Data Model
+
+The plugin creates the `person_phone` table. A person can have multiple phone rows, but each `(phone, person_id)` pair is unique.
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `id` | `id` | Primary row identifier. |
+| `person_id` | `person_id` | Person that owns the phone number. |
+| `phone_type` | `string` | Phone category. Allowed values are `Personal`, `Cell`, `Home`, `Work`, `Fax`, and `Other`. Defaults to `Personal`. |
+| `phone` | `string` | Phone number. Inbound transforms keep digits and `+`, and prefix US 10-digit numbers with `+1`. |
+| `preference_order` | `int` | Sort order for preferred phones, where `0` is first. Defaults to `0`. |
+| `sms_status` | `string` | SMS subscription state. Allowed values are `Not Subscribed`, `Subscribed`, `Unsubscribed`, and `Bouncing`. Defaults to `Not Subscribed`. |
+| `sms_deliverability_score` | `int` | SMS deliverability score on a 1–100 scale (`100` is most deliverable). Defaults to `100`. |
+| `call_status` | `string` | Voice-call subscription state. Same allowed values as `sms_status`. Defaults to `Not Subscribed`. |
+| `remote_phone_id` | `string` | Remote system phone record id when available. |
+| `phone_hash_v1` | `hash` | SHA-256 hash of the normalized phone number. Used as an identifier and for privacy-preserving exports. |
+| `source_input_id` | `foreign_uuid` | Input that originally created the phone row. Existing rows keep their original source when updated. |
+| `created_at` | `created_at` | Timestamp when the row was created. |
+| `modified_at` | `modified_at` | Timestamp when the row was last modified. |
+
+Indexes:
+
+- `person_id`
+- unique `(phone, person_id)`
+- `remote_phone_id`
+
+## Inbound Behavior
+
+The inbound `id` transform extracts a `phone_hash_v1` identifier from either `phone` or an existing `phone_hash_v1` field.
+
+- `cell`, `mobile`, and `mobile_phone` fill `phone` when it is empty, and default `phone_type` to `Cell`.
+- A bare `phone` with no type defaults to `Home`.
+- Numbers shorter than 8 digits after cleaning are dropped.
+- US numbers without a `+` prefix are normalized (`10` digits become `+1…`; `11` digits starting with `1` become `+1…`).
+
+The inbound `upsert` transform writes rows to `person_phone`.
+
+- Existing rows for the same `person_id` and phone are updated instead of duplicated.
+- Newly inserted phones default to `sms_status` `Not Subscribed` when no status is provided.
+- SMS unsubscribe-like entries set `sms_status` to `Unsubscribed` when no status was provided. This includes `entry_type` values `SMS_UNSUBSCRIBE` and `SMS_SPAM`, plus `entry_type_id` values `34` and `38`.
+
+## Outbound Behavior
+
+`appendPhoneHash` hashes the row's normalized `phone` into `phone_hash_v1` when the hash is not already present.
+
+## Search
+
+The `phones` search lets users find people who have a phone number, optionally filtered by:
+
+- `smsStatus`, using the same values as `sms_status`
+- `callStatus`, using the same values as `call_status`
+- `phoneMatch`, matched against the `phone` column with `LIKE`
+
+## Segments
+
+Predefined audiences shipped with this schema plugin. On install, each key becomes a `segment` row whose definition path is `@engine9/schemas/person_phone:segments:<key>`.
+
+### Textable People
+
+| | |
+| --- | --- |
+| **Key** | `textable` |
+| **Definition path** | `@engine9/schemas/person_phone:segments:textable` |
+| **Who is included** | People with at least one phone whose `sms_status` is `Subscribed` |
+| **Who is excluded** | People whose phones are only `Not Subscribed`, `Unsubscribed`, or `Bouncing` for SMS |
+| **How it is built** | Direct `person_phone` condition `sms_status = Subscribed` |
+| **Universe** | None — membership is current `person_phone` rows, not a time window or message input set |
+
+`call_status` is not part of this segment. A person can be textable without being subscribed for voice calls.
+
+## UI
+
+The console UI adds a `Phones` tab on person records. It displays each person's phone rows with the number and type.

package/person_phone/index.js

@@ -0,0 +1,33 @@
+import schema from './schema.js';
+import search from './search.js';
+import segments from './segments.js';
+import extractPhoneHashes from './transforms/inbound/extract_identifiers.js';
+import upsertPersonPhone from './transforms/inbound/upsert_tables.js';
+import appendPhoneHash from './transforms/outbound/appendPhoneHash.js';
+const metadata = {
+  name: '@engine9/schemas/person_phone',
+  dependencies: {
+    '@engine9/schemas/person': '>=1.9.0'
+  },
+  // Inbound people pipeline slots -> transform export keys (woven by core when installed)
+  inbound: {
+    id: ['extractPhoneHashes'],
+    upsert: ['upsertPersonPhone']
+  }
+};
+export const transforms = {
+  extractPhoneHashes,
+  upsertPersonPhone,
+  appendPhoneHash
+};
+export { metadata };
+export { schema };
+export { search };
+export { segments };
+export default {
+  metadata,
+  schema,
+  transforms,
+  search,
+  segments
+};

package/person_phone/schema.js

@@ -0,0 +1,58 @@
+export const tables = [
+  {
+    name: 'person_phone',
+    columns: {
+      id: 'id',
+      person_id: 'person_id',
+      phone_type: {
+        type: 'string',
+        nullable: false,
+        default_value: 'Personal',
+        values: ['', 'Personal', 'Cell', 'Home', 'Work', 'Fax', 'Other']
+      },
+      phone: 'string',
+      preference_order: {
+        type: 'int',
+        nullable: false,
+        default_value: 0,
+        description: 'Order in the preference stack, 0 is first'
+      },
+      sms_status: {
+        type: 'string',
+        nullable: false,
+        default_value: 'Not Subscribed',
+        values: ['Not Subscribed', 'Subscribed', 'Unsubscribed', 'Bouncing']
+      },
+      sms_deliverability_score: {
+        type: 'int',
+        description:
+          'SMS deliverability score on a 1–100 scale (100 = most deliverable / verified cell). Use lower values for weaker confidence; 0 may appear from legacy sources meaning undeliverable.',
+        nullable: false,
+        default_value: 100
+      },
+      call_status: {
+        type: 'string',
+        nullable: false,
+        default_value: 'Not Subscribed',
+        values: ['Not Subscribed', 'Subscribed', 'Unsubscribed', 'Bouncing']
+      },
+      remote_phone_id: {
+        type: 'string',
+        nullable: true,
+        description: 'Remote system phone record id when available'
+      },
+      phone_hash_v1: 'hash',
+      source_input_id: 'foreign_uuid',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'person_id' },
+      { columns: ['phone', 'person_id'], unique: true },
+      { columns: 'remote_phone_id' }
+    ]
+  }
+];
+export default {
+  tables
+};

package/person_phone/search.js

@@ -0,0 +1,79 @@
+export const phones = {
+  title: 'Phones',
+  form: {
+    title: 'Phones',
+    type: 'object',
+    properties: {
+      phoneMatch: {
+        type: 'string'
+      },
+      smsStatus: {
+        type: 'string',
+        enum: ['Not Subscribed', 'Subscribed', 'Unsubscribed', 'Bouncing']
+      },
+      callStatus: {
+        type: 'string',
+        enum: ['Not Subscribed', 'Subscribed', 'Unsubscribed', 'Bouncing']
+      }
+    },
+    required: []
+  },
+  /* map from provided user data into an EQL structure */
+  optionsToEQL(options) {
+    const { phoneMatch, smsStatus, callStatus } = options;
+    const conditions = [];
+    let text = [];
+    if (smsStatus) {
+      text.push(`A SMS ${smsStatus.toLowerCase()} phone`);
+      conditions.push({
+        type: 'EQUALS',
+        values: [
+          {
+            ref: { column: 'sms_status' }
+          },
+          {
+            value: { value: smsStatus }
+          }
+        ]
+      });
+    }
+    if (callStatus) {
+      text.push(`A voice ${callStatus.toLowerCase()} phone`);
+      conditions.push({
+        type: 'EQUALS',
+        values: [
+          {
+            ref: { column: 'call_status' }
+          },
+          {
+            value: { value: callStatus }
+          }
+        ]
+      });
+    }
+    if (phoneMatch) {
+      text.push(`A phone that matches ${phoneMatch}`);
+      conditions.push({
+        type: 'LIKE',
+        values: [
+          {
+            ref: { column: 'phone' }
+          },
+          {
+            value: { value: phoneMatch }
+          }
+        ]
+      });
+    }
+    return {
+      text,
+      eql: {
+        table: 'person_phone',
+        conditions
+      }
+    };
+  }
+};
+export default {
+  phones
+};