@engine9/schemas 1.9.0

package/person_phone/segments.js

@@ -0,0 +1,26 @@
+export const textable = {
+  name: 'Textable People',
+  search: {
+    and: [
+      {
+        table: 'person_phone',
+        conditions: [
+          {
+            type: 'EQUALS',
+            values: [
+              {
+                ref: { column: 'sms_status' }
+              },
+              {
+                value: { value: 'Subscribed' }
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+};
+export default {
+  textable
+};

package/person_phone/transforms/inbound/extract_identifiers.js

@@ -0,0 +1,59 @@
+import nodecrypto from 'node:crypto';
+const { createHash } = nodecrypto;
+export const type = 'id';
+const blankHash = createHash('sha256').update('').digest('hex');
+const MIN_PHONE_LENGTH = 8;
+export async function transform({ batch }) {
+  const ids = [];
+  let hasPhoneHash = false;
+  batch.forEach((o) => {
+    o.identifiers = o.identifiers || [];
+    // Prefer mobile_phone / cell aliases when phone is empty
+    if (!o.phone && (o.cell || o.mobile || o.mobile_phone)) {
+      o.phone = o.cell || o.mobile || o.mobile_phone;
+      o.phone_type = o.phone_type || 'Cell';
+    } else if (o.phone && !o.phone_type) {
+      // Bare phone with no type defaults to Home (mobile should already be prioritized upstream)
+      o.phone_type = 'Home';
+    }
+
+    let phone = (o.phone || '').replace(/[^0-9+]*/g, '').trim();
+    // clean and add a plus
+    if (phone && phone.length >= MIN_PHONE_LENGTH) {
+      // clean us based phones.  If it's already prefixed with '+' then assume
+      // it's intentional
+      if (phone.indexOf('+') < 0) {
+        if (phone.length === 10) phone = `+1${phone}`;
+        else if (phone.length === 11 && phone.slice(0, 1) === '1') phone = `+${phone}`;
+      }
+      // no secret or createHmac for this use case
+      // it's basically a shared id hashing setup, so a secret will be exposed anyhow
+      const value = createHash('sha256').update(phone).digest('hex');
+      // Push all identifiers, later in the pipeline the priority will be determined
+      o.identifiers.push({
+        path: 'person_phone',
+        type: 'phone_hash_v1',
+        value
+      });
+      o.phone = phone;
+      o.phone_hash_v1 = value;
+      hasPhoneHash = true;
+    } else if (phone && phone.length > 0 && phone.length < MIN_PHONE_LENGTH) {
+      o.phone = null;
+      o.phone_hash_v1 = null;
+    } else if (o.phone_hash_v1 && o.phone_hash_v1 != blankHash) {
+      o.identifiers.push({
+        path: 'person_phone',
+        type: 'phone_hash_v1',
+        value: o.phone_hash_v1
+      });
+      hasPhoneHash = true;
+    }
+  });
+  if (hasPhoneHash) batch.forEach((d) => (d.phone_hash_v1 = d.phone_hash_v1 || null));
+  return ids;
+}
+export default {
+  type,
+  transform
+};

package/person_phone/transforms/inbound/upsert_tables.js

@@ -0,0 +1,119 @@
+import { mergeIntoQueue } from '@engine9/input-tools';
+
+export const type = 'upsert';
+const MIN_PHONE_LENGTH = 8;
+function cleanedPhoneLength(phone) {
+  return String(phone || '')
+    .replace(/[^0-9+]*/g, '')
+    .trim().length;
+}
+export const bindings = {
+  tablesToUpsert: { path: 'sql.tables.upsert' },
+  databasePhones: { path: 'sql.query', options: { table: 'person_phone', lookup: ['phone'] } }
+};
+
+function mergePersonPhone(existing, incoming) {
+  return {
+    ...existing,
+    ...incoming,
+    id: existing.id ?? incoming.id ?? null,
+    phone: existing.phone || incoming.phone,
+    person_id: existing.person_id ?? incoming.person_id,
+    source_input_id: existing.source_input_id ?? incoming.source_input_id,
+    // These columns can be optional in inbound payloads, and `upsertArray`
+    // is strict about consistent object-key presence across samples.
+    // Normalize them so every queued object has the same key set.
+    phone_hash_v1: incoming.phone_hash_v1 ?? existing.phone_hash_v1 ?? '',
+    remote_phone_id: incoming.remote_phone_id ?? existing.remote_phone_id ?? null,
+    sms_deliverability_score: incoming.sms_deliverability_score ?? existing.sms_deliverability_score ?? 100,
+    call_status: incoming.call_status ?? existing.call_status ?? 'Not Subscribed',
+    sms_status: incoming.sms_status ?? existing.sms_status,
+    phone_type: incoming.phone_type ?? existing.phone_type,
+    preference_order:
+      incoming.preference_order !== undefined && incoming.preference_order !== null
+        ? incoming.preference_order
+        : existing.preference_order
+  };
+}
+
+function queuePersonPhone(tablesToUpsert, row) {
+  tablesToUpsert.person_phone = tablesToUpsert.person_phone || [];
+  const normalizedRow = {
+    ...row,
+    phone_hash_v1: row.phone_hash_v1 ?? '',
+    remote_phone_id: row.remote_phone_id ?? null,
+    sms_deliverability_score: row.sms_deliverability_score ?? 100,
+    call_status: row.call_status ?? 'Not Subscribed'
+  };
+  mergeIntoQueue(tablesToUpsert.person_phone, normalizedRow, {
+    keyFields: ['phone', 'person_id'],
+    merge: mergePersonPhone,
+    label: 'person_phone'
+  });
+}
+
+export async function transform({ batch, tablesToUpsert, databasePhones }) {
+  batch.forEach((o) => {
+    if (!o.phone || cleanedPhoneLength(o.phone) < MIN_PHONE_LENGTH) return;
+    // phone should be already cleaned in extract ids
+    const matchingPhones = databasePhones.filter((d) => d.phone === o.phone);
+    const personPhones = matchingPhones.filter((em) => o.person_id && em.person_id === o.person_id);
+    if (personPhones.length > 1) {
+      throw new Error(
+        `Cannot update phone, there are 2 database entries for person_id ${o.person_id} with phone ${o.phone}`
+      );
+    }
+    if (
+      o.entry_type === 'SMS_UNSUBSCRIBE' ||
+      o.entry_type === 'SMS_SPAM' ||
+      o.entry_type_id === 34 ||
+      o.entry_type_id === 38
+    ) {
+      if (!o.sms_status) o.sms_status = 'Unsubscribed';
+    }
+    const sms_status = o.sms_status || personPhones[0]?.sms_status || 'Not Subscribed';
+    const phone_type = o.phone_type || personPhones[0]?.phone_type || 'Personal';
+    const preference_order =
+      o.preference_order !== undefined && o.preference_order !== null
+        ? o.preference_order
+        : (personPhones[0]?.preference_order ?? 0);
+
+    if (o.call_status !== undefined && !o.call_status) o.call_status = 'Not Subscribed';
+
+    const { id, ...rest } = o;
+    if (id) {
+      //this is undoubtedly NOT the ID of the record
+    }
+    if (personPhones[0]) {
+      if (!personPhones[0].source_input_id)
+        throw new Error('Invalid source_input_id for existing person_phone record:' + JSON.stringify(personPhones[0]));
+      queuePersonPhone(tablesToUpsert, {
+        ...rest,
+        id: personPhones[0].id,
+        person_id: personPhones[0].person_id,
+        phone: personPhones[0].phone,
+        source_input_id: personPhones[0].source_input_id,
+        sms_status,
+        phone_type,
+        preference_order
+      });
+    } else {
+      queuePersonPhone(tablesToUpsert, {
+        ...rest,
+        id: null,
+        person_id: o.person_id,
+        phone: o.phone,
+        source_input_id: o.input_id,
+        sms_status,
+        phone_type,
+        preference_order
+      });
+    }
+  });
+  return batch;
+}
+export default {
+  type,
+  bindings,
+  transform
+};

package/person_phone/transforms/outbound/appendPhoneHash.js

@@ -0,0 +1,37 @@
+import nodecrypto from 'node:crypto';
+const { createHash } = nodecrypto;
+export const type = 'id';
+const MIN_PHONE_LENGTH = 8;
+export async function transform({ batch }) {
+  const ids = [];
+  batch.forEach((e) => {
+    e.phone_hash_v1 = e.phone_hash_v1 || '';
+    if (e.phone_hash_v1) {
+      // do nothing
+    } else if (e.phone) {
+      // clean and add a plus
+      let phone = e.phone.replace(/[^0-9+]*/g, '').trim();
+      if (phone.length >= MIN_PHONE_LENGTH) {
+        // clean us based phones.  If it's already prefixed with '+' then assume
+        // it's intentional
+        if (phone.indexOf('+') < 0) {
+          if (phone.length === 10) phone = `+1${phone}`;
+          else if (phone.length === 11 && phone.slice(0, 1) === '1') phone = `+${phone}`;
+        }
+        // no secret or createHmac for this use case
+        // it's basically a shared id hashing setup, so a secret will be exposed anyhow
+        const value = createHash('sha256').update(phone).digest('hex');
+        e.phone = phone;
+        e.phone_hash_v1 = value;
+      } else {
+        e.phone = null;
+        e.phone_hash_v1 = null;
+      }
+    }
+  });
+  return ids;
+}
+export default {
+  type,
+  transform
+};

package/person_phone/ui.console.json5

@@ -0,0 +1,31 @@
+{
+  routes: {
+    'person/:id/*': {
+      components: {
+        sidebar: {
+          items2: [
+            {
+              component: 'Link',
+              properties: {
+                label: 'Phones',
+                to: 'phones'
+              }
+            }
+          ]
+        },
+        main: [
+          {
+            path: 'phones',
+            label: 'Phones',
+            component: 'RecordDisplay',
+            properties: {
+              table: 'person_phone',
+              conditions: [{ eql: 'person_id={{person_id}}' }],
+              rows: ['{{phone}}', '{{phone_type}}']
+            }
+          }
+        ]
+      }
+    }
+  }
+}

package/person_phone/upsert.test.js

@@ -0,0 +1,50 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import upsertPhone from './transforms/inbound/upsert_tables.js';
+
+describe('person_phone upsert_tables', () => {
+  it('normalizes optional schema columns so every queued row has stable keys', async () => {
+    const tablesToUpsert = {};
+    await upsertPhone.transform({
+      batch: [
+        {
+          person_id: 1,
+          phone: '+12025550143',
+          input_id: 'input-a',
+          sms_status: 'Subscribed',
+          phone_type: 'Cell',
+          preference_order: 2,
+          phone_hash_v1: 'hash-a',
+          remote_phone_id: 'remote-1',
+          sms_deliverability_score: 7,
+          call_status: 'Subscribed'
+        },
+        {
+          person_id: 2,
+          phone: '+12025550144',
+          input_id: 'input-b',
+          sms_status: 'Subscribed',
+          phone_type: 'Cell',
+          preference_order: 2
+          // intentionally omit phone_hash_v1 / remote_phone_id / sms_deliverability_score / call_status
+        }
+      ],
+      databasePhones: [],
+      tablesToUpsert
+    });
+
+    assert.equal(tablesToUpsert.person_phone.length, 2);
+    const rowMissing = tablesToUpsert.person_phone.find((r) => r.person_id === 2);
+    assert.ok(rowMissing, 'expected queued row for person_id 2');
+
+    assert.ok('phone_hash_v1' in rowMissing, 'phone_hash_v1 key must exist');
+    assert.ok('remote_phone_id' in rowMissing, 'remote_phone_id key must exist');
+    assert.ok('sms_deliverability_score' in rowMissing, 'sms_deliverability_score key must exist');
+    assert.ok('call_status' in rowMissing, 'call_status key must exist');
+
+    assert.equal(rowMissing.phone_hash_v1, '', 'missing phone_hash_v1 normalizes to blank string');
+    assert.equal(rowMissing.remote_phone_id, null, 'missing remote_phone_id normalizes to null');
+    assert.equal(rowMissing.sms_deliverability_score, 100, 'missing sms_deliverability_score normalizes to 100');
+    assert.equal(rowMissing.call_status, 'Not Subscribed', 'missing call_status normalizes to Not Subscribed');
+  });
+});

package/person_remote/index.js

@@ -0,0 +1,58 @@
+import schema from './schema.js';
+import metrics from './metrics.js';
+import extractRemotePersonIds from './transforms/inbound/extract_identifiers.js';
+import upsertPersonRemote from './transforms/inbound/upsert_tables.js';
+import appendRemotePersonId from './transforms/outbound/appendRemotePersonId.js';
+const metadata = {
+  name: '@engine9/schemas/person_remote',
+  dependencies: {
+    '@engine9/schemas/person': '>=1.9.0'
+  },
+  // Inbound people pipeline slots -> transform export keys (woven by core when installed)
+  inbound: {
+    id: ['extractRemotePersonIds'],
+    upsert: ['upsertPersonRemote']
+  }
+};
+export const search = {
+  all: {
+    title: 'Remote people',
+    name: 'All remote people',
+    form: {
+      title: 'Remote people',
+      type: 'object',
+      properties: {
+        pluginId: {
+          title: 'Plugin ID',
+          type: 'string'
+        }
+      },
+      required: []
+    },
+    optionsToEQL: (options) => ({
+      table: 'person_remote',
+      columns: ['person_id'],
+      joins: [
+        {
+          table: 'input',
+          join_eql: `person_remote.source_input_id=input.id AND input.plugin_id='${options.pluginId}'`
+        }
+      ]
+    })
+  }
+};
+export const transforms = {
+  extractRemotePersonIds,
+  upsertPersonRemote,
+  appendRemotePersonId
+};
+export { metadata };
+export { schema };
+export { metrics };
+export default {
+  metadata,
+  schema,
+  metrics,
+  search,
+  transforms
+};

package/person_remote/metrics.js

@@ -0,0 +1,17 @@
+export const people = ({ plugin: { tablePrefix } }) => {
+  return {
+    label: 'Remote People',
+    description: '',
+    eql: {
+      table: tablePrefix + 'person',
+      columns: [
+        { eql: 'min(date_created)', name: 'first_date' },
+        { eql: 'max(date_created)', name: 'last_date' },
+        { eql: 'count', name: 'records' }
+      ]
+    }
+  };
+};
+export default {
+  people
+};

package/person_remote/schema.js

@@ -0,0 +1,21 @@
+export const tables = [
+  {
+    name: 'person_remote',
+    columns: {
+      id: 'id',
+      person_id: 'person_id',
+      remote_person_id: 'string',
+      source_input_id: 'foreign_uuid',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'person_id' },
+      { columns: 'remote_person_id' },
+      { columns: ['source_input_id', 'remote_person_id', 'person_id'], unique: true }
+    ]
+  }
+];
+export default {
+  tables
+};

package/person_remote/transforms/inbound/extract_identifiers.js

@@ -0,0 +1,34 @@
+import { uuidIsValid } from '@engine9/input-tools';
+export const type = 'id';
+export const description = 'Extract the remote_person_id for use in lookups';
+export async function transform({ batch, pluginId }) {
+  const ids = [];
+  batch.forEach((o) => {
+    o.identifiers = o.identifiers || [];
+    if (o.remote_person_id) {
+      const parts = String(o.remote_person_id).split('.');
+      let value = null;
+      if (parts.length > 1 && uuidIsValid(parts[0])) {
+        // There may already be a pluginId prefixed with this
+        value = String(o.remote_person_id).toLowerCase();
+      } else {
+        // Otherwise prefix it with the current plugin
+        if (!pluginId) throw new Error('A remote_person_id requires a pluginId as well, not found in options');
+        if (!uuidIsValid(pluginId))
+          throw new Error("A pluginId was specified, but it's not a UUID. Perhaps you meant remote_plugin_id?");
+        value = `${pluginId}.${o.remote_person_id}`.toLowerCase(); // Must be case insensitive, because the unique indexes are as well
+      }
+      o.identifiers.push({
+        path: 'person_remote',
+        type: 'remote_person_id',
+        value
+      });
+    }
+  });
+  return ids;
+}
+export default {
+  type,
+  description,
+  transform
+};

package/person_remote/transforms/inbound/upsert_tables.js

@@ -0,0 +1,92 @@
+import { mergeIntoQueue } from '@engine9/input-tools';
+
+export const bindings = ({ pluginId } = {}) => {
+  if (!pluginId) {
+    throw new Error('pluginId is required for person_remote upsert lookup');
+  }
+  // Raw eql condition — EQUALS+string ref gets an illegal AS alias in WHERE
+  const safePluginId = String(pluginId).replace(/'/g, "''");
+  return {
+    databaseRemotes: {
+      path: 'sql.query',
+      options: {
+        table: 'person_remote',
+        columns: ['*'],
+        lookup: ['remote_person_id'],
+        joins: [
+          {
+            table: 'input',
+            join_eql: 'person_remote.source_input_id=input.id'
+          }
+        ],
+        conditions: [{ eql: `input.plugin_id='${safePluginId}'` }]
+      }
+    },
+    tablesToUpsert: { path: 'sql.tables.upsert' }
+  };
+};
+export const type = 'upsert';
+
+function mergePersonRemote(existing, incoming) {
+  return {
+    ...existing,
+    ...incoming,
+    id: existing.id ?? incoming.id ?? null,
+    person_id: existing.person_id ?? incoming.person_id,
+    remote_person_id: existing.remote_person_id || incoming.remote_person_id,
+    source_input_id: existing.source_input_id ?? incoming.source_input_id
+  };
+}
+
+function queuePersonRemote(tablesToUpsert, row) {
+  tablesToUpsert.person_remote = tablesToUpsert.person_remote || [];
+  mergeIntoQueue(tablesToUpsert.person_remote, row, {
+    keyFields: ['source_input_id', 'remote_person_id', 'person_id'],
+    merge: mergePersonRemote,
+    label: 'person_remote'
+  });
+}
+
+export async function transform({ batch, databaseRemotes, tablesToUpsert }) {
+  batch.forEach((o) => {
+    if (!o.remote_person_id || !o.person_id) return;
+    const remotePersonId = String(o.remote_person_id).trim();
+    if (!remotePersonId) return;
+    // databaseRemotes is already scoped to this plugin_id via the input join
+    const matchingRemotes = databaseRemotes.filter((d) => d.remote_person_id === remotePersonId);
+    const personRemotes = matchingRemotes.filter((d) => d.person_id === o.person_id);
+    const existing =
+      personRemotes.find((d) => d.source_input_id === o.input_id) || personRemotes[0] || null;
+    const { id, ...rest } = o;
+    if (id) {
+      // this is undoubtedly NOT the ID of the person_remote record
+    }
+    if (existing) {
+      if (!existing.source_input_id) {
+        throw new Error(
+          'Invalid source_input_id for existing person_remote record:' + JSON.stringify(existing)
+        );
+      }
+      queuePersonRemote(tablesToUpsert, {
+        ...rest,
+        id: existing.id,
+        person_id: existing.person_id,
+        remote_person_id: existing.remote_person_id,
+        source_input_id: existing.source_input_id
+      });
+    } else {
+      queuePersonRemote(tablesToUpsert, {
+        ...rest,
+        id: null,
+        person_id: o.person_id,
+        remote_person_id: remotePersonId,
+        source_input_id: o.input_id
+      });
+    }
+  });
+}
+export default {
+  bindings,
+  type,
+  transform
+};

package/person_remote/transforms/outbound/appendRemotePersonId.js

@@ -0,0 +1,59 @@
+export const description =
+  'Set a remote person id field on each row (default remote_person_id). Use options.pluginId or options.pluginIds (first match wins in list order) and optional outputField.';
+
+export const bindings = {
+  remoteIds: {
+    path: 'sql.query',
+    options: {
+      table: 'person_remote',
+      columns: [
+        'person_id',
+        'remote_person_id',
+        { eql: 'input.id', name: 'input_id' },
+        { eql: 'input.plugin_id', name: 'plugin_id' }
+      ],
+      lookup: ['person_id'],
+      joins: [
+        {
+          table: 'input',
+          join_eql: 'person_remote.source_input_id=input.id'
+        }
+      ]
+    }
+  }
+};
+
+export const transform = (opts) => {
+  const { batch, remoteIds, options = {} } = opts;
+  const { pluginId, pluginIds, outputField = 'remote_person_id' } = options;
+  const orderedPluginIds = pluginIds && pluginIds.length > 0 ? pluginIds : pluginId ? [pluginId] : [];
+  if (orderedPluginIds.length === 0) {
+    throw new Error('appendRemotePersonId requires options.pluginId or options.pluginIds');
+  }
+  const byPerson = {};
+  for (const b of remoteIds) {
+    if (!orderedPluginIds.includes(b.plugin_id)) continue;
+    const v = b.remote_person_id;
+    if (!byPerson[b.person_id]) byPerson[b.person_id] = {};
+    if (!byPerson[b.person_id][b.plugin_id]) byPerson[b.person_id][b.plugin_id] = v;
+  }
+  batch.forEach((data) => {
+    let val = null;
+    const per = byPerson[data.person_id];
+    if (per) {
+      for (const pid of orderedPluginIds) {
+        if (per[pid]) {
+          val = per[pid];
+          break;
+        }
+      }
+    }
+    data[outputField] = val || null;
+  });
+};
+
+export default {
+  description,
+  bindings,
+  transform
+};

package/plugin/index.js

@@ -0,0 +1,13 @@
+import schema from './schema.js';
+import { settings } from './settings.js';
+const metadata = {
+  name: '@engine9/schemas/plugin'
+};
+export { metadata };
+export { schema };
+export { settings };
+export default {
+  metadata,
+  schema,
+  settings
+};