@engine9/schemas 1.9.0

package/person_address/transforms/outbound/appendAddress.js

@@ -0,0 +1,87 @@
+export const description = 'Append the best postal address (all person_address columns)';
+
+export const bindings = {
+  addresses: {
+    path: 'sql.query',
+    options: {
+      table: 'person_address',
+      columns: [
+        'id',
+        'person_id',
+        'name',
+        'type',
+        'status',
+        'street_1',
+        'street_2',
+        'street_3',
+        'city',
+        'region',
+        'postal_code',
+        'country',
+        'subscription_status',
+        'deliverability_score',
+        'preference_order',
+        'is_best_address',
+        'created_at',
+        'modified_at',
+        'source_input_id'
+      ],
+      lookup: ['person_id'],
+      conditions: []
+    }
+  }
+};
+
+function truthyBest(v) {
+  return v === true || v === 1 || v === '1' || v === 't';
+}
+
+function pickAddressRow(rows) {
+  if (!rows?.length) return null;
+  const best = rows.find((r) => truthyBest(r.is_best_address));
+  if (best) return best;
+  return rows.slice().sort((a, b) => (Number(a.preference_order) || 0) - (Number(b.preference_order) || 0))[0];
+}
+
+export const transform = ({ batch, addresses, options = {} }) => {
+  const { subscriptionStatus } = options;
+  let filter = () => true;
+  if (subscriptionStatus) filter = (d) => d.subscription_status === subscriptionStatus;
+
+  const filtered = addresses.filter(filter);
+  const byPerson = {};
+  for (const a of filtered) {
+    if (!byPerson[a.person_id]) byPerson[a.person_id] = [];
+    byPerson[a.person_id].push(a);
+  }
+
+  batch.forEach((data) => {
+    const row = pickAddressRow(byPerson[data.person_id]);
+    if (!row) return;
+
+    data.person_address_id = data.person_address_id ?? row.id ?? null;
+    data.name = data.name || row.name || null;
+    data.type = data.type || row.type || null;
+    data.status = data.status || row.status || null;
+    data.street_1 = data.street_1 || row.street_1 || null;
+    data.street_2 = data.street_2 || row.street_2 || null;
+    data.street_3 = data.street_3 || row.street_3 || null;
+    data.city = data.city || row.city || null;
+    data.region = data.region || row.region || null;
+    data.postal_code = data.postal_code || row.postal_code || null;
+    data.country = data.country || row.country || null;
+    data.subscription_status = data.subscription_status || row.subscription_status || null;
+    data.deliverability_score = data.deliverability_score ?? row.deliverability_score ?? null;
+    data.preference_order = data.preference_order ?? row.preference_order ?? null;
+    data.is_best_address = data.is_best_address ?? row.is_best_address ?? null;
+    data.created_at = data.created_at ?? row.created_at ?? null;
+    data.modified_at = data.modified_at ?? row.modified_at ?? null;
+    data.source_input_id = data.source_input_id ?? row.source_input_id ?? null;
+  });
+};
+
+export default {
+  description,
+  bindings,
+  transform
+};

package/person_address/ui.console.json5

@@ -0,0 +1,31 @@
+{
+  routes: {
+    'person/:id/*': {
+      components: {
+        sidebar: {
+          items: [
+            {
+              component: 'Link',
+              properties: {
+                label: 'Addresses',
+                to: 'address'
+              }
+            }
+          ]
+        },
+        main: [
+          {
+            path: 'address',
+            label: 'Addresses',
+            component: 'RecordDisplay',
+            properties: {
+              table: 'person_address',
+              conditions: [{ eql: 'person_id={{person_id}}' }],
+              rows: ['{{street}}']
+            }
+          }
+        ]
+      }
+    }
+  }
+}

package/person_custom/index.js

@@ -0,0 +1,24 @@
+import upsertPersonCustom from './transforms/inbound/upsert_tables.js';
+import appendRemoteId from './transforms/appendRemoteId.js';
+const metadata = {
+  name: 'Custom Fields',
+  prefix: 'person_custom',
+  unique: false,
+  dependencies: {
+    '@engine9/schemas/person': '>=1.9.0'
+  },
+  // Inbound people pipeline slots -> transform export keys. Core expands the
+  // upsert once per installed row (one custom field table per table_prefix).
+  inbound: {
+    upsert: ['upsertPersonCustom']
+  }
+};
+export const transforms = {
+  upsertPersonCustom,
+  appendRemoteId
+};
+export { metadata };
+export default {
+  metadata,
+  transforms
+};

package/person_custom/transforms/appendRemoteId.js

@@ -0,0 +1,31 @@
+export const bindings = {
+  tablesToUpsert: { path: 'sql.tables.upsert' }
+};
+export const type = 'upsert';
+export async function transform({ batch, options = {}, tablesToUpsert }) {
+  const {
+    table,
+    // schema,
+    columns
+  } = options;
+  batch.forEach((o) => {
+    const vals = {};
+    let hasData = false;
+    columns.forEach((f) => {
+      const v = o[`${table}.${f.name}`];
+      if (v !== undefined) {
+        hasData = true;
+        vals[f.name] = v;
+      }
+    });
+    if (hasData) {
+      tablesToUpsert[table] = (tablesToUpsert[table] || []).concat(vals);
+    }
+  });
+  return batch;
+}
+export default {
+  bindings,
+  type,
+  transform
+};

package/person_custom/transforms/inbound/upsert_tables.js

@@ -0,0 +1,31 @@
+export const bindings = {
+  tablesToUpsert: { path: 'sql.tables.upsert' }
+};
+export const type = 'upsert';
+export async function transform({ batch, options = {}, tablesToUpsert }) {
+  const {
+    table,
+    // schema,
+    columns
+  } = options;
+  batch.forEach((o) => {
+    const vals = {};
+    let hasData = false;
+    columns.forEach((f) => {
+      const v = o[`${table}.${f.name}`];
+      if (v !== undefined) {
+        hasData = true;
+        vals[f.name] = v;
+      }
+    });
+    if (hasData) {
+      tablesToUpsert[table] = (tablesToUpsert[table] || []).concat(vals);
+    }
+  });
+  return batch;
+}
+export default {
+  bindings,
+  type,
+  transform
+};

package/person_email/README.md

@@ -0,0 +1,82 @@
+# Person Email Schema Plugin
+
+The `@engine9/schemas/person_email` plugin stores email addresses for people and provides the common email-related behavior used by imports, matching, searches, segments, and exports.
+
+It depends on `@engine9/schemas/person` because every email row belongs to a person through `person_id`.
+
+Browser / public forms should use the column name **`email_type`** (never `type`).
+See [`@engine9/id` forms docs](../../id/docs/forms.md).
+
+## Data Model
+
+The plugin creates the `person_email` table. A person can have multiple email rows, but each `(email, person_id)` pair is unique.
+
+| Field | Type | Description |
+| --- | --- | --- |
+| `id` | `id` | Primary row identifier. |
+| `person_id` | `person_id` | Person that owns the email address. |
+| `email_type` | `string` | Email category. Allowed values are `Personal`, `Work`, and `Other`. Defaults to `Personal`. |
+| `email` | `string` | Email address. Inbound transforms trim whitespace but otherwise preserve the provided casing. Matching and hashing compare the lowercased value. |
+| `subscription_status` | `string` | Email subscription state. Allowed values are `Not Subscribed`, `Subscribed`, `Unsubscribed`, `Bouncing`, and `Spam`. The schema default is `Not Subscribed`; the inbound upsert transform defaults newly imported emails to `Subscribed` when no status is provided. |
+| `confirmation_status` | `string` | Opt-in confirmation state. Allowed values are `Not Confirmed`, `Confirmation Sent`, and `Confirmed`. Defaults to `Not Confirmed`. |
+| `deliverability_score` | `int` | Deliverability score for the address. `0` represents undeliverable, `1` represents deliverable, and higher values can be used by callers for additional states. Defaults to `1`. |
+| `preference_order` | `int` | Sort order for preferred emails, where `0` is first. Defaults to `0`. |
+| `email_hash_v1` | `hash` | SHA-256 hash of the trimmed, lowercased email address. Used as an identifier and for privacy-preserving exports. |
+| `source_input_id` | `foreign_uuid` | Input that originally created the email row. Existing rows keep their original source when updated. |
+| `created_at` | `created_at` | Timestamp when the row was created. |
+| `modified_at` | `modified_at` | Timestamp when the row was last modified. |
+
+Indexes:
+
+- `person_id`
+- unique `(email, person_id)`
+
+## Inbound Behavior
+
+The inbound `id` transform extracts an `email_hash_v1` identifier from either `email` or an existing `email_hash_v1` field. When an `email` is present and at least five characters after trimming, it is trimmed, lowercased for hashing, and hashed with SHA-256.
+
+The inbound `upsert` transform writes rows to `person_email`.
+
+- Emails are matched case-insensitively after trimming.
+- Existing rows for the same `person_id` and email are updated instead of duplicated.
+- The transform accepts either `email_subscription_status` or `subscription_status`.
+- Newly inserted emails default to `Subscribed` if no status is provided.
+- Unsubscribe-like entries set `subscription_status` to `Unsubscribed` when no status was provided. This includes `entry_type` values `EMAIL_UNSUBSCRIBE` and `EMAIL_SPAM`, plus `entry_type_id` values `44` and `48`.
+- When an email is unsubscribed, all matching email rows for that address are also updated to `Unsubscribed`.
+
+## Outbound Behavior
+
+The plugin provides outbound transforms for appending email data to batches:
+
+- `appendEmail` looks up `person_email` rows by `person_id` and appends `email` to each batch row. It can be filtered by `subscriptionStatus`, and it preserves any existing `email` value already on the row.
+- `appendEmailHash` appends `email_hash_v1` by hashing the row's trimmed, lowercased `email` when the hash is not already present.
+
+## Search
+
+The `emails` search lets users find people who have an email address, optionally filtered by:
+
+- `subscriptionStatus`, using the same values as `subscription_status`
+- `emailMatch`, matched against the `email` column with `LIKE`
+
+## Segments
+
+Predefined audiences shipped with this schema plugin. On install, each key becomes a `segment` row whose definition path is `@engine9/schemas/person_email:segments:<key>`.
+
+### Email Subscribers
+
+| | |
+| --- | --- |
+| **Key** | `subscribers` |
+| **Definition path** | `@engine9/schemas/person_email:segments:subscribers` |
+| **Who is included** | People with at least one email whose `subscription_status` is `Subscribed` |
+| **Who is excluded** | People whose emails are only `Not Subscribed`, `Unsubscribed`, `Bouncing`, or `Spam` |
+| **How it is built** | The `emails` search with `subscriptionStatus: Subscribed` |
+| **Universe** | None — membership is current `person_email` rows, not a time window or message input set |
+
+A person with mixed statuses is included if any of their addresses is `Subscribed`.
+
+## Reports and UI
+
+Email subscription dashboards live on `@engine9/plugins/reports/people` (`subscription_status`). See [`plugins/reports/people/README.md`](../../plugins/reports/people/README.md).
+
+The console UI adds an `Emails` tab on person records. It displays each person's email rows with the email address, type, subscription status, and modified timestamp.

package/person_email/index.js

@@ -0,0 +1,35 @@
+import schema from './schema.js';
+import extractEmailHashes from './transforms/inbound/extract_identifiers.js';
+import upsertPersonEmail from './transforms/inbound/upsert_tables.js';
+import search from './search.js';
+import appendEmail from './transforms/outbound/appendEmail.js';
+import appendEmailHash from './transforms/outbound/appendEmailHash.js';
+import segments from './segments.js';
+const metadata = {
+  name: '@engine9/schemas/person_email',
+  dependencies: {
+    '@engine9/schemas/person': '>=1.9.0'
+  },
+  // Inbound people pipeline slots -> transform export keys (woven by core when installed)
+  inbound: {
+    id: ['extractEmailHashes'],
+    upsert: ['upsertPersonEmail']
+  }
+};
+export const transforms = {
+  extractEmailHashes,
+  upsertPersonEmail,
+  appendEmail,
+  appendEmailHash
+};
+export { metadata };
+export { schema };
+export { search };
+export { segments };
+export default {
+  metadata,
+  schema,
+  search,
+  segments,
+  transforms
+};

package/person_email/schema.js

@@ -0,0 +1,49 @@
+export const tables = [
+  {
+    name: 'person_email',
+    columns: {
+      id: 'id',
+      person_id: 'person_id',
+      email_type: {
+        type: 'string',
+        nullable: false,
+        default_value: 'Personal',
+        values: ['', 'Personal', 'Work', 'Other']
+      },
+      email: 'string',
+      subscription_status: {
+        type: 'string',
+        nullable: false,
+        default_value: 'Not Subscribed',
+        values: ['Not Subscribed', 'Subscribed', 'Unsubscribed', 'Bouncing', 'Spam']
+      },
+      confirmation_status: {
+        type: 'string',
+        nullable: true,
+        default_value: 'Not Confirmed',
+        values: ['Not Confirmed', 'Confirmation Sent', 'Confirmed']
+      },
+      deliverability_score: {
+        type: 'int',
+        description:
+          'Score representating deliverability status of email, e.g. undeliverable(0), deliverable(1), higher values mean different things',
+        nullable: false,
+        default_value: 1
+      },
+      preference_order: {
+        type: 'int',
+        description: 'Order in the preference stack, 0 is first',
+        nullable: false,
+        default_value: 0
+      },
+      email_hash_v1: 'hash',
+      source_input_id: 'foreign_uuid',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [{ columns: 'person_id' }, { columns: ['email', 'person_id'], unique: true }]
+  }
+];
+export default {
+  tables
+};

package/person_email/search.js

@@ -0,0 +1,61 @@
+export const emails = {
+  title: 'Emails',
+  form: {
+    title: 'Emails',
+    type: 'object',
+    properties: {
+      emailMatch: {
+        type: 'string'
+      },
+      subscriptionStatus: {
+        type: 'string',
+        enum: ['Not Subscribed', 'Subscribed', 'Unsubscribed', 'Bouncing', 'Spam']
+      }
+    },
+    required: []
+  },
+  /* map from provided user data into an EQL structure */
+  optionsToEQL(options) {
+    const { emailMatch, subscriptionStatus } = options;
+    let text = 'Has an email';
+    const conditions = [];
+    if (subscriptionStatus) {
+      text = `Has a ${subscriptionStatus.toLowerCase()} email`;
+      conditions.push({
+        type: 'EQUALS',
+        values: [
+          {
+            ref: { column: 'subscription_status' }
+          },
+          {
+            value: { value: subscriptionStatus }
+          }
+        ]
+      });
+    }
+    if (emailMatch) {
+      text += ` that matches ${emailMatch}`;
+      conditions.push({
+        type: 'LIKE',
+        values: [
+          {
+            ref: { column: 'email' }
+          },
+          {
+            value: { value: emailMatch }
+          }
+        ]
+      });
+    }
+    return {
+      text,
+      eql: {
+        table: 'person_email',
+        conditions
+      }
+    };
+  }
+};
+export default {
+  emails
+};

package/person_email/segments.js

@@ -0,0 +1,16 @@
+export const subscribers = {
+  name: 'Email Subscribers',
+  search: {
+    and: [
+      {
+        path: '@engine9/schemas/person_email:search:emails',
+        options: {
+          subscriptionStatus: 'Subscribed'
+        }
+      }
+    ]
+  }
+};
+export default {
+  subscribers
+};

package/person_email/transforms/inbound/extract_identifiers.js

@@ -0,0 +1,33 @@
+import nodecrypto from 'node:crypto';
+const { createHash } = nodecrypto;
+export const type = 'id';
+const blankHash = createHash('sha256').update('').digest('hex');
+export async function transform({ batch }) {
+  batch.forEach((e) => {
+    e.identifiers = e.identifiers || [];
+    if (e.email && e.email.trim().length >= 5) {
+      e.email = e.email.trim(); // no spaces in emails
+      const hashable = e.email.toLowerCase();
+      // no secret or createHmac for this use case
+      // it's basically a shared id hashing setup, so a secret will be exposed anyhow
+      const value = createHash('sha256').update(hashable).digest('hex');
+      // Push all identifiers, later in the pipeline the priority will be determined
+      e.identifiers.push({
+        path: 'person_email',
+        type: 'email_hash_v1',
+        value
+      });
+      // e.email_hash_v1 = value;
+    } else if (e.email_hash_v1 && e.email_hash_v1 != blankHash) {
+      e.identifiers.push({
+        path: 'person_email',
+        type: 'email_hash_v1',
+        value: e.email_hash_v1
+      });
+    }
+  });
+}
+export default {
+  type,
+  transform
+};

package/person_email/transforms/inbound/upsert_tables.js

@@ -0,0 +1,142 @@
+import { mergeIntoQueue } from '@engine9/input-tools';
+
+export const bindings = {
+  // existingPersonEmails: { path: 'sql.query', table: 'person_email', lookup:['person_id'] },
+  databaseEmails: { path: 'sql.query', options: { table: 'person_email', lookup: ['email'] } },
+  tablesToUpsert: { path: 'sql.tables.upsert' }
+};
+export const type = 'upsert';
+
+const emailKeyNormalize = (field, value) => {
+  if (field === 'email' && typeof value === 'string') return value.trim().toLowerCase();
+  // person_id may arrive as number or string from different pipeline stages
+  if (field === 'person_id' && value != null && value !== '') return String(value);
+  return value ?? '';
+};
+
+/** Last queued batch row wins on subscription_status (file order). */
+function mergePersonEmail(existing, incoming) {
+  return {
+    ...existing,
+    ...incoming,
+    id: existing.id ?? incoming.id ?? null,
+    email: existing.email || incoming.email,
+    person_id: existing.person_id ?? incoming.person_id,
+    subscription_status: incoming.subscription_status ?? existing.subscription_status,
+    // `person_email.email_hash_v1` is not nullable in MySQL; use blank string when unknown.
+    // Also, never drop the key entirely (upsertArray column inference depends on key presence).
+    email_hash_v1: incoming.email_hash_v1 ?? existing.email_hash_v1 ?? '',
+    source_input_id: existing.source_input_id ?? incoming.source_input_id
+  };
+}
+
+function queuePersonEmail(tablesToUpsert, row, { keyFields = ['email', 'person_id'], key } = {}) {
+  tablesToUpsert.person_email = tablesToUpsert.person_email || [];
+  // Ensure the key is present for every queued object so core upsertArray's
+  // "inconsistent upsert columns" check never trips.
+  // `type` is the old field name for `email_type`. Every queued row needs the
+  // same keys; a batch that mixes the two names fails the upsert column check.
+  const normalizedRow = {
+    ...row,
+    email_hash_v1: row.email_hash_v1 ?? '',
+    email_type: row.email_type ?? row.type ?? 'Personal'
+  };
+  delete normalizedRow.type;
+  mergeIntoQueue(tablesToUpsert.person_email, normalizedRow, {
+    keyFields: key ? undefined : keyFields,
+    key,
+    normalizeField: key ? undefined : emailKeyNormalize,
+    merge: mergePersonEmail,
+    label: 'person_email'
+  });
+}
+
+export async function transform(props) {
+  const { batch, databaseEmails, tablesToUpsert } = props;
+  if (batch.length === 0) return;
+  batch.forEach((o) => {
+    if (!o.email) return;
+    // People like to believe email is case sensitive
+    // emails are always trimmed, but that's it for inbound modifications.
+    const email = o.email.trim();
+    const lcEmail = email.toLowerCase();
+    const matchingEmails = databaseEmails.filter((em) => em?.email?.trim().toLowerCase() === lcEmail);
+    const personEmails = matchingEmails.filter((em) => o.person_id && em.person_id === o.person_id);
+    if (personEmails.length > 1) {
+      throw new Error(
+        `Cannot update emails, there are 2 database entries for person_id ${o.person_id} with email ${email}`
+      );
+    }
+    if (
+      o.entry_type === 'EMAIL_UNSUBSCRIBE' ||
+      o.entry_type === 'EMAIL_SPAM' ||
+      o.entry_type_id === 44 ||
+      o.entry_type_id === 48
+    ) {
+      if (!o.subscription_status) o.subscription_status = 'Unsubscribed';
+    }
+    let status = o.email_subscription_status || o.subscription_status;
+    const { id, ...rest } = o;
+    if (id) {
+      //this is undoubtedly NOT the ID of the person_email record
+    }
+    if (personEmails[0]) {
+      // if it's explicitly specified, then update it, otherwise set it to what it was before
+      status = status || personEmails[0].subscription_status;
+      queuePersonEmail(tablesToUpsert, {
+        ...rest,
+        id: personEmails[0].id,
+        person_id: personEmails[0].person_id,
+        email,
+        email_type: o.email_type ?? o.type ?? personEmails[0].email_type ?? 'Personal',
+        subscription_status: status,
+        //make sure this doesn't change
+        source_input_id: personEmails[0].source_input_id
+        //original: personEmails[0]
+      });
+    } else {
+      status = status || 'Subscribed'; // Default subscribed
+      queuePersonEmail(tablesToUpsert, {
+        ...rest,
+        id: null,
+        person_id: o.person_id,
+        email,
+        subscription_status: status,
+        source_input_id: o.input_id
+      });
+    }
+    // IF an unsubscribe, we need to update the subscription status for ALL related emails,
+    // including new ones
+    if (status === 'Unsubscribed') {
+      matchingEmails.forEach((original) => {
+        if (original.subscription_status !== 'Unsubscribed') {
+          let updatedRecord = {};
+          //this is to make sure the keys match
+          Object.keys(rest).forEach((k) => {
+            updatedRecord[k] = original[k] ?? null;
+          });
+          Object.assign(updatedRecord, {
+            id: original.id,
+            person_id: original.person_id,
+            email,
+            email_type: o.email_type ?? o.type ?? original.email_type ?? 'Personal',
+            subscription_status: 'Unsubscribed',
+            //make sure this doesn't change
+            source_input_id: original.source_input_id
+          });
+          queuePersonEmail(tablesToUpsert, updatedRecord, { key: (row) => String(row.id) });
+        }
+      });
+      tablesToUpsert.person_email
+        .filter((d) => d.id == null && d.email?.trim().toLowerCase() === lcEmail)
+        .forEach((d) => {
+          d.subscription_status = 'Unsubscribed';
+        });
+    }
+  });
+}
+export default {
+  bindings,
+  type,
+  transform
+};

package/person_email/transforms/outbound/appendEmail.js

@@ -0,0 +1,32 @@
+export const description = 'Append an email';
+export const bindings = {
+  emails: {
+    path: 'sql.query',
+    options: {
+      table: 'person_email',
+      columns: ['person_id', 'email', 'subscription_status'],
+      lookup: ['person_id'],
+      conditions: [
+        // { eql: 'id_type=\'remote_person_id\'' },
+      ]
+    }
+  }
+};
+export const transform = ({ batch, emails, options = {} }) => {
+  const { subscriptionStatus } = options;
+  let filter = () => true;
+  if (subscriptionStatus) filter = (d) => d.subscription_status === subscriptionStatus;
+  const emailMap = emails.filter(filter).reduce((a, b) => {
+    a[b.person_id] = a[b.person_id] || b.email;
+    return a;
+  }, {});
+  batch.forEach((data) => {
+    // keep existing email if there is one
+    data.email = data.email || emailMap[data.person_id] || null;
+  });
+};
+export default {
+  description,
+  bindings,
+  transform
+};

package/person_email/transforms/outbound/appendEmailHash.js

@@ -0,0 +1,20 @@
+import nodecrypto from 'node:crypto';
+const { createHash } = nodecrypto;
+export async function transform({ batch }) {
+  batch.forEach((e) => {
+    e.email_hash_v1 = e.email_hash_v1 || '';
+    if (e.email_hash_v1) {
+      // do nothing, already exists
+    } else if (e.email) {
+      e.email = e.email.trim(); // no spaces in emails
+      const hashable = e.email.toLowerCase();
+      // no secret or createHmac for this use case
+      // it's basically a shared id hashing setup, so a secret will be exposed anyhow
+      const value = createHash('sha256').update(hashable).digest('hex');
+      e.email_hash_v1 = value;
+    }
+  });
+}
+export default {
+  transform
+};