@engine9/schemas 1.9.0

package/transaction/core/attribute.test.js

@@ -0,0 +1,231 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+import attribute from './transforms/attribute.js';
+import upsert from './transforms/inbound/upsert_tables.js';
+
+const pluginId = '00000000-0000-4000-8000-000000000001';
+const inputId = '00000000-0000-4000-8000-000000000002';
+const transactionId = '00000000-0000-4000-8000-000000000010';
+const messageUuid = '00000000-0000-4000-8000-000000000003';
+const laterMessageUuid = '00000000-0000-4000-8000-000000000004';
+
+function row(extra = {}) {
+  return {
+    id: transactionId,
+    ts: '2024-06-15T12:00:00.000Z',
+    person_id: 7,
+    remote_entry_id: 'txn-1',
+    remote_input_id: 'form-1',
+    input_id: inputId,
+    source_code: 'SC_EG_123',
+    source_code_id: 42,
+    amount: 25,
+    ...extra
+  };
+}
+
+function pluginInputs() {
+  return [{ input_id: inputId, remote_input_id: 'form-1', remote_plugin_id: 'bot-9' }];
+}
+
+function message(extra = {}) {
+  return {
+    source_code: 'SC_EG_123',
+    message_id: 88,
+    publish_date: '2024-06-01T00:00:00.000Z',
+    message_uuid: messageUuid,
+    message_publish_date: '2024-06-01T00:00:00.000Z',
+    ...extra
+  };
+}
+
+describe('transaction attribute', () => {
+  it('requires a uuid pluginId on the lookup bindings', () => {
+    assert.throws(() => attribute.bindings({}), /pluginId is required/);
+    assert.throws(() => attribute.bindings({ pluginId: 'bot-9' }), /must be a uuid/);
+  });
+
+  it('patches a transaction already queued by upsert and writes attribution', async () => {
+    const batch = [row()];
+    const tablesToUpsert = { transaction: batch };
+    await attribute.transform({
+      batch,
+      tablesToUpsert,
+      pluginInputs: pluginInputs(),
+      messageSourceCodes: [message()]
+    });
+    assert.equal(tablesToUpsert.transaction.length, 1);
+    assert.equal(tablesToUpsert.transaction[0], batch[0]);
+    assert.equal(batch[0].amount, 25);
+    assert.equal(batch[0].recommended_message_id, messageUuid);
+    assert.equal(batch[0].final_message_id, messageUuid);
+    assert.equal(batch[0].final_source_code_id, 42);
+    assert.equal('entry_type_id' in batch[0], false);
+    assert.deepEqual(tablesToUpsert.attribution, [
+      {
+        transaction_bot_id: 'bot-9',
+        remote_transaction_id: 'txn-1',
+        recommended_message_id: 88
+      }
+    ]);
+  });
+
+  it('re-attributes an existing transaction from an override', async () => {
+    const tablesToUpsert = {};
+    await attribute.transform({
+      batch: [row({ source_code: 'SC_ORIGINAL', source_code_id: 7 })],
+      tablesToUpsert,
+      pluginInputs: pluginInputs(),
+      overrides: [
+        {
+          remote_transaction_id: 'txn-1',
+          transaction_bot_id: 'bot-9',
+          transaction_source_code_override: 'SC_OVERRIDE',
+          dictionary_source_code_id: 99,
+          matched_source_code: 'SC_OVERRIDE',
+          matched_message_id: 55,
+          matched_publish_date: '2024-06-02T00:00:00.000Z',
+          matched_message_uuid: laterMessageUuid,
+          matched_message_publish_date: '2024-06-02T00:00:00.000Z'
+        }
+      ],
+      messageSourceCodes: [message({ source_code: 'SC_ORIGINAL', message_id: 11, message_uuid: messageUuid })]
+    });
+    assert.deepEqual(tablesToUpsert.transaction, [
+      {
+        id: transactionId,
+        override_source_code_id: 99,
+        final_source_code_id: 99,
+        recommended_message_id: laterMessageUuid,
+        override_message_id: null,
+        final_message_id: laterMessageUuid
+      }
+    ]);
+    assert.equal(tablesToUpsert.attribution[0].recommended_message_id, 55);
+  });
+
+  it('matches a message published inside the 29 hour window and rejects one past it', async () => {
+    const within = {};
+    await attribute.transform({
+      batch: [row()],
+      tablesToUpsert: within,
+      pluginInputs: pluginInputs(),
+      messageSourceCodes: [message({ message_id: 11, publish_date: '2024-06-16T16:00:00.000Z', message_publish_date: null })]
+    });
+    assert.equal(within.attribution.length, 1);
+
+    const past = {};
+    await attribute.transform({
+      batch: [row()],
+      tablesToUpsert: past,
+      pluginInputs: pluginInputs(),
+      messageSourceCodes: [message({ message_id: 11, publish_date: '2024-06-16T17:00:01.000Z', message_publish_date: null })]
+    });
+    assert.equal(past.attribution.length, 0);
+    assert.equal(past.transaction[0].recommended_message_id, null);
+  });
+
+  it('picks the latest publish_date and skips do_not_attribute on the transaction source code', async () => {
+    const tablesToUpsert = {};
+    await attribute.transform({
+      batch: [row()],
+      tablesToUpsert,
+      pluginInputs: pluginInputs(),
+      messageSourceCodes: [
+        message({ message_id: 11, publish_date: '2024-06-01T00:00:00.000Z', message_publish_date: '2024-06-01T00:00:00.000Z' }),
+        message({
+          message_id: 22,
+          publish_date: '2024-06-10T00:00:00.000Z',
+          message_publish_date: '2024-06-10T00:00:00.000Z',
+          message_uuid: laterMessageUuid
+        }),
+        message({
+          message_id: 33,
+          publish_date: '2024-06-12T00:00:00.000Z',
+          message_publish_date: '2024-06-12T00:00:00.000Z',
+          message_uuid: '00000000-0000-4000-8000-000000000005',
+          do_not_attribute: 1
+        })
+      ]
+    });
+    assert.equal(tablesToUpsert.transaction[0].recommended_message_id, laterMessageUuid);
+    assert.equal(tablesToUpsert.attribution[0].recommended_message_id, 22);
+  });
+
+  it('does not fall back to the transaction source code when the override code is blank', async () => {
+    const tablesToUpsert = {};
+    await attribute.transform({
+      batch: [row()],
+      tablesToUpsert,
+      pluginInputs: pluginInputs(),
+      overrides: [
+        {
+          remote_transaction_id: 'txn-1',
+          transaction_bot_id: 'bot-9',
+          transaction_source_code_override: ''
+        }
+      ],
+      messageSourceCodes: [message()]
+    });
+    assert.equal(tablesToUpsert.attribution.length, 0);
+    assert.equal(tablesToUpsert.transaction[0].recommended_message_id, null);
+  });
+
+  it('does not attribute when the override code has no message', async () => {
+    const tablesToUpsert = {};
+    await attribute.transform({
+      batch: [row()],
+      tablesToUpsert,
+      pluginInputs: pluginInputs(),
+      overrides: [
+        {
+          remote_transaction_id: 'txn-1',
+          transaction_bot_id: 'bot-9',
+          transaction_source_code_override: 'SC_OVERRIDE'
+        }
+      ],
+      messageSourceCodes: [message()]
+    });
+    assert.equal(tablesToUpsert.attribution.length, 0);
+    assert.equal(tablesToUpsert.transaction[0].recommended_message_id, null);
+  });
+
+  it('throws when a legacy message id matches but remote_plugin_id was not loaded', async () => {
+    await assert.rejects(
+      () =>
+        attribute.transform({
+          batch: [
+            row({
+              remote_input_id: 'missing-form',
+              input_id: '00000000-0000-4000-8000-000000000099'
+            })
+          ],
+          tablesToUpsert: {},
+          pluginInputs: pluginInputs(),
+          messageSourceCodes: [message({ message_id: 11, message_uuid: null, message_publish_date: null })]
+        }),
+      /remote_plugin_id was not loaded/
+    );
+  });
+});
+
+describe('transaction upsert_tables', () => {
+  it('maps recurs onto recurs_id', async () => {
+    const tablesToUpsert = {};
+    await upsert.transform({
+      batch: [
+        {
+          ts: '2024-06-15T12:00:00.000Z',
+          person_id: 7,
+          entry_type: 'TRANSACTION',
+          remote_entry_id: 'txn-week',
+          recurs: 'weekly'
+        }
+      ],
+      tablesToUpsert,
+      pluginId
+    });
+    assert.equal(tablesToUpsert.transaction[0].recurs_id, 2);
+    assert.equal(tablesToUpsert.transaction[0].entry_type_id, 10);
+  });
+});

package/transaction/core/index.js

@@ -0,0 +1,31 @@
+import schema from './schema.js';
+import search from './search.js';
+import metrics from './metrics.js';
+import segments from './segments.js';
+import upsert from './transforms/inbound/upsert_tables.js';
+import attribute from './transforms/attribute.js';
+import appendTransactionSummary from './transforms/appendTransactionSummary.js';
+const metadata = {
+  name: '@engine9/schemas/transaction/core',
+  dependencies: {
+    '@engine9/schemas/person': '>=1.9.0'
+  }
+};
+export const transforms = {
+  upsert,
+  attribute,
+  appendTransactionSummary
+};
+export { metadata };
+export { schema };
+export { metrics };
+export { search };
+export { segments };
+export default {
+  metadata,
+  schema,
+  metrics,
+  transforms,
+  search,
+  segments
+};

package/transaction/core/metrics.js

@@ -0,0 +1,20 @@
+export const summaryByDate = ({ plugin: { tablePrefix } }) => {
+  return {
+    label: 'Transactions',
+    description: '',
+    eql: {
+      table: tablePrefix + 'transaction',
+      columns: [
+        { eql: 'month(ts)', name: 'date' },
+        { eql: 'count(*)', name: 'records' },
+        { eql: 'sum(amount)', name: 'revenue' }
+      ],
+      conditions: [],
+      groupBy: [{ eql: 'month(ts)' }],
+      orderBy: [{ eql: 'month(ts)' }]
+    }
+  };
+};
+export default {
+  summaryByDate
+};

package/transaction/core/schema.js

@@ -0,0 +1,45 @@
+export const tables = [
+  {
+    name: 'transaction',
+    columns: {
+      id: 'id_uuid',
+      ts: 'datetime',
+      input_id: 'uuid', // keep as uuid because it's required
+      entry_type_id: 'int',
+      person_id: 'person_id',
+      amount: 'currency',
+      remote_entry_id: 'string',
+      remote_page_name: 'string',
+      remote_recurring_id: 'string',
+      recurs_id: {
+        type: 'int',
+        nullable: false,
+        default_value: 0
+      },
+      recurring_number: 'int',
+      refund_amount: 'currency',
+      given_name: 'string',
+      family_name: 'string',
+      email: 'string',
+      source_code_id: 'source_code_id',
+      override_source_code_id: 'source_code_id',
+      final_source_code_id: 'source_code_id',
+      recommended_message_id: 'uuid',
+      override_message_id: 'uuid',
+      final_message_id: 'uuid',
+      extra: 'json',
+      created_at: 'created_at',
+      modified_at: 'modified_at'
+    },
+    indexes: [
+      { columns: 'id', primary: true },
+      { columns: 'ts' },
+      { columns: 'person_id' },
+      { columns: 'remote_entry_id' },
+      { columns: 'modified_at' }
+    ]
+  }
+];
+export default {
+  tables
+};

package/transaction/core/search.js

@@ -0,0 +1,87 @@
+import { relativeDate, isValidDate } from '@engine9/input-tools';
+
+export const all = {
+  title: 'Transactions',
+  form: {
+    title: 'Transactions',
+    type: 'object',
+    properties: {
+      plugin_id: {
+        type: 'string'
+      },
+      start: {
+        title: 'Start',
+        type: 'string',
+        description:
+          "Optional; only include rows with ts >= start. Parsed with relativeDate; supports ISO datetimes or relative expressions like '-30d'."
+      },
+      end: {
+        title: 'End',
+        type: 'string',
+        description:
+          'Optional; only include rows with ts < end (exclusive upper bound). Parsed with relativeDate; supports ISO datetimes or relative expressions.'
+      }
+    },
+    required: []
+  },
+  optionsToEQL: (options) => {
+    const { plugin_id, start, end } = options;
+    const conditions = [];
+    let text = 'Has any transactions';
+    if (plugin_id) {
+      conditions.push(`input.plugin_id='${plugin_id}'`);
+      text += ` from plugin ${plugin_id}`;
+    }
+    if (start) {
+      const s = relativeDate(start);
+      if (!isValidDate(s)) throw new Error('Invalid start: ' + start);
+      conditions.push(`transaction.ts>='${s.toISOString()}'`);
+      text += ` with ts >= ${s.toISOString()}`;
+    }
+    if (end) {
+      const e = relativeDate(end);
+      if (!isValidDate(e)) throw new Error('Invalid end: ' + end);
+      conditions.push(`transaction.ts<'${e.toISOString()}'`);
+      text += ` with ts < ${e.toISOString()}`;
+    }
+    return {
+      text,
+      eql: {
+        table: 'transaction',
+        joins: [
+          {
+            table: 'input',
+            join_eql: 'transaction.input_id=input.id'
+          }
+        ],
+        columns: ['person_id'],
+        conditions
+      }
+    };
+  }
+};
+export const minimum = {
+  title: 'Minimum transaction amount',
+  form: {
+    title: 'Minimum transaction amount',
+    type: 'object',
+    properties: {
+      amount: {
+        type: 'number'
+      },
+      filterType: {
+        type: 'string'
+      }
+    },
+    required: []
+  },
+  optionsToEQL: (options) => ({
+    table: 'transaction',
+    columns: ['person_id'],
+    conditions: [`amount>=${parseFloat(options.amount)}`]
+  })
+};
+export default {
+  all,
+  minimum
+};

package/transaction/core/segments.js

@@ -0,0 +1,14 @@
+export const customers = {
+  name: 'Customers',
+  search: {
+    and: [
+      {
+        table: 'transaction',
+        columns: ['person_id']
+      }
+    ]
+  }
+};
+export default {
+  customers
+};

package/transaction/core/transforms/appendTransactionSummary.js

@@ -0,0 +1,101 @@
+const placeholder = {
+  count: null,
+  revenue: null,
+  recurring_count: null,
+  recurring_revenue: null,
+  refund_count: null,
+  refund_revenue: null,
+  first_id: null,
+  first_date: null,
+  first_amount: null,
+  last_id: null,
+  last_date: null,
+  last_amount: null,
+  smallest_id: null,
+  smallest_date: null,
+  smallest_amount: null,
+  largest_id: null,
+  largest_date: null,
+  largest_amount: null,
+  first_recurring_id: null,
+  first_recurring_date: null,
+  first_recurring_amount: null,
+  last_recurring_id: null,
+  last_recurring_date: null,
+  last_recurring_amount: null
+};
+export const bindings = {
+  allTransactions: {
+    path: 'sql.query',
+    options: {
+      table: 'transaction',
+      columns: 'id,person_id,ts,amount,recurs_id,refund_amount'.split(','),
+      lookup: ['person_id']
+    }
+  }
+};
+export async function transform({ batch, allTransactions }) {
+  function sumAmount(a, b) {
+    return (parseFloat(a) + parseFloat(b.amount || 0)).toFixed(2);
+  }
+  const personIdMap = {};
+  allTransactions.sort((a, b) => (a.ts < b.ts ? -1 : 1));
+  allTransactions.forEach((d) => {
+    d.ts = new Date(d.ts).toISOString();
+    personIdMap[d.person_id] = personIdMap[d.person_id] || { person_id: d.person_id, transactions: [] };
+    personIdMap[d.person_id].transactions.push(d);
+  });
+  batch.forEach((person) => {
+    Object.entries(placeholder).forEach(([k, v]) => {
+      person[`transaction.${k}`] = v;
+    });
+    const { transactions } = personIdMap[person.person_id] || { transactions: [] };
+    if (transactions.length === 0) return;
+    const recurring = transactions.filter((d) => d.recurs_id > 0 && d.amount > 0);
+    const largest = transactions.reduce((a, b) => (a.amount > b.amount ? a : b), { amount: -1 });
+    const smallest = transactions.reduce((a, b) => (a.amount < b.amount ? a : b), {
+      amount: 100000000
+    });
+    let refundCount = 0;
+    let refundAmount = 0;
+    transactions.forEach((t) => {
+      if (t.refund_amount) {
+        refundCount += 1;
+        refundAmount += parseFloat(t.refund_amount);
+      }
+    });
+    const output = {
+      count: transactions.length,
+      revenue: transactions.reduce(sumAmount, 0),
+      recurring_count: recurring.length,
+      recurring_revenue: recurring.reduce(sumAmount, 0),
+      refund_count: refundCount,
+      refund_revenue: refundAmount,
+      first_id: transactions[0].id || null,
+      first_date: transactions[0].ts || null,
+      first_amount: transactions[0].amount || null,
+      last_id: transactions.slice(-1)[0].id || null,
+      last_date: transactions.slice(-1)[0].ts || null,
+      last_amount: transactions.slice(-1)[0].amount || null,
+      largest_id: largest.id || null,
+      largest_date: largest.ts || null,
+      largest_amount: largest.amount || null,
+      smallest_id: smallest.id || null,
+      smallest_date: smallest.ts || null,
+      smallest_amount: smallest.amount || null,
+      first_recurring_id: (recurring[0] || {}).id || null,
+      first_recurring_date: (recurring[0] || {}).ts || null,
+      first_recurring_amount: (recurring[0] || {}).amount || null,
+      last_recurring_id: (recurring.slice(-1)[0] || {}).id || null,
+      last_recurring_date: (recurring.slice(-1)[0] || {}).ts || null,
+      last_recurring_amount: (recurring.slice(-1)[0] || {}).amount || null
+    };
+    Object.entries(output).forEach(([k, v]) => {
+      person[`transaction.${k}`] = v;
+    });
+  });
+}
+export default {
+  bindings,
+  transform
+};