@engine9/input-tools 2.0.7

package/index.js

@@ -0,0 +1,426 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import dayjs from 'dayjs';
+import debug$0 from 'debug';
+import unzipper from 'unzipper';
+import { v4 as uuidv4, v5 as uuidv5, v7 as uuidv7, validate as uuidIsValid } from 'uuid';
+import archiver from 'archiver';
+import handlebars from 'handlebars';
+import FileUtilities from './file/FileUtilities.js';
+import tools from './file/tools.js';
+import ForEachEntry from './ForEachEntry.js';
+import { TIMELINE_ENTRY_TYPES } from './timelineTypes.js';
+const debug = debug$0('@engine9/input-tools');
+
+const {
+  appendPostfix,
+  bool,
+  getBatchTransform,
+  getDebatchTransform,
+  getFile,
+  getManifest,
+  getPacketFiles,
+  getStringArray,
+  downloadFile,
+  getTempFilename,
+  getTempDir,
+  isValidDate,
+  parseJSON5,
+  relativeDate,
+  streamPacket,
+  makeStrings,
+  writeTempFile
+} = tools;
+function getFormattedDate(dateObject, format = 'MMM DD,YYYY') {
+  let d = dateObject;
+  if (d === 'now') d = new Date();
+  if (d) return dayjs(d).format(format);
+  return '';
+}
+handlebars.registerHelper('date', (d, f) => {
+  let format;
+  if (typeof f === 'string') format = f;
+  return getFormattedDate(d, format);
+});
+handlebars.registerHelper('json', (d) => JSON.stringify(d));
+handlebars.registerHelper('uuid', () => uuidv7());
+handlebars.registerHelper('percent', (a, b) => `${((100 * a) / b).toFixed(2)}%`);
+handlebars.registerHelper('or', (a, b, c) => a || b || c);
+async function list(_path) {
+  const directory = await unzipper.Open.file(_path);
+  return new Promise((resolve, reject) => {
+    directory.files[0].stream().pipe(fs.createWriteStream('firstFile')).on('error', reject).on('finish', resolve);
+  });
+}
+async function extract(_path, _file) {
+  const directory = await unzipper.Open(_path);
+  // return directory.files.map((f) => f.path);
+  const file = directory.files.find((d) => d.path === _file);
+  const tempFilename = await getTempFilename({ source: _file });
+  return new Promise((resolve, reject) => {
+    file.stream().pipe(fs.createWriteStream(tempFilename)).on('error', reject).on('finish', resolve);
+  });
+}
+function appendFiles(existingFiles, _newFiles, options) {
+  const newFiles = getStringArray(_newFiles);
+  if (newFiles.length === 0) return;
+  let { type, dateCreated } = options || {};
+  if (!type) type = 'unknown';
+  if (!dateCreated) dateCreated = new Date().toISOString();
+  let arr = newFiles;
+  if (!Array.isArray(newFiles)) arr = [arr];
+  arr.forEach((p) => {
+    const item = {
+      type,
+      originalFilename: '',
+      isNew: true,
+      dateCreated
+    };
+    if (typeof p === 'string') {
+      item.originalFilename = path.resolve(process.cwd(), p);
+    } else {
+      item.originalFilename = path.resolve(process.cwd(), item.originalFilename);
+    }
+    const file = item.originalFilename.split(path.sep).pop();
+    item.path = `${type}/${file}`;
+    const existingFile = existingFiles.find((f) => f.path === item.path);
+    if (existingFile) throw new Error('Error adding files, duplicate path found for path:', +item.path);
+    existingFiles.push(item);
+  });
+}
+async function create(options) {
+  const {
+    accountId = 'engine9',
+    pluginId = '',
+    target = '', // target filename, creates one if not specified
+    messageFiles = [], // file with contents of message, used for delivery
+    personFiles = [], // files with data on people
+    timelineFiles = [], // activity entry
+    statisticsFiles = [] // files with aggregate statistics
+  } = options;
+  if (options.peopleFiles) throw new Error('Unknown option: peopleFiles, did you mean personFiles?');
+  const files = [];
+  const dateCreated = new Date().toISOString();
+  appendFiles(files, messageFiles, { type: 'message', dateCreated });
+  appendFiles(files, personFiles, { type: 'person', dateCreated });
+  appendFiles(files, timelineFiles, { type: 'timeline', dateCreated });
+  appendFiles(files, statisticsFiles, { type: 'statistics', dateCreated });
+  const zipFilename = target || (await getTempFilename({ postfix: '.packet.zip' }));
+  const manifest = {
+    accountId,
+    source: {
+      pluginId
+    },
+    dateCreated,
+    files
+  };
+  // create a file to stream archive data to.
+  const output = fs.createWriteStream(zipFilename);
+  const archive = archiver('zip', {
+    zlib: { level: 9 } // Sets the compression level.
+  });
+  return new Promise((resolve, reject) => {
+    debug(`Setting up write stream to ${zipFilename}`);
+    // listen for all archive data to be written
+    // 'close' event is fired only when a file descriptor is involved
+    output.on('close', () => {
+      debug('archiver has been finalized and the output file descriptor has closed, calling success');
+      debug(zipFilename);
+      return resolve({
+        filename: zipFilename,
+        bytes: archive.pointer()
+      });
+    });
+    // This event is fired when the data source is drained no matter what was the data source.
+    // It is not part of this library but rather from the NodeJS Stream API.
+    // @see: https://nodejs.org/api/stream.html#stream_event_end
+    output.on('end', () => {
+      // debug('end event -- Data has been drained');
+    });
+    // warnings could be file not founds, etc, but we error even on those
+    archive.on('warning', (err) => {
+      reject(err);
+    });
+    // good practice to catch this error explicitly
+    archive.on('error', (err) => {
+      reject(err);
+    });
+    archive.pipe(output);
+    files.forEach(({ path: name, originalFilename }) => archive.file(originalFilename, { name }));
+    files.forEach((f) => {
+      delete f.originalFilename;
+      delete f.isNew;
+    });
+    archive.append(Buffer.from(JSON.stringify(manifest, null, 4), 'utf8'), { name: 'manifest.json' });
+    archive.finalize();
+  });
+}
+function intToByteArray(_v) {
+  // we want to represent the input as a 8-bytes array
+  const byteArray = [0, 0, 0, 0, 0, 0, 0, 0];
+  let v = _v;
+  for (let index = 0; index < byteArray.length; index += 1) {
+    const byte = v & 0xff;
+    byteArray[index] = byte;
+    v = (v - byte) / 256;
+  }
+  return byteArray;
+}
+function getPluginUUID(uniqueNamespaceLikeDomainName, valueWithinNamespace) {
+  // Random custom namespace for plugins -- not intended for cryptographically secure, just a unique namespace:
+  return uuidv5(`${uniqueNamespaceLikeDomainName}::${valueWithinNamespace}`, 'f9e1024d-21ac-473c-bac6-64796dd771dd');
+}
+function getInputUUID(a, b) {
+  let pluginId = a;
+  let remoteInputId = b;
+  if (typeof a === 'object') {
+    pluginId = a.pluginId;
+    remoteInputId = a.remoteInputId;
+  }
+  if (!pluginId) throw new Error('getInputUUID: Cowardly rejecting a blank plugin_id');
+  if (!uuidIsValid(pluginId)) throw new Error(`Invalid pluginId:${pluginId}, should be a UUID`);
+  const rid = (remoteInputId || '').trim();
+  if (!rid) throw new Error('getInputUUID: Cowardly rejecting a blank remote_input_id, set a default');
+  // Random custom namespace for inputs -- not secure, just a namespace:
+  // 3d0e5d99-6ba9-4fab-9bb2-c32304d3df8e
+  return uuidv5(`${pluginId}:${rid}`, '3d0e5d99-6ba9-4fab-9bb2-c32304d3df8e');
+}
+const timestampMatch = /^\d{13}$/;
+function dateFromString(s) {
+  if (typeof s === 'number') return new Date(s);
+  if (typeof s === 'string') {
+    if (s.match(timestampMatch)) return new Date(parseInt(s));
+  }
+  return new Date(s);
+}
+function getUUIDv7(date, inputUuid) {
+  /* optional date and input UUID */
+  const uuid = inputUuid || uuidv7();
+  const bytes = Buffer.from(uuid.replace(/-/g, ''), 'hex');
+  if (date !== undefined) {
+    const d = dateFromString(date);
+    // isNaN behaves differently than Number.isNaN -- we're actually going for the
+    // attempted conversion here
+    if (isNaN(d)) throw new Error(`getUUIDv7 got an invalid date:${date || '<blank>'}`);
+    const dateBytes = intToByteArray(d.getTime()).reverse();
+    dateBytes.slice(2, 8).forEach((b, i) => {
+      bytes[i] = b;
+    });
+  }
+  return uuidv4({ random: bytes });
+}
+/* Returns a date from a given uuid (assumed to be a v7, otherwise the results are ... weird */
+function getUUIDTimestamp(uuid) {
+  const ts = parseInt(`${uuid}`.replace(/-/g, '').slice(0, 12), 16);
+  return new Date(ts);
+}
+function getEntryTypeId(o, { defaults = {} } = {}) {
+  let id = o.entry_type_id || defaults.entry_type_id;
+  if (id) return id;
+  const etype = o.entry_type || defaults.entry_type;
+  if (!etype) {
+    throw new Error('No entry_type, nor entry_type_id specified, specify one to generate a timeline suitable ID');
+  }
+  id = TIMELINE_ENTRY_TYPES[etype];
+  if (id === undefined) throw new Error(`Invalid entry_type: ${etype}`);
+  return id;
+}
+function getEntryType(o, defaults = {}) {
+  let etype = o.entry_type || defaults.entry_type;
+  if (etype) return etype;
+  const id = o.entry_type_id || defaults.entry_type_id;
+  etype = TIMELINE_ENTRY_TYPES[id];
+  if (etype === undefined) throw new Error(`Invalid entry_type: ${etype}`);
+  return etype;
+}
+const requiredTimelineEntryFields = ['ts', 'entry_type_id', 'input_id', 'person_id'];
+function getTimelineEntryUUID(inputObject, { defaults = {} } = {}) {
+  const o = { ...defaults, ...inputObject };
+  /*
+        Outside systems CAN specify a unique UUID as remote_entry_uuid,
+        which will be used for updates, etc.
+        If not, it will be generated using whatever info we have
+      */
+  if (o.remote_entry_uuid) {
+    if (!uuidIsValid(o.remote_entry_uuid)) throw new Error('Invalid remote_entry_uuid, it must be a UUID');
+    return o.remote_entry_uuid;
+  }
+  /*
+          Outside systems CAN specify a unique remote_entry_id
+          If not, it will be generated using whatever info we have
+        */
+  if (o.remote_entry_id) {
+    // get a temp ID
+    if (!o.input_id)
+      throw new Error('Error generating timeline entry uuid -- remote_entry_id specified, but no input_id');
+    try {
+      const uuid = uuidv5(String(o.remote_entry_id), o.input_id);
+      // Change out the ts to match the v7 sorting.
+      // But because outside specified remote_entry_uuid
+      // may not match this standard, uuid sorting isn't guaranteed
+      return getUUIDv7(o.ts, uuid);
+    } catch (e) {
+      debug('Error getting uuid with object:', o);
+      throw e;
+    }
+  }
+  o.entry_type_id = getEntryTypeId(o);
+  const missing = requiredTimelineEntryFields.filter((d) => o[d] === undefined); // 0 could be an entry type value
+  if (missing.length > 0) throw new Error(`Missing required fields to append an entry_id:${missing.join(',')}`);
+  const ts = new Date(o.ts);
+  // isNaN behaves differently than Number.isNaN -- we're actually going for the
+  // attempted conversion here
+  if (isNaN(ts)) throw new Error(`getTimelineEntryUUID got an invalid date:${o.ts || '<blank>'}`);
+  const idString = `${ts.toISOString()}-${o.person_id}-${o.entry_type_id}-${o.source_code_id || 0}`;
+  if (!uuidIsValid(o.input_id)) {
+    throw new Error(`Invalid input_id:'${o.input_id}', type ${typeof o.input_id} -- should be a uuid`);
+  }
+  // get a temp ID
+  const uuid = uuidv5(idString, o.input_id);
+  // Change out the ts to match the v7 sorting.
+  // But because outside specified remote_entry_uuid
+  // may not match this standard, uuid sorting isn't guaranteed
+  return getUUIDv7(ts, uuid);
+}
+function getDateRangeArray(startDate, endDate) {
+  const start = new Date(startDate);
+  const end = new Date(endDate);
+  const result = [];
+  const msInDay = 24 * 60 * 60 * 1000;
+  function addDays(date, days) {
+    const d = new Date(date);
+    d.setDate(d.getDate() + days);
+    return d;
+  }
+  function addMonths(date, months) {
+    const d = new Date(date);
+    d.setMonth(d.getMonth() + months);
+    return d;
+  }
+  function addYears(date, years) {
+    const d = new Date(date);
+    d.setFullYear(d.getFullYear() + years);
+    return d;
+  }
+  const diffDays = Math.floor((end - start) / msInDay);
+  const diffMonths = (end.getFullYear() - start.getFullYear()) * 12 + (end.getMonth() - start.getMonth());
+  const diffYears = end.getFullYear() - start.getFullYear();
+  let current = new Date(start);
+  let stepFn;
+  if (diffDays < 10) {
+    stepFn = (date) => addDays(date, 1);
+  } else if (diffDays < 32) {
+    stepFn = (date) => addDays(date, 3);
+  } else if (diffMonths < 4) {
+    stepFn = (date) => addDays(date, 7);
+  } else if (diffYears < 2) {
+    stepFn = (date) => addMonths(date, 1);
+  } else if (diffYears < 4) {
+    stepFn = (date) => addMonths(date, 3);
+  } else {
+    stepFn = (date) => addYears(date, 1);
+  }
+  while (current <= end) {
+    result.push(new Date(current));
+    const next = stepFn(current);
+    if (next > end) break;
+    current = next;
+  }
+  // Ensure the last date is exactly the end date
+  if (result.length === 0 || result[result.length - 1].getTime() !== end.getTime()) {
+    result.push(new Date(end));
+  }
+  return result;
+}
+class ObjectError extends Error {
+  constructor(data) {
+    if (typeof data === 'string') {
+      // normal behavior
+      super(data);
+    } else if (typeof data === 'object') {
+      super(data.message);
+      Object.keys(data).forEach((k) => {
+        this[k] = data[k];
+      });
+      this.status = data.status;
+    } else {
+      super('(No error message)');
+    }
+  }
+}
+export { appendPostfix };
+export { bool };
+export { create };
+export { list };
+export { downloadFile };
+export { extract };
+export { ForEachEntry };
+export { FileUtilities };
+export { getBatchTransform };
+export { getDateRangeArray };
+export { getDebatchTransform };
+export { getEntryType };
+export { getEntryTypeId };
+export { getFile };
+export { getManifest };
+export { getStringArray };
+export { getTempDir };
+export { getTempFilename };
+export { getTimelineEntryUUID };
+export { getPacketFiles };
+export { getPluginUUID };
+export { getInputUUID };
+export { getUUIDv7 };
+export { getUUIDTimestamp };
+export { handlebars };
+export { isValidDate };
+export { makeStrings };
+export { ObjectError };
+export { parseJSON5 };
+export { relativeDate };
+export { streamPacket };
+export { TIMELINE_ENTRY_TYPES };
+export { writeTempFile };
+export { uuidIsValid };
+export { uuidv4 };
+export { uuidv5 };
+export { uuidv7 };
+export default {
+  appendPostfix,
+  bool,
+  create,
+  list,
+  downloadFile,
+  extract,
+  ForEachEntry,
+  FileUtilities,
+  getBatchTransform,
+  getDateRangeArray,
+  getDebatchTransform,
+  getEntryType,
+  getEntryTypeId,
+  getFile,
+  getManifest,
+  getStringArray,
+  getTempDir,
+  getTempFilename,
+  getTimelineEntryUUID,
+  getPacketFiles,
+  getPluginUUID,
+  getInputUUID,
+  getUUIDv7,
+  getUUIDTimestamp,
+  handlebars,
+  isValidDate,
+  makeStrings,
+  ObjectError,
+  parseJSON5,
+  relativeDate,
+  streamPacket,
+  TIMELINE_ENTRY_TYPES,
+  writeTempFile,
+  uuidIsValid,
+  uuidv4,
+  uuidv5,
+  uuidv7
+};

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@engine9/input-tools",
+  "version": "2.0.7",
+  "type": "module",
+  "description": "Tools for dealing with Engine9 inputs",
+  "main": "index.js",
+  "scripts": {
+    "test-big-data": "export DEBUG=*;node --test --test-name-pattern=big-data",
+    "test": "export DEBUG=*;node --test --test-skip-pattern=big-data"
+  },
+  "author": "Engine9",
+  "license": "GPL-3.0-or-later",
+  "devDependencies": {
+    "eslint": "^9.33.0"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.893.0",
+    "@dsnp/parquetjs": "^1.8.7",
+    "archiver": "^7.0.1",
+    "async-mutex": "^0.5.0",
+    "csv": "^6.3.11",
+    "dayjs": "^1.11.13",
+    "debug": "^4.3.4",
+    "detect-file-encoding-and-language": "^2.4.0",
+    "googleapis": "^148.0.0",
+    "handlebars": "^4.7.8",
+    "json5": "^2.2.3",
+    "mime-type": "^5.0.3",
+    "mkdirp": "^3.0.1",
+    "p-limit": "^7.1.1",
+    "parallel-transform": "^1.2.0",
+    "throttle-debounce": "^5.0.2",
+    "unzipper": "^0.12.1",
+    "uuid": "^11.1.0",
+    "xlstream": "^2.5.5",
+    "yargs": "^17.7.2"
+  },
+  "directories": {
+    "test": "test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/engine9-io/input-tools.git"
+  },
+  "keywords": [
+    "Engine9",
+    "CRM",
+    "CDP"
+  ],
+  "bugs": {
+    "url": "https://github.com/engine9-io/input-tools/issues"
+  },
+  "homepage": "https://github.com/engine9-io/input-tools#readme"
+}

package/skills/transaction-mapping/SKILL.md

@@ -0,0 +1,105 @@
+---
+name: transaction-mapping
+description: Assists in writing JavaScript mapping functions that map 3rd-party payment/transaction data into the standard Transaction schema (Frakture Transactions Data). Use when mapping Stripe, PayPal, donor databases, CSV exports, or any external payment data into engine9 transaction records.
+---
+
+# Transaction Mapping
+
+Use this skill when writing a JavaScript function that maps 3rd-party payment or donation data into the standard Transaction schema. The canonical schema is defined in [Frakture Transactions Data](https://frakture.notion.site/Frakture-Transactions-Data-442349ac436a4f7db8e7d732359e7d8f). The codebase implements this in `interfaces/transaction/schema.js` and processes mapped rows via `interfaces/transaction/transforms/inbound/upsert_tables.js`.
+
+## Mapping workflow
+
+1. **Inspect the source** – Identify which 3rd-party fields correspond to Transaction schema fields (amount, date, person identifier, recurring vs one-time, refunds, etc.).
+2. **Implement a pure mapping function** – `(rawRow) => transactionRow`. Do not set `id` or `entry_type_id` in the mapper; the inbound transform derives them from `getEntryTypeId` and `getTimelineEntryUUID`.
+3. **Return a single object per payment** – One 3rd-party record (or one logical payment) → one Transaction-shaped object. For refunds that are separate rows, map each and use `entry_type: 'TRANSACTION_REFUND'` and `refund_amount` where appropriate.
+4. **Validate required fields** – Ensure every returned object has `ts`, `amount`, `remote_person_id`, and either `entry_type` or `entry_type_id`. The pipeline will throw if any of these are missing.
+
+## Required fields (must be present on every mapped row)
+
+| Field              | Type   | Notes                                                                                                                                              |
+| ------------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `ts`               | Date   | Transaction date/time. Accept ISO string or number (ms); converted with `new Date(ts)`.                                                            |
+| `amount`           | number | Transaction amount (currency units).                                                                                                               |
+| `remote_person_id` | string | Payer/donor identifier from the 3rd-party source (e.g. donor_id, customer_id). The pipeline resolves this to the in-house `person_id`.             |
+| `entry_type`       | string | Or `entry_type_id`. Use a value from [Transaction entry types](#transaction-entry-types) (e.g. `'TRANSACTION_ONE_TIME'`, `'TRANSACTION_INITIAL'`). |
+
+**Note:** Do not set `input_id` in the mapper. It is calculated by the pipeline from `remote_page_name` or `remote_input_id`.
+
+## Optional but important fields
+
+| Field                                                               | Type   | Notes                                                                                                                                 |
+| ------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------- |
+| `refund_amount`                                                     | number | For refunds; use with `entry_type: 'TRANSACTION_REFUND'` when applicable.                                                             |
+| `remote_transaction_id`                                             | string | Idempotency key from 3rd-party system; used for upserts and UUID derivation.                                                          |
+| `remote_page_name`                                                  | string | e.g. campaign or form name; used (with `remote_input_id`) to derive `input_id`.                                                       |
+| `remote_recurring_id`                                               | string | External subscription/recurring series id.                                                                                            |
+| `recurs`                                                            | string | Frequency: `'daily'`, `'weekly'`, `'monthly'`, `'quarterly'`, `'annually'`, `'semi-annually'`. Inbound transform maps to `recurs_id`. |
+| `recurring_number`                                                  | number | Occurrence index in a series (e.g. 1st, 2nd donation in a recurring set).                                                             |
+| `given_name`, `family_name`, `email`                                | string | Payer/donor info; useful for matching or display.                                                                                     |
+| `source_code_id`, `override_source_code_id`, `final_source_code_id` | number | Attribution/source.                                                                                                                   |
+| `recommended_message_id`, `override_message_id`, `final_message_id` | UUID   | Message attribution.                                                                                                                  |
+| `extra`                                                             | object | JSON blob for vendor-specific data that doesn’t fit the schema.                                                                       |
+| `remote_entry_uuid`                                                 | UUID   | If the 3rd-party system provides a stable UUID, set this and it will be used as the record id.                                        |
+
+## Transaction entry types
+
+Use exactly these `entry_type` string values (or the numeric `entry_type_id`). Import `TIMELINE_ENTRY_TYPES` from `@engine9-io/input-tools` for the full map.
+
+| entry_type               | entry_type_id | Use when                                                     |
+| ------------------------ | ------------- | ------------------------------------------------------------ |
+| `TRANSACTION`            | 10            | Generic; prefer a more specific type when known.             |
+| `TRANSACTION_ONE_TIME`   | 11            | Single, non-recurring payment.                               |
+| `TRANSACTION_INITIAL`    | 12            | First payment in a recurring series.                         |
+| `TRANSACTION_SUBSEQUENT` | 13            | Later payment in a recurring series.                         |
+| `TRANSACTION_RECURRING`  | 14            | Recurring payment, order unknown.                            |
+| `TRANSACTION_REFUND`     | 15            | Refund; set `refund_amount` and optionally link to original. |
+
+## Mapping function template
+
+```javascript
+/**
+ * Maps a single 3rd-party payment record to the Transaction schema.
+ * @param {Object} row - Raw record from the payment system (e.g. Stripe charge, CSV row).
+ * @returns {Object} Transaction-shaped object (required: ts, amount, remote_person_id, entry_type).
+ */
+function mapPaymentToTransaction(row) {
+  return {
+    ts: row.created_at ?? row.date ?? row.timestamp,
+    amount: parseFloat(row.amount ?? row.total ?? 0),
+    remote_person_id: row.donor_id ?? row.customer_id ?? row.external_id,
+    entry_type: row.recurring ? 'TRANSACTION_SUBSEQUENT' : 'TRANSACTION_ONE_TIME',
+    remote_transaction_id: row.id ?? row.transaction_id,
+    remote_page_name: row.campaign ?? row.form_name ?? null,
+    email: row.email ?? null,
+    given_name: row.first_name ?? null,
+    family_name: row.last_name ?? null,
+    remote_recurring_id: row.subscription_id ?? null,
+    recurs: row.interval === 'month' ? 'monthly' : null,
+    recurring_number: row.occurrence ?? null,
+    extra: row.raw ? { raw: row.raw } : undefined
+  };
+}
+```
+
+- **Do not** set `id`, `entry_type_id`, or `input_id` in the mapper; the inbound transform/pipeline sets them (e.g. `input_id` from `remote_page_name` or `remote_input_id`).
+- **Do** normalize dates to something `new Date(ts)` can parse (ISO string or ms).
+- **Do** use `remote_person_id` for the 3rd-party payer/donor identifier; the pipeline resolves it to in-house `person_id`.
+- **Do** use `remote_transaction_id` when the source has a stable id for idempotent upserts.
+- **Do** use `entry_type` (string) unless you have a reason to use numeric `entry_type_id`.
+
+## Refunds
+
+- If the source has a separate refund row: set `entry_type: 'TRANSACTION_REFUND'`, `refund_amount`, and `ts`; keep `amount` as 0 or the original amount depending on product rules.
+- If the source only has a flag: you may emit one row with `amount` and optionally `refund_amount`, and still use `TRANSACTION_REFUND` for the entry_type when it’s a refund.
+
+## Validation
+
+- After writing the mapper, verify that sample rows include `ts`, `amount`, `remote_person_id`, and `entry_type` (or `entry_type_id`). The pipeline will throw on the first row missing any of these.
+- Ensure `remote_person_id` is the identifier from the 3rd-party source (e.g. donor_id, customer_id); the pipeline resolves it to the in-house `person_id`.
+
+## Additional reference
+
+- Full schema and column types: [reference.md](reference.md)
+- Schema in code: `interfaces/transaction/schema.js`
+- Inbound transform (sets `id`, `entry_type_id`, `recurs_id`): `interfaces/transaction/transforms/inbound/upsert_tables.js`
+- Entry type constants: `input-tools/timelineTypes.js` (`TIMELINE_ENTRY_TYPES`)

package/skills/transaction-mapping/reference.md

@@ -0,0 +1,72 @@
+# Transaction schema reference
+
+Canonical documentation: [Frakture Transactions Data](https://frakture.notion.site/Frakture-Transactions-Data-442349ac436a4f7db8e7d732359e7d8f).
+
+This file summarizes the schema as implemented in `interfaces/transaction/schema.js` and how the inbound transform and input-tools use it.
+
+## Table: transaction
+
+| Column                  | DB type        | Required for mapping?     | Notes                                                                                                            |
+| ----------------------- | -------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+| id                      | id_uuid        | No (set by transform)     | Set by `getTimelineEntryUUID` in inbound transform.                                                              |
+| ts                      | datetime       | **Yes**                   | Transaction date/time.                                                                                           |
+| input_id                | uuid           | No (calculated)           | Derived by pipeline from `remote_page_name` or `remote_input_id`; do not set in mapper.                          |
+| entry_type_id           | int            | **Yes** (or entry_type)   | Set by transform from `entry_type` string if not provided.                                                       |
+| person_id               | person_id      | No (resolved by pipeline) | In-house person id; resolved from `remote_person_id` by the pipeline.                                            |
+| remote_person_id        | string         | **Yes**                   | Payer/donor identifier from 3rd-party source; pipeline resolves to `person_id`.                                  |
+| amount                  | currency       | **Yes**                   | Transaction amount (currency units).                                                                             |
+| remote_transaction_id   | string         | No                        | External id; used for upserts and id derivation.                                                                 |
+| remote_page_name        | string         | No                        | e.g. campaign/form name; used (with remote_input_id) to derive input_id.                                         |
+| remote_recurring_id     | string         | No                        | External recurring/subscription id.                                                                              |
+| recurs_id               | int            | No                        | Set by transform from `recurs` (daily=1, weekly=2, monthly=3, quarterly=4, annually=5, semi-annually=6, else 0). |
+| recurring_number        | int            | No                        | Occurrence index in series.                                                                                      |
+| refund_amount           | currency       | No                        | For refunds.                                                                                                     |
+| given_name              | string         | No                        | Payer first name.                                                                                                |
+| family_name             | string         | No                        | Payer last name.                                                                                                 |
+| email                   | string         | No                        | Payer email.                                                                                                     |
+| source_code_id          | source_code_id | No                        | Attribution.                                                                                                     |
+| override_source_code_id | source_code_id | No                        |                                                                                                                  |
+| final_source_code_id    | source_code_id | No                        |                                                                                                                  |
+| recommended_message_id  | uuid           | No                        |                                                                                                                  |
+| override_message_id     | uuid           | No                        |                                                                                                                  |
+| final_message_id        | uuid           | No                        |                                                                                                                  |
+| extra                   | json           | No                        | Arbitrary vendor-specific data.                                                                                  |
+
+## recurs → recurs_id (inbound transform)
+
+The transform maps `recurs` (string) to `recurs_id` (int) when not already set:
+
+| recurs                        | recurs_id |
+| ----------------------------- | --------- |
+| 'daily'                       | 1         |
+| 'weekly'                      | 2         |
+| 'monthly'                     | 3         |
+| 'quarterly'                   | 4         |
+| 'annually'                    | 5         |
+| 'semi-annually'               | 6         |
+| (none) + recurring_number > 1 | 3         |
+| (none)                        | 0         |
+
+## Transaction entry_type → entry_type_id
+
+From `input-tools/timelineTypes.js`:
+
+| entry_type             | entry_type_id |
+| ---------------------- | ------------- |
+| TRANSACTION            | 10            |
+| TRANSACTION_ONE_TIME   | 11            |
+| TRANSACTION_INITIAL    | 12            |
+| TRANSACTION_SUBSEQUENT | 13            |
+| TRANSACTION_RECURRING  | 14            |
+| TRANSACTION_REFUND     | 15            |
+
+## Required fields from the mapper
+
+The mapper must provide (the pipeline derives or resolves the rest):
+
+- `ts`
+- `amount`
+- `remote_person_id` (3rd-party payer/donor identifier; pipeline resolves to `person_id`)
+- `entry_type` or `entry_type_id`
+
+The pipeline calculates `input_id` from `remote_page_name` or `remote_input_id`. The inbound transform uses `input_id` and `person_id` (resolved from `remote_person_id`) with `getTimelineEntryUUID` to set `id`. If `remote_entry_uuid` is set and valid, it is used as `id` and the composite is not needed.

package/test/cli.js

@@ -0,0 +1,9 @@
+import yargs from 'yargs/yargs';
+import methods from '../index.js';
+const argv = yargs(process.argv.slice(2)).parse();
+async function run() {
+  if (typeof methods[argv._[0]] !== 'function') throw new Error(`${argv._[0]} is not a function`);
+  const output = await methods[argv._[0]](argv);
+  console.log(output);
+}
+run();