@arcforgelabs/flow 2026.10.26 → 2026.10.29

package/server/core/line-mapping.js

@@ -0,0 +1,337 @@
+import { and, asc, eq, isNull, sql } from 'drizzle-orm';
+import { LINE_MAPPING_KINDS, lineMappings, pipelineCursors, priceItems } from '../db/schema.js';
+import { FlowError, notFound } from './errors.js';
+
+// Line mapping (docs/LINE_MAPPING.md): how each line a work system reports
+// finds the lab's price item before the job reaches the ledger. The routine is
+// a list of steps, tried in order; the first step that answers wins. Every
+// step reads either the lab's price list or the lab's own mapping rows, so no
+// lab's labels or codes are fixed in Flow's code.
+//
+//   same_code           the line's own code is a price item code
+//   account             an account row: a line with no price item (a surcharge)
+//   saved               a row a person saved for this label at this price
+//   catalog_name        exactly one price item has this name
+//   price_alias         a row for this label at this price
+//   catalog_name_price  the price items with this name, at this price: one wins,
+//                       more than one stops the line for a person
+//   catalog_score       the price item at this price whose name is closest
+//   alias               a row for this label, or for the start of it
+//
+// A mapped line keeps the work system's amount and tax flag; the price item
+// gives its code, its text and, when the work system gave none, its price and
+// tax. A line no step answers keeps no code, so the ledger holds the job.
+
+export const STEPS = Object.freeze([
+  'same_code', 'account', 'saved', 'catalog_name', 'price_alias', 'catalog_name_price', 'catalog_score', 'alias',
+]);
+export const STEP_LABELS = Object.freeze({
+  same_code: 'Same code',
+  account: 'Account lines',
+  saved: 'Saved choices',
+  catalog_name: 'Price list name',
+  price_alias: 'Label and price',
+  catalog_name_price: 'Price list name and price',
+  catalog_score: 'Closest name at the same price',
+  alias: 'Label',
+});
+export const ZERO_LINES = Object.freeze(['keep', 'drop']);
+export const MAPPING_DEFAULTS = Object.freeze({ steps: [...STEPS], zeroLines: 'keep' });
+
+const SCORE_MIN = 16;
+const SCORE_GAP = 8;
+const MAX_MAPPINGS = 2000;
+const invalid = (message) => new FlowError('invalid', message);
+
+// Text in the form every comparison uses: lower case, "&" as "and", every
+// other mark a space.
+export function lineText(value) {
+  return String(value ?? '').trim().toLowerCase().replace(/&/g, ' and ').replace(/[^a-z0-9]+/g, ' ').replace(/\s+/g, ' ').trim();
+}
+
+// The label without what varies between orders of the same item: sizes
+// ("2 x 3"), arch ("upper", "lower 4") and "per set" or "per unit".
+export function lineLookupText(value) {
+  return lineText(value)
+    .replace(/\b\d+\s*x\s*\d+(?:\s+\d+)?\b/g, ' ')
+    .replace(/\b(upper|lower)\s+\d+\b/g, ' ')
+    .replace(/\b(upper|lower)\b/g, ' ')
+    .replace(/\bper set\b/g, ' ')
+    .replace(/\bper unit\b/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+// A label that carries its own quantity and unit price: "Teeth (6 x $12.50)".
+export function quantityOf(text) {
+  const raw = String(text ?? '').trim();
+  const found = raw.match(/\((\d+(?:\.\d+)?)\s*x\s*\$?([0-9]+(?:\.\d{1,2})?)\)/i);
+  if (!found) return { base: raw, quantity: 1, unitAmountCents: null };
+  const quantity = Number(found[1]);
+  const base = raw.replace(found[0], ' ').replace(/\s+/g, ' ').trim();
+  return {
+    base: base || raw,
+    quantity: Number.isInteger(quantity) && quantity > 0 ? quantity : 1,
+    unitAmountCents: Number.isInteger(quantity) && quantity > 0 ? Math.round(Number(found[2]) * 100) : null,
+  };
+}
+
+const tokens = (text) => [...new Set(text.split(' ').filter((t) => t.length > 1))];
+
+// How close a price item's name is to a label. The weights are the earlier
+// lab app's, so its results carry over.
+export function nameScore(item, { exact, lookup, original }) {
+  const texts = [lineText(item.name), lineText(item.description)];
+  const lookups = [lineLookupText(item.name), lineLookupText(item.description)];
+  let score = 0;
+  if (exact && texts.includes(exact)) score += 120;
+  if (lookup && lookups.includes(lookup)) score += 90;
+  if (lookup && lookups.some((t) => t && (t.includes(lookup) || lookup.includes(t)))) score += 35;
+  const lookupTokens = new Set(lookups.flatMap(tokens));
+  for (const token of tokens(lookup)) if (lookupTokens.has(token)) score += 8;
+  const exactTokens = new Set(texts.flatMap(tokens));
+  for (const token of tokens(exact)) if (exactTokens.has(token)) score += 8;
+  if (original && texts.some((t) => t && t.includes(original))) score += 20;
+  return score;
+}
+
+// The lab's price list and mapping rows, indexed once for a run.
+export function mappingContext({ items = [], mappings = [], config = MAPPING_DEFAULTS } = {}) {
+  const live = items.filter((i) => !i.archivedAt);
+  const byCode = new Map(live.map((i) => [i.code, i]));
+  const byCodeText = new Map(live.map((i) => [lineText(i.code), i]));
+  const byText = new Map();
+  const byPrice = new Map();
+  const add = (map, key, item) => {
+    if (!key) return;
+    if (!map.has(key)) map.set(key, []);
+    if (!map.get(key).includes(item)) map.get(key).push(item);
+  };
+  for (const item of live) {
+    for (const text of [lineText(item.name), lineText(item.description), lineLookupText(item.name), lineLookupText(item.description)]) add(byText, text, item);
+    add(byPrice, item.unitAmountCents, item);
+  }
+  const rows = mappings.filter((m) => m.enabled !== false);
+  const rowsOf = (kind) => rows.filter((m) => m.kind === kind);
+  return { byCode, byCodeText, byText, byPrice, rowsOf, config: { ...MAPPING_DEFAULTS, ...config } };
+}
+
+const rowMatches = (row, lookup) => (row.match === 'starts_with' ? lookup === row.label || lookup.startsWith(`${row.label} `) : lookup === row.label);
+
+// Maps one line. Answers the line as the ledger takes it, with `mappedBy` (the
+// step) and `label` (the lookup text), or `skip: true` for a zero line the
+// routine drops. A line no step answers comes back with `unmapped` set.
+export function mapLine(line, context) {
+  const { config } = context;
+  const original = String(line.description ?? '').trim();
+  const meta = quantityOf(original);
+  const exact = lineText(meta.base);
+  const lookup = lineLookupText(meta.base);
+  const lineCents = line.unitAmountCents ?? null;
+  const price = meta.unitAmountCents ?? lineCents;
+  const quantity = meta.unitAmountCents !== null ? meta.quantity : (line.quantity ?? 1);
+  const base = { ...line, label: lookup || exact || null };
+  if (lineCents !== null && lineCents <= 0 && config.zeroLines === 'drop') return { ...base, skip: true, mappedBy: 'zero_line' };
+
+  const toItem = (item, step, taxCode = null) => ({
+    ...base,
+    itemCode: item.code,
+    accountCode: null,
+    description: (item.description || item.name || original).trim(),
+    quantity,
+    unitAmountCents: price ?? item.unitAmountCents,
+    taxCode: taxCode ?? line.taxCode ?? item.taxCode,
+    mappedBy: step,
+  });
+  const fromRow = (row, step) => {
+    const item = context.byCode.get(row.itemCode);
+    return item ? toItem(item, step, row.taxCode) : null;
+  };
+  const unmapped = (reason) => ({ ...base, itemCode: null, accountCode: null, mappedBy: null, unmapped: reason });
+
+  for (const step of config.steps) {
+    switch (step) {
+      case 'same_code': {
+        const item = line.itemCode ? context.byCode.get(line.itemCode.trim()) ?? context.byCodeText.get(lineText(line.itemCode)) : null;
+        if (item) return toItem(item, step);
+        break;
+      }
+      case 'account': {
+        const row = context.rowsOf('account').find((r) => rowMatches(r, lookup));
+        if (row) {
+          return {
+            ...base, itemCode: null, accountCode: row.accountCode, description: row.description || original, quantity,
+            unitAmountCents: price, taxCode: row.taxCode ?? line.taxCode, mappedBy: step,
+          };
+        }
+        break;
+      }
+      case 'saved':
+      case 'price_alias': {
+        if (price === null) break;
+        const row = context.rowsOf(step).find((r) => r.unitAmountCents === price && rowMatches(r, lookup));
+        const mapped = row && fromRow(row, step);
+        if (mapped) return mapped;
+        break;
+      }
+      case 'catalog_name': {
+        const found = [...new Set([...(context.byText.get(exact) ?? []), ...(context.byText.get(lookup) ?? [])])];
+        if (found.length === 1) return toItem(found[0], step);
+        break;
+      }
+      case 'catalog_name_price': {
+        if (price === null) break;
+        const found = [...new Set([...(context.byText.get(exact) ?? []), ...(context.byText.get(lookup) ?? [])])]
+          .filter((i) => i.unitAmountCents === price);
+        if (found.length === 1) return toItem(found[0], step);
+        if (found.length > 1) return unmapped(`more than one price item is named "${original}" at this price`);
+        break;
+      }
+      case 'catalog_score': {
+        if (price === null) break;
+        const scored = (context.byPrice.get(price) ?? [])
+          .map((item) => ({ item, score: nameScore(item, { exact, lookup, original: lineText(original) }) }))
+          .sort((a, b) => b.score - a.score || a.item.name.localeCompare(b.item.name) || a.item.code.localeCompare(b.item.code));
+        const [top, next] = scored;
+        if (top && top.score >= SCORE_MIN && !(next && top.score < next.score + SCORE_GAP)) return toItem(top.item, step);
+        break;
+      }
+      case 'alias': {
+        const rows = context.rowsOf('alias');
+        const row = rows.find((r) => r.match === 'exact' && rowMatches(r, lookup)) ?? rows.find((r) => r.match === 'starts_with' && rowMatches(r, lookup));
+        const mapped = row && fromRow(row, step);
+        if (mapped) return mapped;
+        break;
+      }
+      default:
+        break;
+    }
+  }
+  return unmapped(original ? `no mapping step found a price item for "${original}"` : 'the line has no text');
+}
+
+// Maps every line of a job. Lines the routine drops leave the job.
+export function mapLines(lines, context) {
+  return lines.map((line) => mapLine(line, context)).filter((l) => !l.skip)
+    .map(({ skip: _skip, unmapped: _unmapped, ...line }) => line);
+}
+
+function readConfig(raw = {}) {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw invalid('the routine settings must be an object');
+  const out = { ...MAPPING_DEFAULTS };
+  if ('steps' in raw) {
+    if (!Array.isArray(raw.steps) || raw.steps.some((s) => !STEPS.includes(s)) || new Set(raw.steps).size !== raw.steps.length) {
+      throw invalid(`steps must list some of ${STEPS.join(', ')}, each once`);
+    }
+    out.steps = raw.steps;
+  }
+  if ('zeroLines' in raw) {
+    if (!ZERO_LINES.includes(raw.zeroLines)) throw invalid(`zeroLines must be one of ${ZERO_LINES.join(', ')}`);
+    out.zeroLines = raw.zeroLines;
+  }
+  return out;
+}
+export { readConfig as readMappingConfig };
+
+const shape = (r) => ({
+  id: r.id, kind: r.kind, match: r.match, label: r.label, unitAmountCents: r.unitAmountCents, itemCode: r.itemCode,
+  accountCode: r.accountCode, taxCode: r.taxCode, description: r.description, enabled: r.enabled, origin: r.origin, updatedAt: r.updatedAt,
+});
+
+const shortText = (value, name, max = 200) => {
+  if (value === null || value === undefined || value === '') return null;
+  if (typeof value !== 'string' || value.trim().length > max) throw invalid(`${name} must be text of ${max} characters at most`);
+  return value.trim() || null;
+};
+
+export function readMapping(raw) {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw invalid('the mapping must be an object');
+  const known = ['kind', 'match', 'label', 'unitAmountCents', 'itemCode', 'accountCode', 'taxCode', 'description', 'enabled'];
+  const unknown = Object.keys(raw).filter((k) => !known.includes(k));
+  if (unknown.length) throw invalid(`unknown field: ${unknown[0]}`);
+  if (!LINE_MAPPING_KINDS.includes(raw.kind)) throw invalid(`kind must be one of ${LINE_MAPPING_KINDS.join(', ')}`);
+  const match = raw.match ?? 'exact';
+  if (!['exact', 'starts_with'].includes(match)) throw invalid('match must be exact or starts_with');
+  const label = lineLookupText(raw.label) || lineText(raw.label);
+  if (!label || label.length > 200) throw invalid('the label must be text of 200 characters at most');
+  const priced = ['price_alias', 'saved'].includes(raw.kind);
+  let unitAmountCents = raw.unitAmountCents ?? null;
+  if (unitAmountCents !== null && (!Number.isInteger(unitAmountCents) || unitAmountCents < 0 || unitAmountCents > 100_000_000)) {
+    throw invalid('unitAmountCents must be whole cents');
+  }
+  if (priced && unitAmountCents === null) throw invalid('this kind needs a unit price');
+  if (!priced) unitAmountCents = null;
+  const taxCode = raw.taxCode ?? null;
+  if (taxCode !== null && !['GST', 'GST_FREE'].includes(taxCode)) throw invalid('taxCode must be GST or GST_FREE');
+  const itemCode = shortText(raw.itemCode, 'itemCode', 100);
+  const accountCode = shortText(raw.accountCode, 'accountCode', 20);
+  if (raw.kind === 'account' ? !accountCode : !itemCode) throw invalid(raw.kind === 'account' ? 'an account line needs an account code' : 'the mapping needs a price item');
+  const enabled = raw.enabled ?? true;
+  if (typeof enabled !== 'boolean') throw invalid('enabled must be true or false');
+  return {
+    kind: raw.kind, match, label, unitAmountCents,
+    itemCode: raw.kind === 'account' ? null : itemCode,
+    accountCode: raw.kind === 'account' ? accountCode : null,
+    taxCode, description: shortText(raw.description, 'description'), enabled,
+  };
+}
+
+export async function listMappings(db, organisationId) {
+  const rows = await db.select().from(lineMappings).where(eq(lineMappings.organisationId, organisationId))
+    .orderBy(asc(lineMappings.kind), asc(lineMappings.label), asc(lineMappings.unitAmountCents));
+  return rows.map(shape);
+}
+
+// A change to the mappings reads every job again on the next pipeline run, so
+// jobs already in the ledger take the new mapping. It reads Flow's own copy of
+// the work system, not the work system.
+export async function remapJobs(db, organisationId) {
+  await db.delete(pipelineCursors).where(and(eq(pipelineCursors.organisationId, organisationId), sql`${pipelineCursors.name} like '%.work'`));
+}
+
+export async function createMapping(db, organisationId, raw, { personId = null, origin = 'person' } = {}) {
+  const mapping = readMapping(raw);
+  const [{ count }] = await db.select({ count: sql`count(*)::int` }).from(lineMappings).where(eq(lineMappings.organisationId, organisationId));
+  if (count >= MAX_MAPPINGS) throw invalid(`an organisation can have ${MAX_MAPPINGS} line mappings at most`);
+  const [row] = await db.insert(lineMappings).values({ organisationId, ...mapping, origin, updatedBy: personId })
+    .onConflictDoNothing().returning();
+  if (!row) throw new FlowError('conflict', 'a mapping for this label already exists');
+  await remapJobs(db, organisationId);
+  return shape(row);
+}
+
+async function loadMapping(db, organisationId, id) {
+  const [row] = await db.select().from(lineMappings).where(and(eq(lineMappings.organisationId, organisationId), eq(lineMappings.id, id)));
+  if (!row) throw notFound('mapping');
+  return row;
+}
+
+export async function updateMapping(db, organisationId, id, changes, { personId = null } = {}) {
+  const row = await loadMapping(db, organisationId, id);
+  if (!changes || typeof changes !== 'object' || Array.isArray(changes)) throw invalid('the change must be an object');
+  const mapping = readMapping({ ...shape(row), ...changes, id: undefined, origin: undefined, updatedAt: undefined });
+  try {
+    const [saved] = await db.update(lineMappings).set({ ...mapping, updatedBy: personId, updatedAt: sql`now()` })
+      .where(eq(lineMappings.id, row.id)).returning();
+    await remapJobs(db, organisationId);
+    return shape(saved);
+  } catch (error) {
+    if (error?.code === '23505' || error?.cause?.code === '23505') throw new FlowError('conflict', 'a mapping for this label already exists');
+    throw error;
+  }
+}
+
+export async function deleteMapping(db, organisationId, id) {
+  const row = await loadMapping(db, organisationId, id);
+  await db.delete(lineMappings).where(eq(lineMappings.id, row.id));
+  await remapJobs(db, organisationId);
+}
+
+// The price list and mapping rows for a pipeline run.
+export async function loadMappingContext(db, organisationId, config) {
+  const [items, mappings] = await Promise.all([
+    db.select().from(priceItems).where(and(eq(priceItems.organisationId, organisationId), isNull(priceItems.archivedAt))),
+    db.select().from(lineMappings).where(eq(lineMappings.organisationId, organisationId)),
+  ]);
+  return mappingContext({ items, mappings, config });
+}

package/server/core/monitoring.js

@@ -0,0 +1,34 @@
+// The logger every part of Flow writes to. It passes each line to the host's
+// own log first, unchanged, then sends each error to the error services that
+// are on (the `captureError` capability, docs/SENTRY.md). With none on, it is
+// the host's log. A line's leading "[tag]" is its source.
+//
+// `reporters()` answers the adapters to send to; it is read at each error,
+// so an integration turned on or off later is followed. A failure to send is
+// logged as a warning at most once a minute, never sent, so it cannot loop.
+export function createMonitoredLog({ log = console, reporters = () => [], now = Date.now }) {
+  let lastFailure = -Infinity;
+
+  function error(...args) {
+    log.error?.(...args);
+    const cause = args.find((arg) => arg instanceof Error) ?? null;
+    const message = args.map((arg) => (arg instanceof Error ? arg.message : String(arg))).join(' ');
+    const source = /^\[([a-z0-9_-]+)\]/i.exec(message)?.[1].toLowerCase() ?? 'server';
+    for (const adapter of reporters()) {
+      try {
+        adapter.capabilities.captureError({ message, source, error: cause });
+      } catch (failure) {
+        const at = now();
+        if (at - lastFailure < 60_000) continue;
+        lastFailure = at;
+        log.warn?.(`[monitoring] ${adapter.id} could not take an error: ${failure.message}`);
+      }
+    }
+  }
+
+  return {
+    info: (...args) => log.info?.(...args),
+    warn: (...args) => log.warn?.(...args),
+    error,
+  };
+}

package/server/core/pipeline.js

@@ -4,7 +4,10 @@ import {
   billingObligations, contacts, externalLinks, linkRejections, linkSuggestions, obligationEvents, organisations, pipelineCursors,
 } from '../db/schema.js';
 import { FlowError } from './errors.js';
+import { listAutomations, runAutomations } from './automations.js';
+import { readInvoicingSettings } from './invoicing-rules.js';
 import { observeJob, recordAccountingEvidence, setHold } from './ledger.js';
+import { loadMappingContext, mapLines } from './line-mapping.js';
 import { linkRecord, priorLinkFor } from './links.js';
 import { enqueueJob, runNextJob } from './queue.js';
 
@@ -20,6 +23,9 @@ import { enqueueJob, runNextJob } from './queue.js';
 //    sales invoices. Exact evidence links the job. A likely invoice is a
 //    suggestion for a person. When the copy is complete and fresh and holds no
 //    invoice that could bill the job, the job has none.
+// 4. Automations (automations.js): the routines that are on, for example
+//    marking a job ready to invoice from its work system status, and drafting
+//    the invoice of a ready job. Drafts wait in Flow for a person.
 //
 // The matcher reads only Flow's copies, so matching years of history costs no
 // API calls (docs/JOB_INVOICE_LINKS.md).
@@ -129,13 +135,18 @@ export async function ingestWork(db, organisationId, { adapter, restart = false,
   let cursor = restart ? null : await readCursor(db, organisationId, name);
   const links = await activeLinks(db, organisationId, { system: adapter.id, entity: 'contact' });
   const contactOf = new Map(links.map((l) => [l.externalId, l.flowId]));
-  const result = { observed: 0, withoutCustomer: 0, skipped: 0, sentToReview: 0, more: false };
+  const result = { observed: 0, withoutCustomer: 0, skipped: 0, sentToReview: 0, unmappedLines: 0, more: false };
+  // Line mapping (line-mapping.js): each line finds its price item first.
+  const mapping = (await listAutomations(db, organisationId)).find((a) => a.key === 'map_lines');
+  const context = mapping?.enabled ? await loadMappingContext(db, organisationId, mapping.config) : null;
   for (let page = 0; page < pages; page += 1) {
     const { items, withdrawn, cursor: next, more } = await adapter.capabilities.pullWorkItems(cursor, { limit: 200 });
     for (const item of items) {
       const contactId = contactOf.get(String(item.clientId ?? '')) ?? null;
+      const job = context ? { ...item.ledgerJob, lines: mapLines(item.ledgerJob.lines ?? [], context) } : item.ledgerJob;
+      if (context) result.unmappedLines += job.lines.filter((l) => !l.itemCode && !l.accountCode).length;
       try {
-        await observeJob(db, organisationId, { ...item.ledgerJob, contactId });
+        await observeJob(db, organisationId, { ...job, contactId });
         result.observed += 1;
         if (!contactId) result.withoutCustomer += 1;
       } catch (error) {
@@ -425,8 +436,10 @@ export async function runPipelineOnce(db, integrations, { now = new Date() } = {
         system: accounting.id, customers: await accounting.capabilities.listCustomers({ organisationId }),
       });
     }
-    summary.matching = await matchJobs(db, organisationId, { accounting, now });
+    const settings = await readInvoicingSettings(db, organisationId);
+    summary.matching = await matchJobs(db, organisationId, { accounting, now, staleDays: settings.staleDays });
   }
+  if (work || accounting) summary.automations = await runAutomations(db, organisationId, { accounting: accounting ?? null, now });
   return summary;
 }