@arcforgelabs/flow 2026.10.20-beta.1 → 2026.10.22

package/server/db/migrations/meta/_journal.json

@@ -36,6 +36,13 @@
       "when": 1792329649575,
       "tag": "0004_ledger_events_append_only",
       "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1792388648997,
+      "tag": "0005_data_spine",
+      "breakpoints": true
     }
   ]
 }

package/server/db/schema.js

@@ -1,15 +1,18 @@
 import { sql } from 'drizzle-orm';
 import {
+  bigint,
   boolean,
   check,
   date,
   foreignKey,
+  index,
   integer,
   jsonb,
   pgTable,
   text,
   timestamp,
   unique,
+  uniqueIndex,
   uuid,
 } from 'drizzle-orm/pg-core';
 
@@ -304,3 +307,130 @@ export const obligationWritebacks = pgTable('obligation_writebacks', {
   }),
   check('obligation_writebacks_state_valid', inList('state', WRITEBACK_STATES)),
 ]);
+
+// The data spine (docs/DATA_SPINE.md, issue 56): the tables every integration
+// shares. Each row changes state; the spine never deletes one.
+
+// Identity map (issue 57). Which external record is each Flow record. At most
+// one active link per external record and per Flow record, in each system. A
+// link that is not exact stays `proposed` until a person confirms it.
+export const LINK_ENTITIES = ['contact', 'price_item', 'obligation', 'invoice', 'payment', 'credit_note'];
+export const LINK_MATCHES = ['created_by_flow', 'exact_id', 'imported', 'name', 'person'];
+export const LINK_STATES = ['active', 'proposed', 'superseded'];
+
+export const externalLinks = pgTable('external_links', {
+  id: id(),
+  organisationId: organisationId().references(() => organisations.id),
+  entity: text('entity').notNull(),
+  flowId: uuid('flow_id').notNull(),
+  system: text('system').notNull(),
+  externalId: text('external_id').notNull(),
+  matchedBy: text('matched_by').notNull(),
+  state: text('state').notNull(),
+  confirmedBy: uuid('confirmed_by').references(() => people.id, { onDelete: 'set null' }),
+  confirmedAt: timestamp('confirmed_at', { withTimezone: true }),
+  createdAt: createdAt(),
+  updatedAt: updatedAt(),
+}, (t) => [
+  unique('external_links_org_id').on(t.organisationId, t.id),
+  uniqueIndex('external_links_active_external')
+    .on(t.organisationId, t.system, t.entity, t.externalId).where(sql`state = 'active'`),
+  uniqueIndex('external_links_active_flow')
+    .on(t.organisationId, t.system, t.entity, t.flowId).where(sql`state = 'active'`),
+  index('external_links_flow').on(t.organisationId, t.entity, t.flowId),
+  check('external_links_entity_valid', inList('entity', LINK_ENTITIES)),
+  check('external_links_matched_by_valid', inList('matched_by', LINK_MATCHES)),
+  check('external_links_state_valid', inList('state', LINK_STATES)),
+  check('external_links_name_needs_person', sql`${t.matchedBy} <> 'name' or ${t.state} <> 'active' or ${t.confirmedAt} is not null`),
+]);
+
+// Background job queue (issue 6). Workers claim with FOR UPDATE SKIP LOCKED and
+// hold a lease; a lease that runs out makes the job claimable again. One queued
+// or running job per (organisation, kind, dedupe key).
+export const BACKGROUND_JOB_STATES = ['queued', 'running', 'succeeded', 'failed', 'dead'];
+
+export const backgroundJobs = pgTable('background_jobs', {
+  id: id(),
+  organisationId: organisationId().references(() => organisations.id),
+  kind: text('kind').notNull(),
+  dedupeKey: text('dedupe_key').notNull().default(''),
+  payload: jsonb('payload').notNull().default(sql`'{}'::jsonb`),
+  state: text('state').notNull().default('queued'),
+  runAfter: timestamp('run_after', { withTimezone: true }).notNull().defaultNow(),
+  attempts: integer('attempts').notNull().default(0),
+  maxAttempts: integer('max_attempts').notNull().default(8),
+  lockedBy: text('locked_by'),
+  lockedUntil: timestamp('locked_until', { withTimezone: true }),
+  result: jsonb('result'),
+  lastError: text('last_error'),
+  createdAt: createdAt(),
+  updatedAt: updatedAt(),
+}, (t) => [
+  uniqueIndex('background_jobs_one_open')
+    .on(t.organisationId, t.kind, t.dedupeKey).where(sql`state in ('queued', 'running')`),
+  index('background_jobs_due').on(t.state, t.runAfter),
+  check('background_jobs_state_valid', inList('state', BACKGROUND_JOB_STATES)),
+  check('background_jobs_attempts_valid', sql`${t.attempts} >= 0 and ${t.maxAttempts} > 0`),
+]);
+
+// Outbox. Every write to another system is a row here before the call
+// leaves, written in the same transaction as the Flow change it reports. The
+// idempotency key is unique, so a retry never makes a second record.
+export const OUTBOX_STATES = ['pending', 'sending', 'ambiguous', 'sent', 'failed', 'cancelled'];
+
+export const outbox = pgTable('outbox', {
+  id: id(),
+  organisationId: organisationId().references(() => organisations.id),
+  system: text('system').notNull(),
+  action: text('action').notNull(),
+  entity: text('entity').notNull(),
+  flowId: uuid('flow_id').notNull(),
+  idempotencyKey: text('idempotency_key').notNull(),
+  request: jsonb('request').notNull(),
+  state: text('state').notNull().default('pending'),
+  externalId: text('external_id'),
+  attempts: integer('attempts').notNull().default(0),
+  lastError: text('last_error'),
+  createdBy: text('created_by').notNull(),
+  sentAt: timestamp('sent_at', { withTimezone: true }),
+  createdAt: createdAt(),
+  updatedAt: updatedAt(),
+}, (t) => [
+  unique('outbox_org_id').on(t.organisationId, t.id),
+  unique('outbox_idempotency_key').on(t.idempotencyKey),
+  index('outbox_pending').on(t.system, t.state, t.createdAt),
+  check('outbox_state_valid', inList('state', OUTBOX_STATES)),
+  check('outbox_sent_has_id', sql`${t.state} <> 'sent' or ${t.externalId} is not null`),
+]);
+
+// Review queue (issue 59). One list of everything a person must decide. One
+// open item per (organisation, kind, entity, Flow id); a repeat finding
+// updates its evidence.
+export const REVIEW_STATES = ['open', 'resolved', 'dismissed'];
+
+export const reviewItems = pgTable('review_items', {
+  id: id(),
+  organisationId: organisationId().references(() => organisations.id),
+  kind: text('kind').notNull(),
+  entity: text('entity').notNull(),
+  flowId: uuid('flow_id').notNull(),
+  systems: text('systems').array().notNull().default(sql`'{}'::text[]`),
+  evidence: jsonb('evidence').notNull().default(sql`'{}'::jsonb`),
+  moneyAtRiskCents: bigint('money_at_risk_cents', { mode: 'number' }).notNull().default(0),
+  state: text('state').notNull().default('open'),
+  seenCount: integer('seen_count').notNull().default(1),
+  resolution: text('resolution'),
+  note: text('note'),
+  resolvedBy: text('resolved_by'),
+  resolvedAt: timestamp('resolved_at', { withTimezone: true }),
+  createdAt: createdAt(),
+  updatedAt: updatedAt(),
+}, (t) => [
+  unique('review_items_org_id').on(t.organisationId, t.id),
+  uniqueIndex('review_items_one_open')
+    .on(t.organisationId, t.kind, t.entity, t.flowId).where(sql`state = 'open'`),
+  index('review_items_queue').on(t.organisationId, t.state, t.moneyAtRiskCents),
+  check('review_items_state_valid', inList('state', REVIEW_STATES)),
+  check('review_items_money_not_negative', sql`${t.moneyAtRiskCents} >= 0`),
+  check('review_items_closed_has_resolution', sql`${t.state} = 'open' or (${t.resolution} is not null and ${t.resolvedBy} is not null and ${t.resolvedAt} is not null)`),
+]);

package/server/import/at-start.js

@@ -0,0 +1,52 @@
+import { sql } from 'drizzle-orm';
+import { organisations } from '../db/schema.js';
+import { applyImport, planImport, readEarlierFlow, summarisePlan } from './earlier-flow.js';
+
+// The import from the earlier codebase, run once by the server itself. On a
+// host with no shell next to the app (a published Replit app) the operator
+// moves the earlier tables into their own schema, then sets
+// FLOW_IMPORT_EARLIER_SCHEMA and FLOW_IMPORT_ORGANISATION. While the database
+// has no organisation, the next start reads that schema in a read-only
+// transaction and writes the records (server/import/earlier-flow.js). Once an
+// organisation exists, it does nothing.
+
+const SCHEMA_NAME = /^[a-z_][a-z0-9_]{0,62}$/;
+
+export function resolveImportAtStart(env = process.env) {
+  const schema = (env.FLOW_IMPORT_EARLIER_SCHEMA ?? '').trim();
+  if (!schema) return null;
+  if (!SCHEMA_NAME.test(schema)) throw new Error('FLOW_IMPORT_EARLIER_SCHEMA must be a plain lower-case schema name, e.g. earlier');
+  const organisationName = (env.FLOW_IMPORT_ORGANISATION ?? '').trim();
+  if (!organisationName) throw new Error('FLOW_IMPORT_ORGANISATION (the lab\'s name) is required with FLOW_IMPORT_EARLIER_SCHEMA');
+  return { schema, organisationName };
+}
+
+// Returns the import's counts, or null when there was nothing to do.
+export async function importEarlierAtStart({ db, pool, settings, log = console }) {
+  if (!settings) return null;
+  const [{ count }] = (await db.execute(sql`select count(*)::int as count from ${organisations}`)).rows;
+  if (count > 0) {
+    log.info?.('[import] skipped: this database already has an organisation (unset FLOW_IMPORT_EARLIER_SCHEMA)');
+    return null;
+  }
+
+  let plan;
+  const client = await pool.connect();
+  try {
+    await client.query('BEGIN TRANSACTION READ ONLY');
+    // The schema name was checked against SCHEMA_NAME, so it is safe to quote.
+    await client.query(`SET LOCAL search_path TO "${settings.schema}"`);
+    plan = planImport(await readEarlierFlow(client));
+    await client.query('COMMIT');
+  } catch (error) {
+    await client.query('ROLLBACK').catch(() => {});
+    throw new Error(`could not read schema "${settings.schema}": ${error.message}`);
+  } finally {
+    client.release();
+  }
+
+  log.info?.(`[import] plan from schema "${settings.schema}": ${JSON.stringify(summarisePlan(plan))}`);
+  const result = await applyImport(db, plan, { organisationName: settings.organisationName });
+  log.info?.(`[import] imported: ${JSON.stringify(result.counts)}`);
+  return result.counts;
+}

package/server/import/earlier-flow.js

@@ -45,6 +45,12 @@ function parseJson(text) {
   }
 }
 
+// The earlier codebase stored some accounting payloads as the API sent them
+// (`UnitPrice`) and some as its SDK returned them (`unitPrice`).
+function field(object, name) {
+  return object[name] ?? object[name[0].toLowerCase() + name.slice(1)];
+}
+
 function isTrue(value) {
   return value === true || value === 1 || value === '1' || value === 'true';
 }
@@ -82,8 +88,8 @@ function planContacts(rows, skipped) {
       skipped.push({ table: 'contacts', key, reason: 'unreadable_raw_json' });
       continue;
     }
-    const customer = isTrue(raw.IsCustomer);
-    const supplier = isTrue(raw.IsSupplier);
+    const customer = isTrue(field(raw, 'IsCustomer'));
+    const supplier = isTrue(field(raw, 'IsSupplier'));
     // A contact the accounting system has never billed or paid is most often
     // a new customer, so it comes across as one.
     const kinds = supplier && !customer ? ['supplier'] : supplier ? ['customer', 'supplier'] : ['customer'];
@@ -120,12 +126,12 @@ function planPriceItems(rows, skipped, defaulted) {
       skipped.push({ table: 'price_items', key, reason: 'unreadable_sales_details' });
       continue;
     }
-    const unitAmountCents = toCents(sales.UnitPrice);
+    const unitAmountCents = toCents(field(sales, 'UnitPrice'));
     if (unitAmountCents === null) {
       skipped.push({ table: 'price_items', key, reason: 'no_sales_price' });
       continue;
     }
-    const taxType = String(sales.TaxType ?? '').trim().toUpperCase();
+    const taxType = String(field(sales, 'TaxType') ?? '').trim().toUpperCase();
     if (!taxType) defaulted.push({ table: 'price_items', key, reason: 'no_tax_type_used_gst' });
     const taxCode = !taxType || GST_TAX_TYPES.has(taxType) ? 'GST' : 'GST_FREE';
     planned.push({ sourceKey: key, code, name, unitAmountCents, taxCode });

package/server/index.js

@@ -5,6 +5,7 @@ import { seedFirstOwner } from './auth/people.js';
 import { createPersonAuth } from './auth/session.js';
 import { resolveSignInSettings } from './auth/settings.js';
 import { migrateDatabase, openDatabase } from './db/index.js';
+import { importEarlierAtStart, resolveImportAtStart } from './import/at-start.js';
 import { createIntegrations } from './integrations/index.js';
 import { resolveDefaultFamily } from './theme/themes.js';
 import { createUpdateChecker, UPDATE_CHECK_INTERVAL_MS } from './update-check.js';
@@ -34,11 +35,26 @@ try {
   process.exit(1);
 }
 
+let importAtStart;
+try {
+  importAtStart = resolveImportAtStart();
+} catch (error) {
+  console.error(`[import] ${error.message}`);
+  process.exit(1);
+}
+
 // Flow's records live in PostgreSQL. Migrations run before the server accepts
 // requests, so a release never serves an older schema. Then each enabled
 // integration's own tables, and no others (docs/INTEGRATIONS.md).
 const database = openDatabase();
 await migrateDatabase(database.db);
+// A failed import stops the start, so Flow never opens with half its records.
+try {
+  await importEarlierAtStart({ db: database.db, pool: database.pool, settings: importAtStart });
+} catch (error) {
+  console.error(`[import] ${error.message}`);
+  process.exit(1);
+}
 await integrations.start(database.db);
 const integrationsOn = integrations.list().filter((a) => a.enabled).map((a) => a.id);
 console.log(`[integrations] on: ${integrationsOn.join(', ') || 'none'} (FLOW_INTEGRATIONS)`);

package/server/ui/routes.js

@@ -21,6 +21,13 @@ const THEME_FILES = new Set(['themes.js', 'color-contrast.js', 'openclaw-palette
 const UI_FILES = new Set(['flow.css', 'appearance.js', 'clerk-appearance.js', 'app.js', 'flow-mark.svg']);
 const MAX_AGE = '1h';
 
+// Send one named file from dir. Passing dir as root limits the dotfile check
+// to the file name, so an install under a dot directory (such as
+// .flow-deploy/) still serves its assets. A missing file falls through to the
+// app's own 404.
+const sendNamed = (res, next, dir, file) =>
+  res.sendFile(file, { root: dir, maxAge: MAX_AGE }, (err) => err && !res.headersSent && next());
+
 export function createUiAssetsRouter({ rateLimitPerMinute = 600 } = {}) {
   const router = express.Router();
   const css = themeStylesheet();
@@ -33,14 +40,14 @@ export function createUiAssetsRouter({ rateLimitPerMinute = 600 } = {}) {
 
   const sendFrom = (dir, allowed) => (req, res, next) => {
     if (!allowed.has(req.params.file)) return next();
-    res.sendFile(join(dir, req.params.file), { maxAge: MAX_AGE });
+    sendNamed(res, next, dir, req.params.file);
   };
   router.get('/assets/theme/:file', sendFrom(themeDir, THEME_FILES));
   router.get('/assets/ui/:file', sendFrom(here, UI_FILES));
 
   for (const name of FONT_PACKAGES) {
     const dir = dirname(require.resolve(`@fontsource-variable/${name}/LICENSE`));
-    router.get(`/assets/fonts/${name}/index.css`, (_req, res) => res.sendFile(join(dir, 'index.css'), { maxAge: MAX_AGE }));
+    router.get(`/assets/fonts/${name}/index.css`, (_req, res, next) => sendNamed(res, next, dir, 'index.css'));
     router.use(`/assets/fonts/${name}/files`, express.static(join(dir, 'files'), { maxAge: MAX_AGE, index: false, fallthrough: false }));
   }