@arcforgelabs/flow 2026.10.32 → 2026.10.34

package/server/integrations/seazona/migrations/meta/_journal.json

@@ -36,6 +36,27 @@
       "when": 1792453111948,
       "tag": "0004_history_import",
       "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1792458014317,
+      "tag": "0005_record_reread",
+      "breakpoints": true
+    },
+    {
+      "idx": 6,
+      "version": "7",
+      "when": 1792459854631,
+      "tag": "0006_sweep_findings",
+      "breakpoints": true
+    },
+    {
+      "idx": 7,
+      "version": "7",
+      "when": 1792472936253,
+      "tag": "0007_stage_moves",
+      "breakpoints": true
     }
   ]
 }

package/server/integrations/seazona/reconcile.js

@@ -0,0 +1,40 @@
+import { readBillableWork } from './feed.js';
+
+// The billable jobs in Flow's copy of Seazona, by day, behind the port's
+// `billableJobs` (the daily reconcile, server/core/reconcile.js). It walks the
+// billing feed from its start, so a job counts here exactly when the feed
+// gives it to the ledger, on the day the ledger files it (its `jobDate`). It
+// reads Flow's copy only; it never calls Seazona.
+//
+// A feed key only moves forward, so an order read again while the walk runs
+// is counted once, and none is missed.
+export async function readBillableJobs(db, { from, to, pageSize = 1000 } = {}) {
+  const jobs = new Map();
+  let cursor = null;
+  for (;;) {
+    const page = await readBillableWork(db, { cursor, limit: pageSize });
+    for (const { ledgerJob: job } of page.items) {
+      // A job with no lines never reaches the ledger (pipeline.js, ingestWork).
+      if (!job.lines.length) {
+        jobs.delete(job.sourceJobRef);
+        continue;
+      }
+      if (!job.jobDate || (from && job.jobDate < from) || (to && job.jobDate > to)) {
+        jobs.delete(job.sourceJobRef);
+        continue;
+      }
+      // Unknown when a line has no amount: its price item fills it.
+      const priced = job.lines.every((l) => Number.isInteger(l.unitAmountCents));
+      jobs.set(job.sourceJobRef, {
+        ref: job.sourceJobRef,
+        day: job.jobDate,
+        subtotalCents: priced ? job.lines.reduce((sum, l) => sum + l.unitAmountCents * l.quantity, 0) : null,
+      });
+    }
+    // A job that stopped being billable, or was deleted, is not counted.
+    for (const { sourceId } of page.withdrawn) jobs.delete(sourceId);
+    if (!page.more) break;
+    cursor = page.cursor;
+  }
+  return { jobs: [...jobs.values()] };
+}

package/server/integrations/seazona/scheduler.js

@@ -6,6 +6,11 @@ import { readLaneStates } from './store.js';
 // the next lane is due, between 15 seconds and 30 minutes. One run at a time.
 // A stopped lane (issue #101) is not due until a person or a release resets it.
 //
+// Issue #143: a run that has not ended after `stuckMs` is stuck (for example a
+// database call that never answers). It is reported once, /health says so,
+// and the scheduler goes on with the next run without it. Lane claims
+// (store.js, claimLane) keep two runs off one lane.
+//
 // Issue #101: each held failure is reported once to the error services
 // (log.error, server/core/monitoring.js): when a lane starts failing or its
 // failure code changes, and when records are quarantined. A repeat of the
@@ -23,12 +28,15 @@ export function createSeazonaScheduler({
   log = console,
   timers = { setTimeout, clearTimeout },
   firstDelayMs = 5_000,
+  stuckMs = settings?.sync?.stuckLaneMs ?? 20 * 60_000,
 } = {}) {
   let timer = null;
   let running = null;
   let stopped = false;
   let nextRunAt = null;
   let last = null;
+  // When the run that is stuck now started, or null.
+  let stuckSince = null;
   // lane -> the failure code last reported, while that lane keeps failing.
   const reported = new Map();
   const quarantineAt = settings?.sync?.orderQuarantineFailures ?? null;
@@ -45,6 +53,8 @@ export function createSeazonaScheduler({
         reported.delete(lane);
         const count = newlyQuarantined(outcome);
         if (count) log.error?.(`[seazona] ${lane}: records quarantined after repeated failed reads`);
+        // Issue #126: what the sweep found, once per run that found it.
+        for (const line of outcome.reports ?? []) log.error?.(`[seazona] ${lane}: ${line}`);
         continue;
       }
       if (reported.get(lane) === outcome.code) {
@@ -78,8 +88,9 @@ export function createSeazonaScheduler({
   async function tick() {
     timer = null;
     if (stopped) return;
-    running = (async () => {
-      const startedAt = new Date(Number(now())).toISOString();
+    const startedAt = new Date(Number(now())).toISOString();
+    const turn = { abandoned: false };
+    const thisRun = (async () => {
       try {
         const result = await run();
         const failed = Object.entries(result.results).filter(([, lane]) => !lane.ok);
@@ -90,7 +101,24 @@ export function createSeazonaScheduler({
         log.error?.(`[seazona] sync run failed: ${last.error}`);
       }
     })();
-    await running;
+    running = thisRun;
+    const watchdog = timers.setTimeout(() => {
+      if (stopped || turn.abandoned) return;
+      turn.abandoned = true;
+      stuckSince = startedAt;
+      running = null;
+      log.error?.(`[seazona] a sync run has not ended after ${Math.round(stuckMs / 60_000)} minutes; the next run starts without it`);
+      schedule(MIN_DELAY_MS);
+    }, stuckMs);
+    watchdog?.unref?.();
+    await thisRun;
+    timers.clearTimeout(watchdog);
+    // A stuck run that ends late leaves the schedule to the run after it.
+    if (turn.abandoned) {
+      if (stuckSince === startedAt) stuckSince = null;
+      return;
+    }
+    stuckSince = null;
     running = null;
     schedule(await nextDelayMs().catch(() => MIN_DELAY_MS));
   }
@@ -108,7 +136,7 @@ export function createSeazonaScheduler({
       await running;
     },
     status() {
-      return { running: Boolean(running), nextRunAt, lastRun: last };
+      return { running: Boolean(running), nextRunAt, lastRun: last, stuckSince };
     },
   };
 }

package/server/integrations/seazona/schema.js

@@ -170,9 +170,12 @@ export const seazonaOrderReadFailures = pgTable('seazona_order_read_failures', {
 // Issue #101: each failed detail read of an invoice or a client, per record.
 // Orders keep their own table (issue #12). After a few failures in a row the
 // record is quarantined: the lane skips it and its cursor moves on.
+// Issue #142: `detail_id` is the id its detail path needs (an invoice's key
+// is its number), so the lane can read it again when its wait ends.
 export const seazonaRecordReadFailures = pgTable('seazona_record_read_failures', {
   resource: text('resource').notNull(),
   recordKey: text('record_key').notNull(),
+  detailId: text('detail_id'),
   failureCount: integer('failure_count').notNull(),
   lastAttemptAt: at('last_attempt_at').notNull(),
   lastHttpStatus: integer('last_http_status'),
@@ -271,3 +274,47 @@ export const seazonaHistoryJobs = pgTable('seazona_history_jobs', {
   index('seazona_history_jobs_invoice').on(t.invoiceNumber),
   index('seazona_history_jobs_changed').on(t.lastChangedAt),
 ]);
+
+// Issue #126: what the sweep found that the cursor lanes missed, one row per
+// record and kind. `changed` and `created`: an edit or a new record the lanes
+// passed by. `deleted`: an order Seazona answered 404 for. `missing`: an
+// invoice or client that Seazona no longer lists; the row stays, and the
+// mark clears when Seazona lists the record again.
+export const seazonaSweepFindings = pgTable('seazona_sweep_findings', {
+  resource: text('resource').notNull(),
+  recordKey: text('record_key').notNull(),
+  kind: text('kind').notNull(),
+  firstFoundAt: at('first_found_at').notNull(),
+  lastFoundAt: at('last_found_at').notNull(),
+  // Status codes and statuses only. Never patient data.
+  detail: jsonb('detail'),
+}, (t) => [
+  primaryKey({ columns: [t.resource, t.recordKey, t.kind] }),
+  index('seazona_sweep_findings_found').on(t.lastFoundAt),
+]);
+
+// A person's stage move on the Jobs board that Seazona must make, because
+// Seazona's API cannot change an order's status (docs/SEAZONA.md, "Stage
+// moves"). One row per order: the latest move. `state` is waiting until the
+// sync reads the new status (done), a day passes (expired), or the person
+// takes it back (cancelled). Statuses only; never patient data.
+export const seazonaStageMoves = pgTable('seazona_stage_moves', {
+  orderId: text('order_id').primaryKey(),
+  fromStatus: text('from_status'),
+  toStatus: text('to_status').notNull(),
+  requestedBy: text('requested_by'),
+  requestedAt: at('requested_at').notNull(),
+  expiresAt: at('expires_at').notNull(),
+  state: text('state').notNull().default('waiting'),
+  settledAt: at('settled_at'),
+}, (t) => [
+  index('seazona_stage_moves_state').on(t.state, t.expiresAt),
+]);
+
+// Each person's own Seazona choices. `ask_before_stage_moves`: show the
+// "Move in Seazona" dialog before a stage move on the Jobs board.
+export const seazonaPersonSettings = pgTable('seazona_person_settings', {
+  personId: text('person_id').primaryKey(),
+  askBeforeStageMoves: boolean('ask_before_stage_moves').notNull().default(true),
+  updatedAt: at('updated_at').notNull(),
+});

package/server/integrations/seazona/settings.js

@@ -55,6 +55,16 @@ export function normaliseBaseUrl(value) {
   return `${url.origin}${path}/v1`;
 }
 
+function orderUrlTemplate(value) {
+  const raw = text(value);
+  if (!raw.includes('{ref}')) return null;
+  try {
+    return new URL(raw.replaceAll('{ref}', 'x')).protocol === 'https:' ? raw : null;
+  } catch {
+    return null;
+  }
+}
+
 // Returns the resolved settings, or throws SeazonaSettingsError. `mode` is
 // null when SEAZONA_MODE is unset: the adapter is then not configured.
 export function resolveSeazonaSettings(env = process.env) {
@@ -102,7 +112,16 @@ export function resolveSeazonaSettings(env = process.env) {
       // The history backfill (docs/SEAZONA.md, "History import"). It calls
       // Seazona only while a person has started one.
       history: { cadenceMs: ms('SEAZONA_HISTORY_CADENCE_MS', 2 * MINUTE, MINUTE, DAY_MS), freshnessMs: 30 * DAY_MS },
+      // Issue #126: the slow re-check sweep (sweep.js). A few calls a run,
+      // inside a small daily share.
+      sweep: { cadenceMs: ms('SEAZONA_SWEEP_CADENCE_MS', 15 * MINUTE, MINUTE, DAY_MS), freshnessMs: 2 * DAY_MS },
     },
+    // Issue #126: calls the sweep makes per run and per UTC day, at most. 0 a
+    // day turns the sweep off. Each kind of record (old orders, clients,
+    // invoices) is checked again at most once a cycle.
+    sweepCallsPerRun: clampInt(env.SEAZONA_SWEEP_CALLS_PER_RUN, 10, 1, 25),
+    sweepCallsPerDay: clampInt(env.SEAZONA_SWEEP_CALLS_PER_DAY, 100, 0, 1000),
+    sweepCycleMs: clampInt(env.SEAZONA_SWEEP_CYCLE_DAYS, 7, 1, 90) * DAY_MS,
     // Calls the history backfill makes per run, at most: a small share of
     // the rate limit, so the live lanes always have room.
     historyCallsPerRun: clampInt(env.SEAZONA_HISTORY_CALLS_PER_RUN, 10, 1, 25),
@@ -128,6 +147,9 @@ export function resolveSeazonaSettings(env = process.env) {
     // Issue #12: after this many failed reads in a row an order waits this long.
     orderQuarantineFailures: clampInt(env.SEAZONA_ORDER_QUARANTINE_FAILURES, 3, 1, 20),
     orderQuarantineMs: ms('SEAZONA_ORDER_QUARANTINE_MS', DAY_MS, 5 * MINUTE, 30 * DAY_MS),
+    // Issue #142: the invoice and client lanes read this many quarantined
+    // records again per run, once their wait has ended.
+    recordRereadsPerRun: clampInt(env.SEAZONA_RECORD_REREADS_PER_RUN, 5, 0, 50),
     // A failed lane waits 1 minute, doubling to at most 6 hours.
     retryBaseMs: MINUTE,
     retryMaxMs: 6 * 60 * MINUTE,
@@ -149,6 +171,10 @@ export function resolveSeazonaSettings(env = process.env) {
     rate,
     // The scheduler starts only in live mode (issue #10).
     scheduled: mode === 'live',
+    // The address of one order in Seazona's own screens, with {ref} for the
+    // order number, for "Open in Seazona" on a stage move. Optional; one
+    // without {ref} is ignored.
+    orderUrlTemplate: orderUrlTemplate(env.SEAZONA_ORDER_URL),
   };
 
   if (mode === 'live' || mode === 'sandbox') {

package/server/integrations/seazona/stage-moves.js

@@ -0,0 +1,135 @@
+import { and, eq, gte, inArray, lt, sql } from 'drizzle-orm';
+import { FlowError } from '../../core/errors.js';
+import { orderStatusLabel } from './map.js';
+import { seazonaOrders, seazonaPersonSettings, seazonaStageMoves } from './schema.js';
+
+// Stage moves on the Jobs board (docs/SEAZONA.md, "Stage moves"). Seazona's
+// API cannot change an order's status, so Flow never tries: a move becomes a
+// hand-off. Flow records the move, the person makes it in Seazona, and the
+// next sync that reads the new status settles it. Until then the card shows
+// "Waiting for Seazona" in the new lane. After a day with no change the move
+// expires, the card goes back to Seazona's stage, and a quiet alert says so.
+// Every function here reads and writes Flow's database only.
+
+export const HAND_OFF_MS = 24 * 60 * 60_000;
+// How long an expired move stays in the alert.
+const EXPIRED_ALERT_MS = 7 * 24 * 60 * 60_000;
+const MAX_STATUS = 80;
+
+// The address that opens one order in Seazona, from SEAZONA_ORDER_URL with
+// its {ref} placeholder, or null when the installation has not set one.
+export function orderUrl(template, ref) {
+  if (!template || !template.includes('{ref}')) return null;
+  return template.replaceAll('{ref}', encodeURIComponent(ref));
+}
+
+// Record a person's move of one order to `toStatus`, and answer the hand-off
+// the board shows. A second move of the same order replaces the first.
+export async function requestStageMove(db, { orderId, toStatus, personId = null, orderUrlTemplate = null, now = Date.now() }) {
+  const to = typeof toStatus === 'string' ? toStatus.trim() : '';
+  if (!to || to.length > MAX_STATUS) throw new FlowError('invalid', 'toStatus must be a Seazona status');
+  const [order] = await db.select().from(seazonaOrders).where(eq(seazonaOrders.id, String(orderId ?? '')));
+  if (!order || order.deletedInSourceAt) throw new FlowError('not_found', 'Flow has no Seazona order with that number');
+  const known = await db.selectDistinct({ status: seazonaOrders.status }).from(seazonaOrders)
+    .where(eq(sql`lower(trim(${seazonaOrders.status}))`, to.toLowerCase()));
+  if (!known.length) throw new FlowError('invalid', `Seazona has no status named ${to}`);
+  const from = orderStatusLabel(order) || null;
+  if (from && from.toLowerCase() === to.toLowerCase()) throw new FlowError('conflict', `The order is already ${from} in Seazona`);
+  const at = new Date(Number(now));
+  const row = {
+    orderId: order.id, fromStatus: from, toStatus: known[0].status.trim(), requestedBy: personId,
+    requestedAt: at, expiresAt: new Date(at.getTime() + HAND_OFF_MS), state: 'waiting', settledAt: null,
+  };
+  await db.insert(seazonaStageMoves).values(row).onConflictDoUpdate({ target: seazonaStageMoves.orderId, set: row });
+  return handOff(row, { orderUrlTemplate });
+}
+
+// Take a waiting move back: the card returns to Seazona's stage.
+export async function cancelStageMove(db, { orderId, now = Date.now() }) {
+  const rows = await db.update(seazonaStageMoves).set({ state: 'cancelled', settledAt: new Date(Number(now)) })
+    .where(and(eq(seazonaStageMoves.orderId, String(orderId ?? '')), eq(seazonaStageMoves.state, 'waiting'))).returning();
+  return { cancelled: rows.length > 0 };
+}
+
+// Settle the waiting moves against Flow's copy of the orders: done when the
+// order has the new status, expired when a day has passed. An order that
+// moved to some other status settles as done too: Seazona has the last word.
+export async function settleStageMoves(db, { now = Date.now() } = {}) {
+  const at = new Date(Number(now));
+  const waiting = await db.select({ move: seazonaStageMoves, order: seazonaOrders }).from(seazonaStageMoves)
+    .leftJoin(seazonaOrders, eq(seazonaOrders.id, seazonaStageMoves.orderId))
+    .where(eq(seazonaStageMoves.state, 'waiting'));
+  const done = [];
+  const expired = [];
+  for (const { move, order } of waiting) {
+    const status = orderStatusLabel(order) || null;
+    if (status && status !== move.fromStatus) done.push(move.orderId);
+    else if (move.expiresAt <= at) expired.push(move.orderId);
+  }
+  if (done.length) {
+    await db.update(seazonaStageMoves).set({ state: 'done', settledAt: at })
+      .where(and(inArray(seazonaStageMoves.orderId, done), eq(seazonaStageMoves.state, 'waiting')));
+  }
+  if (expired.length) {
+    await db.update(seazonaStageMoves).set({ state: 'expired', settledAt: at })
+      .where(and(inArray(seazonaStageMoves.orderId, expired), eq(seazonaStageMoves.state, 'waiting')));
+  }
+  return { done: done.length, expired: expired.length };
+}
+
+// The waiting moves, by order id, for the board's cards.
+export async function waitingStageMoves(db) {
+  const rows = await db.select().from(seazonaStageMoves).where(eq(seazonaStageMoves.state, 'waiting'));
+  return new Map(rows.map((r) => [r.orderId, r]));
+}
+
+// The moves that expired in the last week, for the quiet alert.
+export async function expiredStageMoves(db, { now = Date.now() } = {}) {
+  return db.select().from(seazonaStageMoves).where(and(
+    eq(seazonaStageMoves.state, 'expired'),
+    gte(seazonaStageMoves.settledAt, new Date(Number(now) - EXPIRED_ALERT_MS)),
+    lt(seazonaStageMoves.settledAt, new Date(Number(now) + 1)),
+  ));
+}
+
+// What the board shows for a waiting move.
+export function handOff(row, { orderUrlTemplate = null } = {}) {
+  return {
+    kind: 'hand_off',
+    system: 'seazona',
+    sourceJobRef: row.orderId,
+    caseRef: row.orderId,
+    from: row.fromStatus,
+    to: row.toStatus,
+    state: row.state,
+    requestedAt: row.requestedAt.toISOString(),
+    expiresAt: row.expiresAt.toISOString(),
+    openUrl: orderUrl(orderUrlTemplate, row.orderId),
+  };
+}
+
+// Each person's own Seazona choices, with their defaults.
+export const PERSON_SETTINGS = Object.freeze([
+  {
+    key: 'askBeforeStageMoves',
+    label: 'Ask before stage moves',
+    hint: 'On the Jobs board, show what to do in Seazona before a card moves to a Seazona stage.',
+    kind: 'onoff',
+    default: true,
+  },
+]);
+
+export async function readPersonSettings(db, personId) {
+  const [row] = await db.select().from(seazonaPersonSettings).where(eq(seazonaPersonSettings.personId, String(personId)));
+  return PERSON_SETTINGS.map((s) => ({ ...s, value: row ? Boolean(row[s.key]) : s.default }));
+}
+
+export async function writePersonSetting(db, personId, { key, value }, { now = Date.now() } = {}) {
+  const setting = PERSON_SETTINGS.find((s) => s.key === key);
+  if (!setting) throw new FlowError('invalid', 'no such setting');
+  if (typeof value !== 'boolean') throw new FlowError('invalid', `${setting.label} must be on or off`);
+  const at = new Date(Number(now));
+  await db.insert(seazonaPersonSettings).values({ personId: String(personId), [key]: value, updatedAt: at })
+    .onConflictDoUpdate({ target: seazonaPersonSettings.personId, set: { [key]: value, updatedAt: at } });
+  return readPersonSettings(db, personId);
+}

package/server/integrations/seazona/store.js

@@ -1,4 +1,4 @@
-import { and, eq, getTableColumns, gt, inArray, isNotNull, isNull, lt, ne, or, sql } from 'drizzle-orm';
+import { and, asc, eq, getTableColumns, gt, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from 'drizzle-orm';
 import { hashOf } from './map.js';
 import {
   seazonaClients,
@@ -215,11 +215,12 @@ export async function clearRecordFailures(db, resource, keys) {
 
 // Records one failed read, as recordOrderFailure does for orders. Answers
 // { failureCount, quarantinedUntil }.
-export async function recordRecordFailure(db, resource, key, failure, { at, quarantineFailures, quarantineMs }) {
+export async function recordRecordFailure(db, resource, key, failure, { at, quarantineFailures, quarantineMs, detailId = null }) {
   const until = new Date(at.getTime() + quarantineMs);
   const [row] = await db.insert(seazonaRecordReadFailures).values({
     resource,
     recordKey: key,
+    detailId,
     failureCount: 1,
     lastAttemptAt: at,
     lastHttpStatus: failure.httpStatus,
@@ -229,6 +230,7 @@ export async function recordRecordFailure(db, resource, key, failure, { at, quar
     target: [seazonaRecordReadFailures.resource, seazonaRecordReadFailures.recordKey],
     set: {
       failureCount: sql`${seazonaRecordReadFailures.failureCount} + 1`,
+      detailId: sql`coalesce(${detailId}::text, ${seazonaRecordReadFailures.detailId})`,
       lastAttemptAt: at,
       lastHttpStatus: failure.httpStatus,
       lastFailureKind: failure.kind,
@@ -239,6 +241,26 @@ export async function recordRecordFailure(db, resource, key, failure, { at, quar
   return { failureCount: row.failureCount, quarantinedUntil: row.quarantinedUntil?.toISOString() ?? null };
 }
 
+// Issue #142: quarantined records whose wait has ended, oldest wait first,
+// that the lane can read again by id.
+export async function readRecordsDueForReread(db, resource, { at, limit }) {
+  return db.select().from(seazonaRecordReadFailures).where(and(
+    eq(seazonaRecordReadFailures.resource, resource),
+    isNotNull(seazonaRecordReadFailures.detailId),
+    lte(seazonaRecordReadFailures.quarantinedUntil, at),
+  )).orderBy(asc(seazonaRecordReadFailures.quarantinedUntil)).limit(limit);
+}
+
+// Per resource: quarantined records whose wait has ended, so they wait for a
+// re-read.
+export async function readRecordRereadCounts(db, { at }) {
+  const rows = await db.select({ resource: seazonaRecordReadFailures.resource, records: sql`count(*)::int` })
+    .from(seazonaRecordReadFailures)
+    .where(lte(seazonaRecordReadFailures.quarantinedUntil, at))
+    .groupBy(seazonaRecordReadFailures.resource);
+  return Object.fromEntries(rows.map((row) => [row.resource, row.records]));
+}
+
 export async function readQuarantinedRecordCounts(db, { at }) {
   const rows = await db.select({ resource: seazonaRecordReadFailures.resource, records: sql`count(*)::int` })
     .from(seazonaRecordReadFailures)