@arcforgelabs/flow 2026.10.32 → 2026.10.33

package/server/integrations/seazona/migrations/meta/_journal.json

@@ -36,6 +36,20 @@
       "when": 1792453111948,
       "tag": "0004_history_import",
       "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "7",
+      "when": 1792458014317,
+      "tag": "0005_record_reread",
+      "breakpoints": true
+    },
+    {
+      "idx": 6,
+      "version": "7",
+      "when": 1792459854631,
+      "tag": "0006_sweep_findings",
+      "breakpoints": true
     }
   ]
 }

package/server/integrations/seazona/reconcile.js

@@ -0,0 +1,40 @@
+import { readBillableWork } from './feed.js';
+
+// The billable jobs in Flow's copy of Seazona, by day, behind the port's
+// `billableJobs` (the daily reconcile, server/core/reconcile.js). It walks the
+// billing feed from its start, so a job counts here exactly when the feed
+// gives it to the ledger, on the day the ledger files it (its `jobDate`). It
+// reads Flow's copy only; it never calls Seazona.
+//
+// A feed key only moves forward, so an order read again while the walk runs
+// is counted once, and none is missed.
+export async function readBillableJobs(db, { from, to, pageSize = 1000 } = {}) {
+  const jobs = new Map();
+  let cursor = null;
+  for (;;) {
+    const page = await readBillableWork(db, { cursor, limit: pageSize });
+    for (const { ledgerJob: job } of page.items) {
+      // A job with no lines never reaches the ledger (pipeline.js, ingestWork).
+      if (!job.lines.length) {
+        jobs.delete(job.sourceJobRef);
+        continue;
+      }
+      if (!job.jobDate || (from && job.jobDate < from) || (to && job.jobDate > to)) {
+        jobs.delete(job.sourceJobRef);
+        continue;
+      }
+      // Unknown when a line has no amount: its price item fills it.
+      const priced = job.lines.every((l) => Number.isInteger(l.unitAmountCents));
+      jobs.set(job.sourceJobRef, {
+        ref: job.sourceJobRef,
+        day: job.jobDate,
+        subtotalCents: priced ? job.lines.reduce((sum, l) => sum + l.unitAmountCents * l.quantity, 0) : null,
+      });
+    }
+    // A job that stopped being billable, or was deleted, is not counted.
+    for (const { sourceId } of page.withdrawn) jobs.delete(sourceId);
+    if (!page.more) break;
+    cursor = page.cursor;
+  }
+  return { jobs: [...jobs.values()] };
+}

package/server/integrations/seazona/scheduler.js

@@ -6,6 +6,11 @@ import { readLaneStates } from './store.js';
 // the next lane is due, between 15 seconds and 30 minutes. One run at a time.
 // A stopped lane (issue #101) is not due until a person or a release resets it.
 //
+// Issue #143: a run that has not ended after `stuckMs` is stuck (for example a
+// database call that never answers). It is reported once, /health says so,
+// and the scheduler goes on with the next run without it. Lane claims
+// (store.js, claimLane) keep two runs off one lane.
+//
 // Issue #101: each held failure is reported once to the error services
 // (log.error, server/core/monitoring.js): when a lane starts failing or its
 // failure code changes, and when records are quarantined. A repeat of the
@@ -23,12 +28,15 @@ export function createSeazonaScheduler({
   log = console,
   timers = { setTimeout, clearTimeout },
   firstDelayMs = 5_000,
+  stuckMs = settings?.sync?.stuckLaneMs ?? 20 * 60_000,
 } = {}) {
   let timer = null;
   let running = null;
   let stopped = false;
   let nextRunAt = null;
   let last = null;
+  // When the run that is stuck now started, or null.
+  let stuckSince = null;
   // lane -> the failure code last reported, while that lane keeps failing.
   const reported = new Map();
   const quarantineAt = settings?.sync?.orderQuarantineFailures ?? null;
@@ -45,6 +53,8 @@ export function createSeazonaScheduler({
         reported.delete(lane);
         const count = newlyQuarantined(outcome);
         if (count) log.error?.(`[seazona] ${lane}: records quarantined after repeated failed reads`);
+        // Issue #126: what the sweep found, once per run that found it.
+        for (const line of outcome.reports ?? []) log.error?.(`[seazona] ${lane}: ${line}`);
         continue;
       }
       if (reported.get(lane) === outcome.code) {
@@ -78,8 +88,9 @@ export function createSeazonaScheduler({
   async function tick() {
     timer = null;
     if (stopped) return;
-    running = (async () => {
-      const startedAt = new Date(Number(now())).toISOString();
+    const startedAt = new Date(Number(now())).toISOString();
+    const turn = { abandoned: false };
+    const thisRun = (async () => {
       try {
         const result = await run();
         const failed = Object.entries(result.results).filter(([, lane]) => !lane.ok);
@@ -90,7 +101,24 @@ export function createSeazonaScheduler({
         log.error?.(`[seazona] sync run failed: ${last.error}`);
       }
     })();
-    await running;
+    running = thisRun;
+    const watchdog = timers.setTimeout(() => {
+      if (stopped || turn.abandoned) return;
+      turn.abandoned = true;
+      stuckSince = startedAt;
+      running = null;
+      log.error?.(`[seazona] a sync run has not ended after ${Math.round(stuckMs / 60_000)} minutes; the next run starts without it`);
+      schedule(MIN_DELAY_MS);
+    }, stuckMs);
+    watchdog?.unref?.();
+    await thisRun;
+    timers.clearTimeout(watchdog);
+    // A stuck run that ends late leaves the schedule to the run after it.
+    if (turn.abandoned) {
+      if (stuckSince === startedAt) stuckSince = null;
+      return;
+    }
+    stuckSince = null;
     running = null;
     schedule(await nextDelayMs().catch(() => MIN_DELAY_MS));
   }
@@ -108,7 +136,7 @@ export function createSeazonaScheduler({
       await running;
     },
     status() {
-      return { running: Boolean(running), nextRunAt, lastRun: last };
+      return { running: Boolean(running), nextRunAt, lastRun: last, stuckSince };
     },
   };
 }

package/server/integrations/seazona/schema.js

@@ -170,9 +170,12 @@ export const seazonaOrderReadFailures = pgTable('seazona_order_read_failures', {
 // Issue #101: each failed detail read of an invoice or a client, per record.
 // Orders keep their own table (issue #12). After a few failures in a row the
 // record is quarantined: the lane skips it and its cursor moves on.
+// Issue #142: `detail_id` is the id its detail path needs (an invoice's key
+// is its number), so the lane can read it again when its wait ends.
 export const seazonaRecordReadFailures = pgTable('seazona_record_read_failures', {
   resource: text('resource').notNull(),
   recordKey: text('record_key').notNull(),
+  detailId: text('detail_id'),
   failureCount: integer('failure_count').notNull(),
   lastAttemptAt: at('last_attempt_at').notNull(),
   lastHttpStatus: integer('last_http_status'),
@@ -271,3 +274,21 @@ export const seazonaHistoryJobs = pgTable('seazona_history_jobs', {
   index('seazona_history_jobs_invoice').on(t.invoiceNumber),
   index('seazona_history_jobs_changed').on(t.lastChangedAt),
 ]);
+
+// Issue #126: what the sweep found that the cursor lanes missed, one row per
+// record and kind. `changed` and `created`: an edit or a new record the lanes
+// passed by. `deleted`: an order Seazona answered 404 for. `missing`: an
+// invoice or client that Seazona no longer lists; the row stays, and the
+// mark clears when Seazona lists the record again.
+export const seazonaSweepFindings = pgTable('seazona_sweep_findings', {
+  resource: text('resource').notNull(),
+  recordKey: text('record_key').notNull(),
+  kind: text('kind').notNull(),
+  firstFoundAt: at('first_found_at').notNull(),
+  lastFoundAt: at('last_found_at').notNull(),
+  // Status codes and statuses only. Never patient data.
+  detail: jsonb('detail'),
+}, (t) => [
+  primaryKey({ columns: [t.resource, t.recordKey, t.kind] }),
+  index('seazona_sweep_findings_found').on(t.lastFoundAt),
+]);

package/server/integrations/seazona/settings.js

@@ -102,7 +102,16 @@ export function resolveSeazonaSettings(env = process.env) {
       // The history backfill (docs/SEAZONA.md, "History import"). It calls
       // Seazona only while a person has started one.
       history: { cadenceMs: ms('SEAZONA_HISTORY_CADENCE_MS', 2 * MINUTE, MINUTE, DAY_MS), freshnessMs: 30 * DAY_MS },
+      // Issue #126: the slow re-check sweep (sweep.js). A few calls a run,
+      // inside a small daily share.
+      sweep: { cadenceMs: ms('SEAZONA_SWEEP_CADENCE_MS', 15 * MINUTE, MINUTE, DAY_MS), freshnessMs: 2 * DAY_MS },
     },
+    // Issue #126: calls the sweep makes per run and per UTC day, at most. 0 a
+    // day turns the sweep off. Each kind of record (old orders, clients,
+    // invoices) is checked again at most once a cycle.
+    sweepCallsPerRun: clampInt(env.SEAZONA_SWEEP_CALLS_PER_RUN, 10, 1, 25),
+    sweepCallsPerDay: clampInt(env.SEAZONA_SWEEP_CALLS_PER_DAY, 100, 0, 1000),
+    sweepCycleMs: clampInt(env.SEAZONA_SWEEP_CYCLE_DAYS, 7, 1, 90) * DAY_MS,
     // Calls the history backfill makes per run, at most: a small share of
     // the rate limit, so the live lanes always have room.
     historyCallsPerRun: clampInt(env.SEAZONA_HISTORY_CALLS_PER_RUN, 10, 1, 25),
@@ -128,6 +137,9 @@ export function resolveSeazonaSettings(env = process.env) {
     // Issue #12: after this many failed reads in a row an order waits this long.
     orderQuarantineFailures: clampInt(env.SEAZONA_ORDER_QUARANTINE_FAILURES, 3, 1, 20),
     orderQuarantineMs: ms('SEAZONA_ORDER_QUARANTINE_MS', DAY_MS, 5 * MINUTE, 30 * DAY_MS),
+    // Issue #142: the invoice and client lanes read this many quarantined
+    // records again per run, once their wait has ended.
+    recordRereadsPerRun: clampInt(env.SEAZONA_RECORD_REREADS_PER_RUN, 5, 0, 50),
     // A failed lane waits 1 minute, doubling to at most 6 hours.
     retryBaseMs: MINUTE,
     retryMaxMs: 6 * 60 * MINUTE,

package/server/integrations/seazona/store.js

@@ -1,4 +1,4 @@
-import { and, eq, getTableColumns, gt, inArray, isNotNull, isNull, lt, ne, or, sql } from 'drizzle-orm';
+import { and, asc, eq, getTableColumns, gt, inArray, isNotNull, isNull, lt, lte, ne, or, sql } from 'drizzle-orm';
 import { hashOf } from './map.js';
 import {
   seazonaClients,
@@ -215,11 +215,12 @@ export async function clearRecordFailures(db, resource, keys) {
 
 // Records one failed read, as recordOrderFailure does for orders. Answers
 // { failureCount, quarantinedUntil }.
-export async function recordRecordFailure(db, resource, key, failure, { at, quarantineFailures, quarantineMs }) {
+export async function recordRecordFailure(db, resource, key, failure, { at, quarantineFailures, quarantineMs, detailId = null }) {
   const until = new Date(at.getTime() + quarantineMs);
   const [row] = await db.insert(seazonaRecordReadFailures).values({
     resource,
     recordKey: key,
+    detailId,
     failureCount: 1,
     lastAttemptAt: at,
     lastHttpStatus: failure.httpStatus,
@@ -229,6 +230,7 @@ export async function recordRecordFailure(db, resource, key, failure, { at, quar
     target: [seazonaRecordReadFailures.resource, seazonaRecordReadFailures.recordKey],
     set: {
       failureCount: sql`${seazonaRecordReadFailures.failureCount} + 1`,
+      detailId: sql`coalesce(${detailId}::text, ${seazonaRecordReadFailures.detailId})`,
       lastAttemptAt: at,
       lastHttpStatus: failure.httpStatus,
       lastFailureKind: failure.kind,
@@ -239,6 +241,26 @@ export async function recordRecordFailure(db, resource, key, failure, { at, quar
   return { failureCount: row.failureCount, quarantinedUntil: row.quarantinedUntil?.toISOString() ?? null };
 }
 
+// Issue #142: quarantined records whose wait has ended, oldest wait first,
+// that the lane can read again by id.
+export async function readRecordsDueForReread(db, resource, { at, limit }) {
+  return db.select().from(seazonaRecordReadFailures).where(and(
+    eq(seazonaRecordReadFailures.resource, resource),
+    isNotNull(seazonaRecordReadFailures.detailId),
+    lte(seazonaRecordReadFailures.quarantinedUntil, at),
+  )).orderBy(asc(seazonaRecordReadFailures.quarantinedUntil)).limit(limit);
+}
+
+// Per resource: quarantined records whose wait has ended, so they wait for a
+// re-read.
+export async function readRecordRereadCounts(db, { at }) {
+  const rows = await db.select({ resource: seazonaRecordReadFailures.resource, records: sql`count(*)::int` })
+    .from(seazonaRecordReadFailures)
+    .where(lte(seazonaRecordReadFailures.quarantinedUntil, at))
+    .groupBy(seazonaRecordReadFailures.resource);
+  return Object.fromEntries(rows.map((row) => [row.resource, row.records]));
+}
+
 export async function readQuarantinedRecordCounts(db, { at }) {
   const rows = await db.select({ resource: seazonaRecordReadFailures.resource, records: sql`count(*)::int` })
     .from(seazonaRecordReadFailures)