@arcforgelabs/flow 2026.10.27 → 2026.10.30

package/server/integrations/seazona/lanes.js

@@ -13,12 +13,16 @@ import { seazonaOrders } from './schema.js';
 import {
   claimLane,
   clearOrderFailure,
+  clearRecordFailures,
   finishLane,
   markOrderDeleted,
   readKnown,
   readLaneStates,
   readQuarantinedOrders,
+  readQuarantinedRecordCounts,
+  readQuarantinedRecords,
   recordOrderFailure,
+  recordRecordFailure,
   selectActiveOrders,
   touchSynced,
   upsertRecords,
@@ -39,6 +43,13 @@ import {
 // Seazona, an empty answer is a failure, each failure is recorded per order
 // and quarantines it after a few in a row, and a lane fails only when most of
 // its reads fail. Errors and /health name each failing order and its status.
+//
+// Issue #101 holds for every lane: invoices and clients quarantine a record
+// after a few failed reads too, so one bad record never holds a cursor. A
+// list call that Seazona refuses (400, 401, 403, 404) means the path or the
+// API client is wrong, and no wait fixes it: the lane stops and runs again
+// only after a person saves the settings or Flow restarts on a new release.
+// Calls the governor stopped (SeazonaCallsStopped) wait out its pause.
 
 export const LANES = Object.freeze(['invoices', 'orders_recent', 'orders_active', 'clients', 'products', 'users']);
 
@@ -73,7 +84,23 @@ function listOrFail(list, what, sync) {
 // A failure that is about Seazona or the rate limit, not one record: the
 // whole lane stops and waits.
 function isLaneLevel(error) {
-  return error?.statusCode === 429 || error?.kind === 'governor_timeout';
+  return error?.statusCode === 429 || error?.statusCode === 401 || error?.statusCode === 403
+    || error?.kind === 'governor_timeout' || error?.kind === 'calls_stopped';
+}
+
+// Answers how a lane that threw should wait: { status, heldReason, delayMs }.
+// A null delay means the lane is stopped until a person or a release.
+const REJECTED = new Set([400, 401, 403, 404]);
+function laneOutcome(error, state, sync) {
+  if (error?.kind === 'calls_stopped') {
+    if (error.reason === 'halted') return { status: 'stopped', heldReason: error.code, delayMs: null };
+    return { status: 'failed', heldReason: null, delayMs: Math.max(sync.retryBaseMs, error.retryAfterMs ?? 0) };
+  }
+  if (error?.name === 'SeazonaApiError' && REJECTED.has(error.statusCode)) {
+    const auth = error.statusCode === 401 || error.statusCode === 403;
+    return { status: 'stopped', heldReason: auth ? 'seazona_api_halted' : 'seazona_request_rejected', delayMs: null };
+  }
+  return { status: error?.held ? 'held' : 'failed', heldReason: error?.held ? error.code : null, delayMs: failureDelayMs(error, state, sync) };
 }
 
 function describeReadFailure(error) {
@@ -156,12 +183,16 @@ function addSummary(total, summary) {
 // Invoices and clients: a list since a cursor, then a detail read for each
 // row whose list fields changed.
 async function syncListWithDetail(ctx, { mirror, what, items, keyOf, idOf, signatureOf, detailPath, map }) {
-  const { db, client, at } = ctx;
+  const { db, client, sync, at } = ctx;
   const keyed = items.map((item) => ({ item, key: keyOf(item) })).filter((entry) => entry.key);
   const known = await readKnown(db, mirror, keyed.map((entry) => entry.key));
+  const quarantined = await readQuarantinedRecords(db, mirror, keyed.map((entry) => entry.key), { at });
   const summary = { created: 0, changed: 0, unchanged: 0 };
   const unchanged = [];
+  const read = [];
   const failures = [];
+  const skipped = [];
+  let newlyQuarantined = 0;
   let detailReads = 0;
   for (const { item, key } of keyed) {
     const signature = signatureOf(item);
@@ -169,6 +200,11 @@ async function syncListWithDetail(ctx, { mirror, what, items, keyOf, idOf, signa
       unchanged.push(key);
       continue;
     }
+    // Issue #101: a quarantined record costs no call until its wait ends.
+    if (quarantined.has(key)) {
+      skipped.push(key);
+      continue;
+    }
     const id = idOf(item);
     let detail = null;
     if (id) {
@@ -178,16 +214,29 @@ async function syncListWithDetail(ctx, { mirror, what, items, keyOf, idOf, signa
         if (!detail || typeof detail !== 'object' || Array.isArray(detail)) throw emptyResponse(`${what} ${key}`);
       } catch (error) {
         if (isLaneLevel(error)) throw error;
-        failures.push({ id: key, ...describeReadFailure(error) });
+        const failure = describeReadFailure(error);
+        const recorded = await recordRecordFailure(db, mirror, key, failure, {
+          at,
+          quarantineFailures: sync.orderQuarantineFailures,
+          quarantineMs: sync.orderQuarantineMs,
+        });
+        if (recorded.quarantinedUntil) newlyQuarantined += 1;
+        failures.push({ id: key, ...failure, ...recorded });
         continue;
       }
     }
+    read.push(key);
     addSummary(summary, await upsertRecords(db, mirror, [map(item, detail)], { at, listSignatures: new Map([[key, signature]]) }));
   }
   await touchSynced(db, mirror, unchanged, { at });
+  await clearRecordFailures(db, mirror, read);
   summary.unchanged += unchanged.length;
-  const failureDetails = judgeReads(`${what} reads`, detailReads, failures);
-  return { summary, detailReads, listCount: items.length, failureDetails, anyFailed: failures.length > 0 };
+  const extra = skipped.length ? { quarantinedIds: skipped.slice(0, MAX_REPORTED) } : {};
+  const failureDetails = judgeReads(`${what} reads`, detailReads, failures, extra);
+  // A quarantined record no longer holds the cursor; a record that may still
+  // be read again does.
+  const anyFailed = failures.some((failure) => !failure.quarantinedUntil);
+  return { summary, detailReads, listCount: items.length, failureDetails, anyFailed, newlyQuarantined, quarantined: skipped.length };
 }
 
 function cursorSince(ctx, fallbackMs) {
@@ -309,8 +358,11 @@ const laneRuns = {
   async clients(ctx) {
     const { client, sync, at } = ctx;
     // The first run reads the whole list; later runs read changes only.
+    // Seazona answers the whole list only to an empty `?lastModified=`; a
+    // bare path is a 404.
     const since = ctx.state?.cursorTime ? cursorSince(ctx, 0) : '';
-    const items = listOrFail(await client.get('clients/', since ? { lastModified: since } : {}), 'client list', sync);
+    const reply = since ? await client.get('clients/', { lastModified: since }) : await client.get('clients/?lastModified=');
+    const items = listOrFail(reply, 'client list', sync);
     const result = await syncListWithDetail(ctx, {
       mirror: 'clients',
       what: 'client',
@@ -355,6 +407,7 @@ export async function runSeazonaSync(db, client, { settings, now = Date.now, onl
   for (const lane of wanted) {
     const state = states.get(lane) ?? null;
     const startMs = Number(now());
+    if (!force && !only.length && state?.status === 'stopped') continue;
     if (!force && !only.length && state?.nextRunAt && state.nextRunAt.getTime() > startMs) continue;
     const at = new Date(startMs);
     if (!(await claimLane(db, lane, { at, stuckMs: sync.stuckLaneMs }))) {
@@ -379,18 +432,19 @@ export async function runSeazonaSync(db, client, { settings, now = Date.now, onl
       const { cursor, cursorTime, anyFailed, ...reported } = result;
       results[lane] = { ok: true, ...reported };
     } catch (error) {
-      const delayMs = failureDelayMs(error, state, sync);
+      const { status, heldReason, delayMs } = laneOutcome(error, state, sync);
       const message = text(error?.message) || 'Seazona lane failed';
+      const code = heldReason ?? error?.code ?? 'seazona_lane_failed';
       await finishLane(db, lane, {
-        status: error?.held ? 'held' : 'failed',
+        status,
         lastAttemptAt: at,
-        nextRunAt: new Date(Number(now()) + delayMs),
+        nextRunAt: delayMs === null ? null : new Date(Number(now()) + delayMs),
         failureCount: (state?.failureCount ?? 0) + 1,
         lastError: message,
-        failureDetails: error?.details ?? null,
-        heldReason: error?.held ? error.code : null,
+        failureDetails: error?.details ?? (Number.isInteger(error?.statusCode) ? { httpStatus: error.statusCode, path: error.path ?? null } : null),
+        heldReason,
       });
-      results[lane] = { ok: false, error: message, code: error?.code ?? 'seazona_lane_failed', details: error?.details ?? null, retryAfterMs: delayMs };
+      results[lane] = { ok: false, status, error: message, code, details: error?.details ?? null, retryAfterMs: delayMs };
     }
   }
   return { ok: Object.values(results).every((result) => result.ok), results };
@@ -404,6 +458,7 @@ export async function readSeazonaSyncHealth(db, settings, { now = Date.now } = {
   const states = await readLaneStates(db);
   const quarantinedOrders = await readQuarantinedOrders(db, { at: new Date(nowMs), limit: MAX_REPORTED });
   const [{ deleted }] = await db.select({ deleted: count() }).from(seazonaOrders).where(isNotNull(seazonaOrders.deletedInSourceAt));
+  const quarantinedRecords = await readQuarantinedRecordCounts(db, { at: new Date(nowMs) });
   const lanes = LANES.map((lane) => {
     const state = states.get(lane);
     const { freshnessMs } = settings.sync.lanes[lane];
@@ -421,10 +476,11 @@ export async function readSeazonaSyncHealth(db, settings, { now = Date.now } = {
       heldReason: state?.heldReason ?? null,
       failureDetails: state?.failureDetails ?? null,
       ...(lane === 'orders_active' ? { quarantinedOrders } : {}),
+      ...(quarantinedRecords[lane] ? { quarantinedRecords: quarantinedRecords[lane] } : {}),
     };
   });
   return {
-    ok: lanes.every((lane) => lane.status !== 'failed' && lane.status !== 'held'),
+    ok: lanes.every((lane) => !['failed', 'held', 'stopped'].includes(lane.status)),
     lanes,
     deletedOrders: Number(deleted),
   };

package/server/integrations/seazona/migrations/0000_api_governor.sql

@@ -1,11 +1,11 @@
-CREATE TABLE "seazona_api_governor" (
+CREATE TABLE IF NOT EXISTS "seazona_api_governor" (
 	"client_key" text PRIMARY KEY NOT NULL,
 	"next_eligible_at" timestamp with time zone DEFAULT 'epoch' NOT NULL,
 	"cooldown_until" timestamp with time zone DEFAULT 'epoch' NOT NULL,
 	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
 );
 --> statement-breakpoint
-CREATE TABLE "seazona_api_requests" (
+CREATE TABLE IF NOT EXISTS "seazona_api_requests" (
 	"id" bigserial PRIMARY KEY NOT NULL,
 	"client_key" text NOT NULL,
 	"workflow" text NOT NULL,
@@ -17,4 +17,4 @@ CREATE TABLE "seazona_api_requests" (
 	"is_retry" boolean DEFAULT false NOT NULL
 );
 --> statement-breakpoint
-CREATE INDEX "seazona_api_requests_window" ON "seazona_api_requests" USING btree ("client_key","admitted_at");
+CREATE INDEX IF NOT EXISTS "seazona_api_requests_window" ON "seazona_api_requests" USING btree ("client_key","admitted_at");

package/server/integrations/seazona/migrations/0001_mirror.sql

@@ -1,4 +1,4 @@
-CREATE TABLE "seazona_clients" (
+CREATE TABLE IF NOT EXISTS "seazona_clients" (
 	"id" text PRIMARY KEY NOT NULL,
 	"account_number" text,
 	"company" text,
@@ -17,7 +17,7 @@ CREATE TABLE "seazona_clients" (
 	"last_changed_at" timestamp with time zone NOT NULL
 );
 --> statement-breakpoint
-CREATE TABLE "seazona_invoices" (
+CREATE TABLE IF NOT EXISTS "seazona_invoices" (
 	"invoice_number" text PRIMARY KEY NOT NULL,
 	"seazona_id" text,
 	"client_id" text,
@@ -41,7 +41,7 @@ CREATE TABLE "seazona_invoices" (
 	"last_changed_at" timestamp with time zone NOT NULL
 );
 --> statement-breakpoint
-CREATE TABLE "seazona_lane_state" (
+CREATE TABLE IF NOT EXISTS "seazona_lane_state" (
 	"lane" text PRIMARY KEY NOT NULL,
 	"status" text DEFAULT 'idle' NOT NULL,
 	"cursor_time" timestamp with time zone,
@@ -56,7 +56,7 @@ CREATE TABLE "seazona_lane_state" (
 	"updated_at" timestamp with time zone DEFAULT now() NOT NULL
 );
 --> statement-breakpoint
-CREATE TABLE "seazona_order_read_failures" (
+CREATE TABLE IF NOT EXISTS "seazona_order_read_failures" (
 	"order_id" text PRIMARY KEY NOT NULL,
 	"failure_count" integer NOT NULL,
 	"last_attempt_at" timestamp with time zone NOT NULL,
@@ -66,7 +66,7 @@ CREATE TABLE "seazona_order_read_failures" (
 	"quarantined_until" timestamp with time zone
 );
 --> statement-breakpoint
-CREATE TABLE "seazona_orders" (
+CREATE TABLE IF NOT EXISTS "seazona_orders" (
 	"id" text PRIMARY KEY NOT NULL,
 	"invoice_number" text,
 	"client_id" text,
@@ -88,7 +88,7 @@ CREATE TABLE "seazona_orders" (
 	"last_changed_at" timestamp with time zone NOT NULL
 );
 --> statement-breakpoint
-CREATE TABLE "seazona_reference" (
+CREATE TABLE IF NOT EXISTS "seazona_reference" (
 	"resource" text NOT NULL,
 	"id" text NOT NULL,
 	"name" text,
@@ -101,6 +101,6 @@ CREATE TABLE "seazona_reference" (
 	CONSTRAINT "seazona_reference_resource_id_pk" PRIMARY KEY("resource","id")
 );
 --> statement-breakpoint
-CREATE INDEX "seazona_invoices_changed" ON "seazona_invoices" USING btree ("last_changed_at");--> statement-breakpoint
-CREATE INDEX "seazona_orders_invoice_number" ON "seazona_orders" USING btree ("invoice_number");--> statement-breakpoint
-CREATE INDEX "seazona_orders_changed" ON "seazona_orders" USING btree ("last_changed_at");
+CREATE INDEX IF NOT EXISTS "seazona_invoices_changed" ON "seazona_invoices" USING btree ("last_changed_at");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "seazona_orders_invoice_number" ON "seazona_orders" USING btree ("invoice_number");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "seazona_orders_changed" ON "seazona_orders" USING btree ("last_changed_at");

package/server/integrations/seazona/migrations/0002_call_safety.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS "seazona_record_read_failures" (
+	"resource" text NOT NULL,
+	"record_key" text NOT NULL,
+	"failure_count" integer NOT NULL,
+	"last_attempt_at" timestamp with time zone NOT NULL,
+	"last_http_status" integer,
+	"last_failure_kind" text,
+	"quarantined_until" timestamp with time zone,
+	CONSTRAINT "seazona_record_read_failures_resource_record_key_pk" PRIMARY KEY("resource","record_key")
+);
+--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "circuit_open_until" timestamp with time zone DEFAULT 'epoch' NOT NULL;--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "circuit_opens" integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "halted_reason" text;--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "halted_at" timestamp with time zone;