@arcforgelabs/flow 2026.10.29 → 2026.10.30

package/server/integrations/seazona/migrations/meta/_journal.json

@@ -15,6 +15,13 @@
       "when": 1792330268730,
       "tag": "0001_mirror",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "7",
+      "when": 1792448378477,
+      "tag": "0002_call_safety",
+      "breakpoints": true
     }
   ]
 }

package/server/integrations/seazona/scheduler.js

@@ -4,12 +4,20 @@ import { readLaneStates } from './store.js';
 // The background sync, for live mode only (issue #10: index.js never creates
 // it in sandbox or replay). It runs the lanes that are due, then sleeps until
 // the next lane is due, between 15 seconds and 30 minutes. One run at a time.
+// A stopped lane (issue #101) is not due until a person or a release resets it.
+//
+// Issue #101: each held failure is reported once to the error services
+// (log.error, server/core/monitoring.js): when a lane starts failing or its
+// failure code changes, and when records are quarantined. A repeat of the
+// same failure is a warning in the host log only. The report text is stable,
+// so the error service groups repeats.
 
 const MIN_DELAY_MS = 15_000;
 const MAX_DELAY_MS = 30 * 60_000;
 
 export function createSeazonaScheduler({
   db,
+  settings = null,
   run,
   now = Date.now,
   log = console,
@@ -21,11 +29,39 @@ export function createSeazonaScheduler({
   let stopped = false;
   let nextRunAt = null;
   let last = null;
+  // lane -> the failure code last reported, while that lane keeps failing.
+  const reported = new Map();
+  const quarantineAt = settings?.sync?.orderQuarantineFailures ?? null;
+
+  function newlyQuarantined(outcome) {
+    if (Number.isInteger(outcome.newlyQuarantined)) return outcome.newlyQuarantined;
+    const failures = outcome.failureDetails?.failures ?? [];
+    return failures.filter((f) => f.quarantinedUntil && f.failureCount === quarantineAt).length;
+  }
+
+  function report(results) {
+    for (const [lane, outcome] of Object.entries(results)) {
+      if (outcome.ok) {
+        reported.delete(lane);
+        const count = newlyQuarantined(outcome);
+        if (count) log.error?.(`[seazona] ${lane}: records quarantined after repeated failed reads`);
+        continue;
+      }
+      if (reported.get(lane) === outcome.code) {
+        log.warn?.(`[seazona] ${lane}: ${outcome.error}`);
+        continue;
+      }
+      reported.set(lane, outcome.code);
+      log.error?.(`[seazona] ${lane} lane ${outcome.status === 'stopped' ? 'stopped' : 'failed'}: ${outcome.code}`);
+      log.warn?.(`[seazona] ${lane}: ${outcome.error}`);
+    }
+  }
 
   async function nextDelayMs() {
     const states = await readLaneStates(db);
     let soonest = Infinity;
     for (const lane of LANES) {
+      if (states.get(lane)?.status === 'stopped') continue;
       const at = states.get(lane)?.nextRunAt?.getTime();
       soonest = Math.min(soonest, Number.isFinite(at) ? at - Number(now()) : 0);
     }
@@ -48,7 +84,7 @@ export function createSeazonaScheduler({
         const result = await run();
         const failed = Object.entries(result.results).filter(([, lane]) => !lane.ok);
         last = { at: startedAt, ok: result.ok, lanes: Object.keys(result.results), failed: failed.map(([lane]) => lane) };
-        for (const [lane, outcome] of failed) log.warn?.(`[seazona] ${lane}: ${outcome.error}`);
+        report(result.results);
       } catch (error) {
         last = { at: startedAt, ok: false, error: error?.message || String(error) };
         log.error?.(`[seazona] sync run failed: ${last.error}`);

package/server/integrations/seazona/schema.js

@@ -10,10 +10,17 @@ const at = (name) => timestamp(name, { withTimezone: true });
 
 // One row per API client: when the next request may go, and any cooldown a
 // 429 asked for. Every process that shares the database shares this row.
+// Issue #101: the circuit breaker (open until, and how many times in a row it
+// opened) and a stop that waits for a person (`halted_reason`, for example
+// `auth` after a 401).
 export const seazonaApiGovernor = pgTable('seazona_api_governor', {
   clientKey: text('client_key').primaryKey(),
   nextEligibleAt: at('next_eligible_at').notNull().default(sql`'epoch'`),
   cooldownUntil: at('cooldown_until').notNull().default(sql`'epoch'`),
+  circuitOpenUntil: at('circuit_open_until').notNull().default(sql`'epoch'`),
+  circuitOpens: integer('circuit_opens').notNull().default(0),
+  haltedReason: text('halted_reason'),
+  haltedAt: at('halted_at'),
   updatedAt: at('updated_at').notNull().defaultNow(),
 });
 
@@ -155,3 +162,18 @@ export const seazonaOrderReadFailures = pgTable('seazona_order_read_failures', {
   lastError: text('last_error'),
   quarantinedUntil: at('quarantined_until'),
 });
+
+// Issue #101: each failed detail read of an invoice or a client, per record.
+// Orders keep their own table (issue #12). After a few failures in a row the
+// record is quarantined: the lane skips it and its cursor moves on.
+export const seazonaRecordReadFailures = pgTable('seazona_record_read_failures', {
+  resource: text('resource').notNull(),
+  recordKey: text('record_key').notNull(),
+  failureCount: integer('failure_count').notNull(),
+  lastAttemptAt: at('last_attempt_at').notNull(),
+  lastHttpStatus: integer('last_http_status'),
+  lastFailureKind: text('last_failure_kind'),
+  quarantinedUntil: at('quarantined_until'),
+}, (t) => [
+  primaryKey({ columns: [t.resource, t.recordKey] }),
+]);

package/server/integrations/seazona/settings.js

@@ -75,6 +75,16 @@ export function resolveSeazonaSettings(env = process.env) {
     maxWaitMs: clampInt(env.SEAZONA_API_MAX_WAIT_MS, 120_000, 1_000, 15 * 60_000),
     // Long enough to answer a vendor's question about last month.
     telemetryRetentionMs: clampInt(env.SEAZONA_API_TELEMETRY_RETENTION_DAYS, 35, 1, 90) * DAY_MS,
+    // Issue #101: a vendor can cut off API access after many failed calls,
+    // even inside the rate limit. This many failed calls inside the window
+    // open the circuit: no call goes out for the pause, which doubles each
+    // time the probe after it fails, to at most 6 hours.
+    circuitFailures: clampInt(env.SEAZONA_CIRCUIT_FAILURES, 5, 1, 100),
+    circuitWindowMs: clampInt(env.SEAZONA_CIRCUIT_WINDOW_MS, 10 * 60_000, 60_000, DAY_MS),
+    circuitPauseMs: clampInt(env.SEAZONA_CIRCUIT_PAUSE_MS, 5 * 60_000, 60_000, 6 * 60 * 60_000),
+    // Failed calls allowed in one UTC day. When they are spent, calls stop
+    // until the next day.
+    dailyFailureBudget: clampInt(env.SEAZONA_DAILY_FAILURE_BUDGET, 50, 1, 10_000),
   };
 
   const MINUTE = 60_000;

package/server/integrations/seazona/store.js

@@ -6,6 +6,7 @@ import {
   seazonaLaneState,
   seazonaOrderReadFailures,
   seazonaOrders,
+  seazonaRecordReadFailures,
   seazonaReference,
 } from './schema.js';
 
@@ -192,6 +193,60 @@ export async function readQuarantinedOrders(db, { at, limit = 25 }) {
   }));
 }
 
+// Issue #101: failed detail reads of invoices and clients, per record.
+// Answers Map(key -> row) for the keys that are quarantined at `at`.
+export async function readQuarantinedRecords(db, resource, keys, { at }) {
+  if (!keys.length) return new Map();
+  const rows = await db.select().from(seazonaRecordReadFailures).where(and(
+    eq(seazonaRecordReadFailures.resource, resource),
+    inArray(seazonaRecordReadFailures.recordKey, keys),
+    gt(seazonaRecordReadFailures.quarantinedUntil, at),
+  ));
+  return new Map(rows.map((row) => [row.recordKey, row]));
+}
+
+export async function clearRecordFailures(db, resource, keys) {
+  if (!keys.length) return;
+  await db.delete(seazonaRecordReadFailures).where(and(
+    eq(seazonaRecordReadFailures.resource, resource),
+    inArray(seazonaRecordReadFailures.recordKey, keys),
+  ));
+}
+
+// Records one failed read, as recordOrderFailure does for orders. Answers
+// { failureCount, quarantinedUntil }.
+export async function recordRecordFailure(db, resource, key, failure, { at, quarantineFailures, quarantineMs }) {
+  const until = new Date(at.getTime() + quarantineMs);
+  const [row] = await db.insert(seazonaRecordReadFailures).values({
+    resource,
+    recordKey: key,
+    failureCount: 1,
+    lastAttemptAt: at,
+    lastHttpStatus: failure.httpStatus,
+    lastFailureKind: failure.kind,
+    quarantinedUntil: quarantineFailures <= 1 ? until : null,
+  }).onConflictDoUpdate({
+    target: [seazonaRecordReadFailures.resource, seazonaRecordReadFailures.recordKey],
+    set: {
+      failureCount: sql`${seazonaRecordReadFailures.failureCount} + 1`,
+      lastAttemptAt: at,
+      lastHttpStatus: failure.httpStatus,
+      lastFailureKind: failure.kind,
+      quarantinedUntil: sql`case when ${seazonaRecordReadFailures.failureCount} + 1 >= ${quarantineFailures}
+        then ${until}::timestamptz else null end`,
+    },
+  }).returning();
+  return { failureCount: row.failureCount, quarantinedUntil: row.quarantinedUntil?.toISOString() ?? null };
+}
+
+export async function readQuarantinedRecordCounts(db, { at }) {
+  const rows = await db.select({ resource: seazonaRecordReadFailures.resource, records: sql`count(*)::int` })
+    .from(seazonaRecordReadFailures)
+    .where(gt(seazonaRecordReadFailures.quarantinedUntil, at))
+    .groupBy(seazonaRecordReadFailures.resource);
+  return Object.fromEntries(rows.map((row) => [row.resource, row.records]));
+}
+
 // Lane state.
 
 export async function readLaneStates(db) {
@@ -214,6 +269,25 @@ export async function claimLane(db, lane, { at, stuckMs }) {
   return claimed.length > 0;
 }
 
+// Issue #101: a lane stopped for a person or a release runs again. Answers
+// the lanes it reset.
+export async function resetStoppedLanes(db, { at }) {
+  const rows = await db.update(seazonaLaneState)
+    .set({ status: 'idle', nextRunAt: at, failureCount: 0, heldReason: null, updatedAt: at })
+    .where(eq(seazonaLaneState.status, 'stopped'))
+    .returning({ lane: seazonaLaneState.lane });
+  return rows.map((row) => row.lane);
+}
+
+// A person acted: every lane that failed or stopped runs at the next tick.
+export async function resetFailedLanes(db, { at }) {
+  const rows = await db.update(seazonaLaneState)
+    .set({ status: 'idle', nextRunAt: at, failureCount: 0, heldReason: null, updatedAt: at })
+    .where(inArray(seazonaLaneState.status, ['stopped', 'failed', 'held']))
+    .returning({ lane: seazonaLaneState.lane });
+  return rows.map((row) => row.lane);
+}
+
 export async function finishLane(db, lane, patch) {
   await db.update(seazonaLaneState).set({ ...patch, updatedAt: patch.lastAttemptAt ?? new Date() }).where(eq(seazonaLaneState.lane, lane));
 }

package/server/integrations/seazona/tables.js

@@ -13,5 +13,6 @@ export const seazonaSchema = Object.freeze({
     'seazona_reference',
     'seazona_lane_state',
     'seazona_order_read_failures',
+    'seazona_record_read_failures',
   ]),
 });

package/server/integrations/xero/import.js

@@ -109,6 +109,7 @@ export function createImportWorker({ db, runtime, log = console, timers = { setT
         if (ran) {
           busy = true;
           const read = (ran.job.result?.results ?? []).map((r) => `${r.resource} ${r.read}`).join(', ');
+          if (ran.outcome === 'dead') log.error?.('[xero] the history import gave up after its last attempt');
           log.info?.(`[xero] import ${ran.outcome} for organisation ${ran.job.organisationId}${read ? `: ${read}` : ''}${ran.job.result?.more ? `; more (${ran.job.result.more})` : ''}`);
         }
       } catch (error) {

package/server/ui/monitoring.js

@@ -16,6 +16,16 @@ function loadScript({ src, integrity }) {
   });
 }
 
+// "Notify me": the form's email field (Sentry's own words) starts empty and
+// is optional. A person who types an address there asks to hear back; Flow
+// tags the report notify=yes, so the support desk sees it on the task.
+// Sentry's form has no tick box of its own; this keeps to its stock options.
+export function tagNotify(event) {
+  if (event?.type !== 'feedback') return;
+  const email = String(event.contexts?.feedback?.contact_email ?? '').trim();
+  event.tags = { ...event.tags, notify: email ? 'yes' : 'no' };
+}
+
 /** Resolves to a function that opens the "Report a problem" form, or null. */
 export async function startMonitoring({ flow, person }) {
   const config = await flow('/api/monitoring');
@@ -30,6 +40,9 @@ export async function startMonitoring({ flow, person }) {
     showBranding: false,
     // Flow already knows who is signed in (setUser below).
     showName: false,
+    // The email field is the opt-in (tagNotify), so it is not filled from
+    // the signed-in person: a key setUser never sets.
+    useSentryUser: { name: 'username', email: 'notify_email' },
     enableScreenshot: true,
     triggerLabel: 'Report a problem',
     formTitle: 'Report a problem',
@@ -45,6 +58,7 @@ export async function startMonitoring({ flow, person }) {
     sendDefaultPii: false,
     integrations,
   });
+  Sentry.getClient()?.on('beforeSendFeedback', tagNotify);
   // Who reported or met the error, so the lab can follow up.
   Sentry.setUser({ id: person.id, email: person.email });
   const feedback = config.feedback ? Sentry.getFeedback() : null;