@arcforgelabs/flow 2026.10.29 → 2026.10.32-beta.1

package/server/integrations/seazona/governor.js

@@ -1,7 +1,10 @@
 import { and, count, desc, eq, gte, lt, sql } from 'drizzle-orm';
+import { CallsStopped, countsAsFailure, createCallGuard, normaliseRequestPath } from '../call-guard.js';
 import { seazonaApiGovernor, seazonaApiRequests } from './schema.js';
 import { MAX_REQUESTS_PER_MINUTE } from './settings.js';
 
+export { countsAsFailure, normaliseRequestPath };
+
 // The Seazona rate governor. Seazona sees every process that holds the same
 // API client as one client, so the limit is kept in PostgreSQL, not in
 // memory: a workspace and a deployment on one database share it.
@@ -10,18 +13,36 @@ import { MAX_REQUESTS_PER_MINUTE } from './settings.js';
 // admitted in the last minute, the minimum gap since the last admission has
 // passed, and no 429 cooldown is running. Admission takes a transaction-level
 // advisory lock per client, so two processes never admit the same slot.
+//
+// Issue #101: a vendor can cut off API access after many failed calls, even
+// inside the rate limit. So the governor also stops calls outright, through
+// the shared call guard (../call-guard.js, issue #111) on these same tables:
+//   - a 401 or 403 halts every call until a person acts (the adapter restarts
+//     with new or saved settings, which calls resume());
+//   - `circuitFailures` failed calls inside `circuitWindowMs` open the circuit:
+//     no call for `circuitPauseMs`, then one probe call; a failed probe opens
+//     it again for twice as long, to at most 6 hours, and a success closes it;
+//   - `dailyFailureBudget` failed calls in one UTC day stop calls until the
+//     next day.
+// A stopped call fails at once with SeazonaCallsStopped; it never waits. The
+// lanes report each held failure once (lanes.js), so the guard does not.
+
+// The call guard's tables and policy. The adapter declares it (index.js), and
+// the shared contract suite runs on it (test/integration-call-guard.test.js).
+export const SEAZONA_CALL_GUARD = Object.freeze({
+  integration: 'seazona',
+  name: 'Seazona',
+  tables: Object.freeze({ state: 'seazona_api_governor', calls: 'seazona_api_requests' }),
+  columns: Object.freeze({ key: 'client_key', at: 'admitted_at' }),
+  callDefaults: Object.freeze({ workflow: 'seazona' }),
+  lockName: 'seazona-governor',
+  policy: Object.freeze({ authStops: 'all' }),
+});
 
 const WINDOW_MS = 60_000;
 const MAX_COOLDOWN_MS = 60_000;
-
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
 
-// Path only: drop any host and query, and keep a safe character set.
-export function normaliseRequestPath(value) {
-  const path = String(value || '').split(/[?#]/)[0].replace(/^[a-z]+:\/\/[^/]+/i, '');
-  return path.replace(/[^A-Za-z0-9/_.:-]+/g, '').slice(0, 160) || null;
-}
-
 function normaliseWorkflow(value) {
   return String(value || 'seazona').trim().toLowerCase().replace(/[^a-z0-9:_-]+/g, '-').slice(0, 80) || 'seazona';
 }
@@ -36,12 +57,27 @@ export class SeazonaGovernorTimeout extends Error {
   }
 }
 
+// Calls are stopped: halted for a person, circuit open, or the day's failed
+// calls are spent. `retryAfterMs` is null for a halt, which no wait clears.
+// The shared CallsStopped, with Seazona's own words.
+export class SeazonaCallsStopped extends CallsStopped {
+  constructor(reason, retryAfterMs = null) {
+    super({ integration: 'seazona', name: 'Seazona', reason, retryAfterMs });
+    if (reason === 'halted') this.message = 'Seazona calls are stopped until a person checks the API client (it was refused)';
+    this.name = 'SeazonaCallsStopped';
+  }
+}
+
 export function createSeazonaGovernor({
   db,
   clientKey,
   requestsPerMinute = MAX_REQUESTS_PER_MINUTE,
   maxWaitMs = 120_000,
   telemetryRetentionMs = 35 * 24 * 60 * 60_000,
+  circuitFailures = 5,
+  circuitWindowMs = 10 * 60_000,
+  circuitPauseMs = 5 * 60_000,
+  dailyFailureBudget = 50,
   now = Date.now,
   sleepFn = sleep,
 } = {}) {
@@ -50,7 +86,14 @@ export function createSeazonaGovernor({
   if (!key) throw new TypeError('clientKey is required');
   const limit = Math.min(MAX_REQUESTS_PER_MINUTE, Math.max(1, Math.trunc(requestsPerMinute) || MAX_REQUESTS_PER_MINUTE));
   const minIntervalMs = Math.ceil(WINDOW_MS / limit);
-  const lockKey = `seazona-governor:${key}`;
+  const lockKey = `${SEAZONA_CALL_GUARD.lockName}:${key}`;
+  const guard = createCallGuard({
+    db,
+    ...SEAZONA_CALL_GUARD,
+    policy: { ...SEAZONA_CALL_GUARD.policy, circuitFailures, circuitWindowMs, circuitPauseMs, dailyFailureBudget, telemetryRetentionMs },
+    report: false,
+    now,
+  });
 
   async function tryAdmit(workflow, isRetry) {
     return db.transaction(async (tx) => {
@@ -59,6 +102,8 @@ export function createSeazonaGovernor({
       const nowAt = new Date(nowMs);
       await tx.insert(seazonaApiGovernor).values({ clientKey: key }).onConflictDoNothing();
       const [state] = await tx.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
+      const stopped = await guard.stoppedNow(tx, key);
+      if (stopped) return { admitted: false, stopped };
       const windowStart = new Date(nowMs - WINDOW_MS);
       const [{ admitted, oldestMs }] = await tx
         .select({ admitted: count(), oldestMs: sql`(extract(epoch from min(${seazonaApiRequests.admittedAt})) * 1000)::float8` })
@@ -90,6 +135,7 @@ export function createSeazonaGovernor({
     for (;;) {
       const decision = await tryAdmit(workflow, isRetry);
       if (decision.admitted) return { requestId: decision.requestId, waitedMs };
+      if (decision.stopped) throw new SeazonaCallsStopped(decision.stopped.reason, decision.stopped.retryAfterMs);
       const delay = Math.max(1, Math.min(decision.waitMs, maxWaitMs));
       if (waitedMs + delay > maxWaitMs) throw new SeazonaGovernorTimeout(waitedMs, delay);
       waitedMs += delay;
@@ -97,31 +143,32 @@ export function createSeazonaGovernor({
     }
   }
 
-  // Records how a request ended, and any cooldown a 429 asked for.
+  // Records how a request ended, any cooldown a 429 asked for, and what the
+  // ending does to the circuit (the shared guard). Answers { halted, opened }
+  // when this response stopped calls.
   async function recordResponse({ requestId, path = null, statusCode = null, failureKind = null, durationMs = null, retryAfterMs = null } = {}) {
-    const nowMs = Number(now());
-    await db.update(seazonaApiRequests).set({
-      requestPath: normaliseRequestPath(path),
-      statusCode: Number.isInteger(statusCode) ? statusCode : null,
-      failureKind: failureKind ? String(failureKind).slice(0, 40) : null,
-      durationMs: Number.isFinite(durationMs) ? Math.max(0, Math.round(durationMs)) : null,
-    }).where(and(eq(seazonaApiRequests.id, requestId), eq(seazonaApiRequests.clientKey, key)));
     if (Number.isFinite(retryAfterMs) && retryAfterMs > 0) {
+      const nowMs = Number(now());
       const until = new Date(nowMs + Math.min(retryAfterMs, MAX_COOLDOWN_MS));
       await db.update(seazonaApiGovernor)
         .set({ cooldownUntil: sql`greatest(${seazonaApiGovernor.cooldownUntil}, ${until})`, updatedAt: new Date(nowMs) })
         .where(eq(seazonaApiGovernor.clientKey, key));
     }
-    await db.delete(seazonaApiRequests).where(and(
-      eq(seazonaApiRequests.clientKey, key),
-      lt(seazonaApiRequests.admittedAt, new Date(nowMs - telemetryRetentionMs)),
-    ));
+    const { halted, opened } = await guard.record(key, { callId: requestId, path, statusCode, failureKind, durationMs });
+    return { ...(halted ? { halted } : {}), ...(opened ? { opened } : {}) };
+  }
+
+  // A person acted (new or saved settings, or the adapter turned on again):
+  // clear a halt and close the circuit. The day's failure budget stays.
+  async function resume() {
+    await guard.resume(key);
   }
 
   // Counts for /health: the last minute, and the last day by status.
   async function status() {
     const nowMs = Number(now());
     const [state] = await db.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
+    const calls = await guard.status(key);
     const [{ lastMinute }] = await db.select({ lastMinute: count() }).from(seazonaApiRequests)
       .where(and(eq(seazonaApiRequests.clientKey, key), gte(seazonaApiRequests.admittedAt, new Date(nowMs - WINDOW_MS))));
     const day = await summarise({ from: new Date(nowMs - 24 * 60 * 60_000), to: new Date(nowMs) });
@@ -131,6 +178,12 @@ export function createSeazonaGovernor({
       minIntervalMs,
       lastMinute: Number(lastMinute),
       cooldownUntil,
+      circuit: calls.circuit,
+      circuitOpenUntil: calls.circuitOpenUntil,
+      haltedReason: calls.haltedReason,
+      haltedAt: calls.haltedAt,
+      failedToday: calls.failedToday,
+      dailyFailureBudget: calls.dailyFailureBudget,
       lastDay: { requests: day.requests, failed: day.failed, rateLimited: day.rateLimited, retries: day.retries },
     };
   }
@@ -170,5 +223,5 @@ export function createSeazonaGovernor({
     };
   }
 
-  return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, status, summarise };
+  return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, resume, status, summarise };
 }

package/server/integrations/seazona/history/backfill.js

@@ -0,0 +1,208 @@
+import { and, eq, inArray } from 'drizzle-orm';
+import { FlowError } from '../../../core/errors.js';
+import { mapOrder, parseSeazonaDate, text } from '../map.js';
+import { seazonaHistoryBatchJobs, seazonaLaneState, seazonaOrders } from '../schema.js';
+import { recordRecordFailure } from '../store.js';
+import { createApiBatch, finishApiBatch, stageApiJobs } from './index.js';
+
+// The history backfill (docs/SEAZONA.md, "History import"). A person starts
+// it with a first day; the `history` lane then walks the orders from that day
+// on with GET orders/?ordered=<day> and its pages, and reads each order's
+// detail. It makes at most
+// `historyCallsPerRun` calls a run, through the same rate governor and call
+// guards as the live lanes, and keeps its place in the lane's cursor, so it
+// resumes after a restart. Orders the live copy already holds are not read
+// again. Each order becomes one job in an API batch: reconciled as it is
+// read, written to Flow's history when it is new, and kept for a person when
+// it is not. The batch rows are the record of what the API confirmed.
+
+export const LANE = 'history';
+const PAGE_SIZE = 500;
+const DAY_MS = 24 * 60 * 60_000;
+const RESOURCE = 'history_orders';
+const dayOf = (ms) => new Date(ms).toISOString().slice(0, 10);
+const orderedParam = (day) => `${day}T00:00:00`;
+
+// An order's detail as a history job.
+export function orderToHistoryJob(record) {
+  return {
+    invoiceNumber: record.invoiceNumber,
+    caseRef: record.id,
+    company: record.company,
+    accountNumber: null,
+    doctorName: record.doctorName,
+    status: record.status,
+    jobDate: null,
+    orderedDate: parseSeazonaDate(record.orderedText),
+    dueDate: parseSeazonaDate(record.dueText),
+    shippedDate: null,
+    lines: (record.lines ?? []).map((l) => ({
+      description: l.description,
+      code: l.code ?? null,
+      amountCents: l.amountCents ?? null,
+      ...(typeof l.taxable === 'boolean' ? { taxable: l.taxable } : {}),
+      ...(l.tooth ? { tooth: l.tooth } : {}),
+    })),
+    subtotalCents: record.subtotalCents,
+    totalCents: null,
+    patientReference: record.patientReference,
+  };
+}
+
+// Starts a backfill from `from` (yyyy-mm-dd) through `through`, by default the
+// day before the live recent-order lane's window. One runs at a time.
+export async function startBackfill(db, { from, through = null, createdBy, settings, now = Date.now }) {
+  const at = new Date(Number(now()));
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(String(from ?? '')) || Number.isNaN(Date.parse(from))) {
+    throw Object.assign(new FlowError('invalid', 'from must be a day, yyyy-mm-dd'), { status: 400 });
+  }
+  const last = through ?? dayOf(at.getTime() - (settings.sync.orderLookbackDays + 1) * DAY_MS);
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(last) || last < from) {
+    throw Object.assign(new FlowError('invalid', 'through must be a day on or after from'), { status: 400 });
+  }
+  const [state] = await db.select().from(seazonaLaneState).where(eq(seazonaLaneState.lane, LANE));
+  if (state?.cursor?.batchId && !state.cursor.done) {
+    throw Object.assign(new FlowError('conflict', 'A history backfill is already running'), { status: 409 });
+  }
+  const batch = await createApiBatch(db, { createdBy, from, through: last, at });
+  const cursor = { batchId: batch.id, from, through: last, nextDay: from, page: 1, pending: [], done: false, days: 0, read: 0 };
+  await db.insert(seazonaLaneState).values({ lane: LANE, status: 'idle', cursor, nextRunAt: at })
+    .onConflictDoUpdate({ target: seazonaLaneState.lane, set: { status: 'idle', cursor, nextRunAt: at, failureCount: 0, lastError: null, heldReason: null, updatedAt: at } });
+  return describeBackfill(cursor);
+}
+
+// Stops a running backfill. Its batch keeps what it read and waits for a person.
+export async function stopBackfill(db, { now = Date.now } = {}) {
+  const at = new Date(Number(now()));
+  const [state] = await db.select().from(seazonaLaneState).where(eq(seazonaLaneState.lane, LANE));
+  const cursor = state?.cursor;
+  if (!cursor?.batchId || cursor.done) throw Object.assign(new FlowError('conflict', 'No history backfill is running'), { status: 409 });
+  const stopped = { ...cursor, done: true, stopped: true };
+  await db.update(seazonaLaneState).set({ cursor: stopped, updatedAt: at }).where(eq(seazonaLaneState.lane, LANE));
+  await finishApiBatch(db, cursor.batchId, { at });
+  return describeBackfill(stopped);
+}
+
+export function describeBackfill(cursor) {
+  if (!cursor?.batchId) return null;
+  return {
+    batchId: cursor.batchId,
+    from: cursor.from,
+    through: cursor.through,
+    nextDay: cursor.done ? null : cursor.nextDay,
+    daysDone: cursor.days ?? 0,
+    reachedDay: cursor.reached ?? null,
+    ordersRead: cursor.read ?? 0,
+    waitingToRead: cursor.pending?.length ?? 0,
+    running: !cursor.done,
+    stopped: Boolean(cursor.stopped),
+  };
+}
+
+export async function readBackfill(db) {
+  const [state] = await db.select().from(seazonaLaneState).where(eq(seazonaLaneState.lane, LANE));
+  return describeBackfill(state?.cursor);
+}
+
+// One run of the `history` lane. `helpers` are the lane module's failure
+// rules, so the backfill fails, holds and stops exactly as the live lanes do.
+export async function runHistoryBackfill(ctx, { isLaneLevel, describeReadFailure, emptyResponse, LaneError }) {
+  const { db, client, sync, at, state } = ctx;
+  const start = state?.cursor ?? {};
+  if (!start.batchId || start.done) return { idle: true, cursor: start, cursorTime: state?.cursorTime ?? null };
+  const cursor = { ...start, pending: [...(start.pending ?? [])] };
+  const budget = sync.historyCallsPerRun;
+  const pageSize = sync.historyPageSize ?? PAGE_SIZE;
+  const jobs = [];
+  const failures = [];
+  let calls = 0;
+  let deleted = 0;
+  let quarantined = 0;
+
+  const inBatch = async (ids) => {
+    if (!ids.length) return new Set();
+    const rows = await db.select({ key: seazonaHistoryBatchJobs.jobKey }).from(seazonaHistoryBatchJobs)
+      .where(and(eq(seazonaHistoryBatchJobs.batchId, cursor.batchId), inArray(seazonaHistoryBatchJobs.jobKey, ids.map((id) => `case:${id}`))));
+    return new Set(rows.map((r) => r.key.slice(5)));
+  };
+
+  try {
+    while (calls < budget) {
+      if (!cursor.pending.length) {
+        if (cursor.nextDay > cursor.through) {
+          cursor.done = true;
+          break;
+        }
+        calls += 1;
+        const list = await client.get('orders/', { ordered: orderedParam(cursor.nextDay), page: cursor.page, pageSize });
+        if (!Array.isArray(list)) throw new LaneError(`Seazona order list for ${cursor.nextDay} answer was not a list`, { code: 'seazona_list_invalid' });
+        // The guide says the list answers orders placed on or after `ordered`,
+        // oldest first, in stable pages; the live lane was built when it
+        // answered one day. Both work here: orders after `through` end the walk,
+        // a full page goes on to the next page, and a short page to the next day.
+        const placed = (item) => (typeof item?.ordered === 'string' ? item.ordered.slice(0, 10) : null);
+        const inRange = list.filter((item) => !(placed(item) > cursor.through));
+        const beyond = inRange.length < list.length;
+        for (const item of inRange) if (placed(item) > (cursor.reached ?? '')) cursor.reached = placed(item);
+        const ids = [...new Set(inRange.map((item) => text(item?.id)).filter(Boolean))];
+        // The live copy holds these already; a read again costs a call for nothing.
+        const live = ids.length
+          ? new Set((await db.select({ id: seazonaOrders.id }).from(seazonaOrders).where(inArray(seazonaOrders.id, ids))).map((r) => r.id))
+          : new Set();
+        const seen = await inBatch(ids);
+        cursor.pending = ids.filter((id) => !live.has(id) && !seen.has(id));
+        if (beyond) {
+          cursor.nextDay = dayOf(Date.parse(cursor.through) + DAY_MS);
+        } else if (list.length >= pageSize) {
+          cursor.page += 1;
+        } else {
+          cursor.page = 1;
+          cursor.nextDay = dayOf(Date.parse(cursor.nextDay) + DAY_MS);
+          cursor.days = (cursor.days ?? 0) + 1;
+        }
+        continue;
+      }
+      const id = cursor.pending.shift();
+      if ((await inBatch([id])).size) continue;
+      calls += 1;
+      try {
+        const detail = await client.get(`orders/${encodeURIComponent(id)}`);
+        if (!detail || typeof detail !== 'object' || Array.isArray(detail)) throw emptyResponse(`order ${id}`);
+        const record = { ...mapOrder({}, detail, { keepPatientReference: sync.keepPatientReference }), id };
+        jobs.push({ jobKey: `case:${id}`, job: orderToHistoryJob(record) });
+        cursor.read = (cursor.read ?? 0) + 1;
+      } catch (error) {
+        if (isLaneLevel(error)) throw error;
+        // Seazona has no such order now: nothing to keep.
+        if (error?.statusCode === 404) {
+          deleted += 1;
+          continue;
+        }
+        const failure = describeReadFailure(error);
+        const recorded = await recordRecordFailure(db, RESOURCE, id, failure, {
+          at, quarantineFailures: sync.orderQuarantineFailures, quarantineMs: sync.orderQuarantineMs,
+        });
+        failures.push({ id, ...failure, ...recorded });
+        // Read again later in the walk, until it is quarantined.
+        if (recorded.quarantinedUntil) quarantined += 1;
+        else cursor.pending.push(id);
+      }
+    }
+  } finally {
+    // What was read is kept even when the lane stops on a lane-level failure.
+    await stageApiJobs(db, cursor.batchId, jobs, { at });
+  }
+  if (cursor.done) await finishApiBatch(db, cursor.batchId, { at });
+  const failureDetails = failures.length
+    ? { attempted: calls, failed: failures.length, failures: failures.slice(0, 25).map(({ error, ...f }) => f), quarantined }
+    : null;
+  return {
+    calls,
+    read: jobs.length,
+    deleted,
+    backfill: describeBackfill(cursor),
+    failureDetails,
+    cursor,
+    cursorTime: at,
+  };
+}

package/server/integrations/seazona/history/csv.js

@@ -0,0 +1,65 @@
+// A small RFC 4180 reader for Seazona's CSV export: quoted fields, doubled
+// quotes, commas and line breaks inside quotes, CRLF or LF, and a leading
+// byte order mark. Answers { headers, rows }, each row an array of strings.
+// Blank lines are skipped. It never evaluates a cell.
+
+export class CsvError extends Error {
+  constructor(message, line) {
+    super(line ? `${message} (line ${line})` : message);
+    this.code = 'csv_invalid';
+    this.line = line ?? null;
+  }
+}
+
+export function parseCsv(input) {
+  const text = String(input ?? '').replace(/^/, '');
+  const records = [];
+  let record = [];
+  let field = '';
+  let quoted = false;
+  let line = 1;
+  let startLine = 1;
+  for (let i = 0; i < text.length; i += 1) {
+    const ch = text[i];
+    if (quoted) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i += 1;
+        } else {
+          quoted = false;
+        }
+      } else {
+        if (ch === '\n') line += 1;
+        field += ch;
+      }
+      continue;
+    }
+    if (ch === '"' && field === '') {
+      quoted = true;
+    } else if (ch === ',') {
+      record.push(field);
+      field = '';
+    } else if (ch === '\n' || ch === '\r') {
+      if (ch === '\r' && text[i + 1] === '\n') i += 1;
+      record.push(field);
+      records.push({ cells: record, line: startLine });
+      record = [];
+      field = '';
+      line += 1;
+      startLine = line;
+    } else {
+      field += ch;
+    }
+  }
+  if (quoted) throw new CsvError('A quoted field has no closing quote', startLine);
+  if (field !== '' || record.length) {
+    record.push(field);
+    records.push({ cells: record, line: startLine });
+  }
+  const nonBlank = records.filter((r) => r.cells.some((cell) => cell.trim() !== ''));
+  if (!nonBlank.length) throw new CsvError('The file has no header row');
+  const [head, ...body] = nonBlank;
+  const headers = head.cells.map((h) => h.trim());
+  return { headers, rows: body.map((r) => ({ line: r.line, cells: r.cells })) };
+}