@arcforgelabs/flow 2026.10.29 → 2026.10.32-beta.1

package/server/integrations/seazona/lanes.js

@@ -9,16 +9,21 @@ import {
   orderListSignature,
   text,
 } from './map.js';
+import { runHistoryBackfill } from './history/backfill.js';
 import { seazonaOrders } from './schema.js';
 import {
   claimLane,
   clearOrderFailure,
+  clearRecordFailures,
   finishLane,
   markOrderDeleted,
   readKnown,
   readLaneStates,
   readQuarantinedOrders,
+  readQuarantinedRecordCounts,
+  readQuarantinedRecords,
   recordOrderFailure,
+  recordRecordFailure,
   selectActiveOrders,
   touchSynced,
   upsertRecords,
@@ -34,46 +39,86 @@ import {
 //   clients        changed clients since a cursor, every 30 min
 //   products       the product list, daily
 //   users          the lab user list, daily
+//   history        past orders, a few calls per run, only while a person
+//                  has started a backfill (history/backfill.js)
 //
 // Issue #12 holds for every order read: a 404 marks the order deleted in
 // Seazona, an empty answer is a failure, each failure is recorded per order
 // and quarantines it after a few in a row, and a lane fails only when most of
 // its reads fail. Errors and /health name each failing order and its status.
+//
+// Issue #125: a list at the row cap is read on, not held: invoices and
+// clients from the last row they answered, orders page by page. A lane holds
+// only where Seazona gives no way on.
+//
+// Issue #101 holds for every lane: invoices and clients quarantine a record
+// after a few failed reads too, so one bad record never holds a cursor. A
+// list call that Seazona refuses (400, 401, 403, 404) means the path or the
+// API client is wrong, and no wait fixes it: the lane stops and runs again
+// only after a person saves the settings or Flow restarts on a new release.
+// Calls the governor stopped (SeazonaCallsStopped) wait out its pause.
 
-export const LANES = Object.freeze(['invoices', 'orders_recent', 'orders_active', 'clients', 'products', 'users']);
+export const LANES = Object.freeze(['invoices', 'orders_recent', 'orders_active', 'clients', 'products', 'users', 'history']);
 
 const DAY_MS = 24 * 60 * 60_000;
 const MAX_REPORTED = 25;
 const FINAL_STATUS_PATTERN = 'shipped|delivered|cancelled|canceled|void|closed|archived|deleted';
 
 class LaneError extends Error {
-  constructor(message, { code, details = null, held = false, retryAfterMs = null } = {}) {
+  constructor(message, { code, details = null, held = false, retryAfterMs = null, progress = null } = {}) {
     super(message);
     this.code = code;
     this.details = details;
     this.held = held;
     this.retryAfterMs = retryAfterMs;
+    // What the lane read before it stopped: { cursor }, kept with the failure.
+    this.progress = progress;
   }
 }
 
-function listOrFail(list, what, sync) {
+function listOrFail(list, what) {
   if (!Array.isArray(list)) throw new LaneError(`Seazona ${what} answer was not a list`, { code: 'seazona_list_invalid' });
-  // Seazona documents no paging. A list exactly at the cap may be cut off,
-  // so hold the cursor rather than trust it.
-  if (list.length === sync.listCap) {
-    throw new LaneError(`Seazona ${what} answer has exactly ${sync.listCap} rows and may be cut off; the cursor is held`, {
-      code: 'seazona_list_at_cap',
-      held: true,
-      details: { observedCount: list.length, listCap: sync.listCap },
-    });
-  }
   return list;
 }
 
+// Issue #125: a list at the cap is read on, page by page or from its last
+// row. The lane holds only where it cannot go on.
+function cannotGoOn(what, sync, reason, progress) {
+  return new LaneError(`Seazona ${what} answer is full (${sync.listCap} rows) and cannot be read further (${reason}); the cursor is held`, {
+    code: 'seazona_list_at_cap',
+    held: true,
+    details: { listCap: sync.listCap, reason },
+    progress,
+  });
+}
+
+// Seazona writes lastModified in UTC with no zone suffix.
+function parseUtc(value) {
+  const stamp = text(value);
+  if (!stamp) return NaN;
+  return Date.parse(/[zZ]|[+-]\d\d:?\d\d$/.test(stamp) ? stamp : `${stamp}Z`);
+}
+
 // A failure that is about Seazona or the rate limit, not one record: the
 // whole lane stops and waits.
 function isLaneLevel(error) {
-  return error?.statusCode === 429 || error?.kind === 'governor_timeout';
+  return error?.statusCode === 429 || error?.statusCode === 401 || error?.statusCode === 403
+    || error?.kind === 'governor_timeout' || error?.kind === 'calls_stopped';
+}
+
+// Answers how a lane that threw should wait: { status, heldReason, delayMs }.
+// A null delay means the lane is stopped until a person or a release.
+const REJECTED = new Set([400, 401, 403, 404]);
+function laneOutcome(error, state, sync) {
+  if (error?.kind === 'calls_stopped') {
+    if (error.reason === 'halted') return { status: 'stopped', heldReason: error.code, delayMs: null };
+    return { status: 'failed', heldReason: null, delayMs: Math.max(sync.retryBaseMs, error.retryAfterMs ?? 0) };
+  }
+  if (error?.name === 'SeazonaApiError' && REJECTED.has(error.statusCode)) {
+    const auth = error.statusCode === 401 || error.statusCode === 403;
+    return { status: 'stopped', heldReason: auth ? 'seazona_api_halted' : 'seazona_request_rejected', delayMs: null };
+  }
+  return { status: error?.held ? 'held' : 'failed', heldReason: error?.held ? error.code : null, delayMs: failureDelayMs(error, state, sync) };
 }
 
 function describeReadFailure(error) {
@@ -156,12 +201,16 @@ function addSummary(total, summary) {
 // Invoices and clients: a list since a cursor, then a detail read for each
 // row whose list fields changed.
 async function syncListWithDetail(ctx, { mirror, what, items, keyOf, idOf, signatureOf, detailPath, map }) {
-  const { db, client, at } = ctx;
+  const { db, client, sync, at } = ctx;
   const keyed = items.map((item) => ({ item, key: keyOf(item) })).filter((entry) => entry.key);
   const known = await readKnown(db, mirror, keyed.map((entry) => entry.key));
+  const quarantined = await readQuarantinedRecords(db, mirror, keyed.map((entry) => entry.key), { at });
   const summary = { created: 0, changed: 0, unchanged: 0 };
   const unchanged = [];
+  const read = [];
   const failures = [];
+  const skipped = [];
+  let newlyQuarantined = 0;
   let detailReads = 0;
   for (const { item, key } of keyed) {
     const signature = signatureOf(item);
@@ -169,6 +218,11 @@ async function syncListWithDetail(ctx, { mirror, what, items, keyOf, idOf, signa
       unchanged.push(key);
       continue;
     }
+    // Issue #101: a quarantined record costs no call until its wait ends.
+    if (quarantined.has(key)) {
+      skipped.push(key);
+      continue;
+    }
     const id = idOf(item);
     let detail = null;
     if (id) {
@@ -178,21 +232,92 @@ async function syncListWithDetail(ctx, { mirror, what, items, keyOf, idOf, signa
         if (!detail || typeof detail !== 'object' || Array.isArray(detail)) throw emptyResponse(`${what} ${key}`);
       } catch (error) {
         if (isLaneLevel(error)) throw error;
-        failures.push({ id: key, ...describeReadFailure(error) });
+        const failure = describeReadFailure(error);
+        const recorded = await recordRecordFailure(db, mirror, key, failure, {
+          at,
+          quarantineFailures: sync.orderQuarantineFailures,
+          quarantineMs: sync.orderQuarantineMs,
+        });
+        if (recorded.quarantinedUntil) newlyQuarantined += 1;
+        failures.push({ id: key, ...failure, ...recorded });
         continue;
       }
     }
+    read.push(key);
     addSummary(summary, await upsertRecords(db, mirror, [map(item, detail)], { at, listSignatures: new Map([[key, signature]]) }));
   }
   await touchSynced(db, mirror, unchanged, { at });
+  await clearRecordFailures(db, mirror, read);
   summary.unchanged += unchanged.length;
-  const failureDetails = judgeReads(`${what} reads`, detailReads, failures);
-  return { summary, detailReads, listCount: items.length, failureDetails, anyFailed: failures.length > 0 };
+  const extra = skipped.length ? { quarantinedIds: skipped.slice(0, MAX_REPORTED) } : {};
+  const failureDetails = judgeReads(`${what} reads`, detailReads, failures, extra);
+  // A quarantined record no longer holds the cursor; a record that may still
+  // be read again does.
+  const anyFailed = failures.some((failure) => !failure.quarantinedUntil);
+  return { summary, detailReads, listCount: items.length, failureDetails, anyFailed, newlyQuarantined, quarantined: skipped.length };
 }
 
-function cursorSince(ctx, fallbackMs) {
+// Where the next changed-list read starts. A run that stopped inside a
+// window keeps the exact start of the next part (`cursor.since`); otherwise
+// the last finished run's start, less the overlap.
+function cursorSince(ctx, fallback) {
+  const exact = ctx.state?.cursor?.since;
+  if (typeof exact === 'string') return exact;
   const cursorMs = ctx.state?.cursorTime?.getTime();
-  return new Date(Number.isFinite(cursorMs) ? cursorMs - ctx.sync.cursorOverlapMs : fallbackMs).toISOString();
+  return Number.isFinite(cursorMs) ? new Date(cursorMs - ctx.sync.cursorOverlapMs).toISOString() : fallback;
+}
+
+function mergeDetails(total, details) {
+  if (!details) return total;
+  if (!total) return details;
+  return {
+    ...total,
+    attempted: (total.attempted ?? 0) + (details.attempted ?? 0),
+    failed: (total.failed ?? 0) + (details.failed ?? 0),
+    failures: [...(total.failures ?? []), ...(details.failures ?? [])].slice(0, MAX_REPORTED),
+    ...(total.quarantinedIds || details.quarantinedIds
+      ? { quarantinedIds: [...(total.quarantinedIds ?? []), ...(details.quarantinedIds ?? [])].slice(0, MAX_REPORTED) }
+      : {}),
+  };
+}
+
+// Invoices and clients: Seazona answers changes since a time in one list,
+// with no paging, cut at the cap. The guide says the invoice list answers
+// changes on or after the time, oldest first, so a full list is one part of
+// the window: the next part starts at its last row (less `stepBackMs`, for a
+// list that answers strictly after). Rows out of order, or a full list inside
+// one second, cannot go on: the lane holds, and keeps where it got to. A run
+// reads at most `listPagesPerRun` parts; the next run goes on from the exact
+// start it stored.
+async function walkChangedList(ctx, { what, firstSince, stepBackMs, list, read }) {
+  const { sync, at, state } = ctx;
+  const total = { summary: { created: 0, changed: 0, unchanged: 0 }, detailReads: 0, listCount: 0, failureDetails: null, newlyQuarantined: 0, quarantined: 0 };
+  let since = firstSince;
+  for (let pages = 1; ; pages += 1) {
+    const items = listOrFail(await list(since), `${what} list`);
+    const result = await read(items);
+    addSummary(total.summary, result.summary);
+    total.detailReads += result.detailReads;
+    total.listCount += result.listCount;
+    total.newlyQuarantined += result.newlyQuarantined;
+    total.quarantined += result.quarantined;
+    total.failureDetails = mergeDetails(total.failureDetails, result.failureDetails);
+    const keep = { ...total, pages, since: firstSince || null };
+    // A failed read keeps this part's start, so the next run reads it again.
+    if (result.anyFailed) return { ...keep, anyFailed: true, cursorTime: state?.cursorTime ?? null, cursor: { since } };
+    if (items.length < sync.listCap) return { ...keep, cursorTime: at, cursor: {} };
+    const sinceMs = since ? parseUtc(since) : -Infinity;
+    let lastMs = -Infinity;
+    for (const item of items) {
+      const ms = parseUtc(item?.lastModified);
+      if (!Number.isFinite(ms) || ms < lastMs) throw cannotGoOn(`${what} list`, sync, 'rows are not oldest first', { cursor: { since } });
+      lastMs = ms;
+    }
+    const next = lastMs - stepBackMs;
+    if (!(next > sinceMs)) throw cannotGoOn(`${what} list`, sync, 'one second holds more than the cap', { cursor: { since } });
+    since = new Date(next).toISOString();
+    if (pages >= sync.listPagesPerRun) return { ...keep, cursorTime: state?.cursorTime ?? null, cursor: { since } };
+  }
 }
 
 const floorUtcDay = (ms) => {
@@ -206,24 +331,43 @@ export function formatOrderedDay(ms) {
   return new Date(floorUtcDay(ms)).toISOString().replace(/\.\d{3}Z$/, '');
 }
 
+// The guide's largest order page.
+const ORDER_PAGE_MAX = 500;
+
+// True when an order row was placed on a later day than `dayMs` (the start
+// of a UTC day). `ordered` is the lab's local time with no zone, written
+// yyyy-MM-ddTHH:mm:ss or dd/MM/yyyy; this compares calendar days only, so a
+// lab ahead of UTC never ends a day early. Unknown dates are not after.
+function placedAfter(item, dayMs) {
+  const raw = text(item?.ordered);
+  const iso = /^(\d{4})-(\d{2})-(\d{2})/.exec(raw);
+  const local = /^(\d{2})\/(\d{2})\/(\d{4})/.exec(raw);
+  const placed = iso ? Date.UTC(+iso[1], +iso[2] - 1, +iso[3]) : local ? Date.UTC(+local[3], +local[2] - 1, +local[1]) : NaN;
+  return Number.isFinite(placed) && placed > dayMs;
+}
+
 const laneRuns = {
   async invoices(ctx) {
     const { client, sync, at } = ctx;
-    const since = cursorSince(ctx, floorUtcDay(at.getTime()) - sync.invoiceLookbackDays * DAY_MS);
-    const items = listOrFail(await client.get('invoices/', { lastModified: since }), 'invoice list', sync);
+    const firstSince = cursorSince(ctx, new Date(floorUtcDay(at.getTime()) - sync.invoiceLookbackDays * DAY_MS).toISOString());
     const mapOptions = { keepPatientReference: sync.keepPatientReference };
-    const result = await syncListWithDetail(ctx, {
-      mirror: 'invoices',
+    return walkChangedList(ctx, {
       what: 'invoice',
-      items,
-      keyOf: (item) => text(item?.invoiceNumber),
-      idOf: (item) => text(item?.id),
-      signatureOf: invoiceListSignature,
-      detailPath: (id) => `invoices/${encodeURIComponent(id)}`,
-      map: (item, detail) => mapInvoice(item, detail, mapOptions),
+      firstSince,
+      // The guide: on or after lastModified (inclusive).
+      stepBackMs: 0,
+      list: (since) => client.get('invoices/', { lastModified: since }),
+      read: (items) => syncListWithDetail(ctx, {
+        mirror: 'invoices',
+        what: 'invoice',
+        items,
+        keyOf: (item) => text(item?.invoiceNumber),
+        idOf: (item) => text(item?.id),
+        signatureOf: invoiceListSignature,
+        detailPath: (id) => `invoices/${encodeURIComponent(id)}`,
+        map: (item, detail) => mapInvoice(item, detail, mapOptions),
+      }),
     });
-    // A failed invoice read keeps the cursor, so the next run reads it again.
-    return { ...result, cursorTime: result.anyFailed ? ctx.state?.cursorTime ?? null : at, since };
   },
 
   async orders_recent(ctx) {
@@ -231,15 +375,44 @@ const laneRuns = {
     const nowMs = at.getTime();
     const lookbackStart = floorUtcDay(nowMs - sync.orderLookbackDays * DAY_MS);
     const doneThrough = Date.parse(ctx.state?.cursor?.completedThrough ?? '');
-    // Re-read the last finished day as well, so late orders on it are seen.
-    const resume = Number.isFinite(doneThrough) ? Math.max(lookbackStart, doneThrough) : lookbackStart;
-    const days = [];
-    for (let day = floorUtcDay(resume); day <= floorUtcDay(nowMs) && days.length < sync.orderDaysPerRun; day += DAY_MS) days.push(day);
+    const partial = ctx.state?.cursor?.partial;
+    const partialDay = Date.parse(partial?.day ?? '');
+    // Go on inside a day a run did not finish; otherwise re-read the last
+    // finished day as well, so late orders on it are seen.
+    const resumeInside = Number.isFinite(partialDay) && partialDay >= lookbackStart;
+    const resume = resumeInside ? partialDay : Number.isFinite(doneThrough) ? Math.max(lookbackStart, doneThrough) : lookbackStart;
 
+    // Issue #125: the list is paged (page, pageSize). A full page goes on to
+    // the next page; a short page, or a page past the day, ends the day. The
+    // guide says the list answers orders on or after `ordered`; a list that
+    // answers one day works the same way.
+    const pageSize = Math.min(ORDER_PAGE_MAX, sync.listCap);
     const byId = new Map();
-    for (const day of days) {
-      const list = listOrFail(await client.get('orders/', { ordered: formatOrderedDay(day) }), `order list for ${formatOrderedDay(day)}`, sync);
-      for (const item of list) if (text(item?.id)) byId.set(text(item.id), item);
+    const days = [];
+    let calls = 0;
+    let lastDone = null;
+    let stoppedAt = null;
+    for (let day = floorUtcDay(resume); day <= floorUtcDay(nowMs) && days.length < sync.orderDaysPerRun && !stoppedAt; day += DAY_MS) {
+      const label = `order list for ${formatOrderedDay(day)}`;
+      let page = resumeInside && day === floorUtcDay(resume) ? Math.max(1, Number(partial.page) || 1) : 1;
+      let previous = null;
+      for (;;) {
+        if (calls >= sync.listPagesPerRun) {
+          stoppedAt = { day, page };
+          break;
+        }
+        calls += 1;
+        if (!days.includes(day)) days.push(day);
+        const list = listOrFail(await client.get('orders/', { ordered: formatOrderedDay(day), page, pageSize }), label);
+        const ids = list.map((item) => text(item?.id)).filter(Boolean);
+        // A Seazona that does not page answers the same rows again.
+        if (previous && ids.length && ids.every((id) => previous.has(id))) throw cannotGoOn(label, sync, 'the next page answers the same orders');
+        for (const item of list) if (text(item?.id)) byId.set(text(item.id), item);
+        if (list.length < pageSize || placedAfter(list[list.length - 1], day + DAY_MS)) break;
+        previous = new Set(ids);
+        page += 1;
+      }
+      if (!stoppedAt) lastDone = day;
     }
     const known = await readKnown(db, 'orders', [...byId.keys()]);
     const summary = { created: 0, changed: 0, unchanged: 0 };
@@ -263,16 +436,20 @@ const laneRuns = {
     await touchSynced(db, 'orders', unchanged, { at });
     summary.unchanged += unchanged.length;
     const failureDetails = judgeReads('recent order reads', detailReads, failures);
-    const lastDay = days.length ? days[days.length - 1] : floorUtcDay(nowMs);
+    const through = lastDone ?? (Number.isFinite(doneThrough) ? doneThrough : null);
     return {
       summary,
       detailReads,
       listCount: byId.size,
+      listCalls: calls,
       days: days.map(formatOrderedDay),
       deletedIds: deleted,
       failureDetails,
-      cursorTime: new Date(lastDay),
-      cursor: { completedThrough: new Date(lastDay).toISOString() },
+      cursorTime: through === null ? ctx.state?.cursorTime ?? null : new Date(through),
+      cursor: {
+        ...(through === null ? {} : { completedThrough: new Date(through).toISOString() }),
+        ...(stoppedAt ? { partial: { day: new Date(stoppedAt.day).toISOString(), page: stoppedAt.page } } : {}),
+      },
     };
   },
 
@@ -307,33 +484,40 @@ const laneRuns = {
   },
 
   async clients(ctx) {
-    const { client, sync, at } = ctx;
+    const { client } = ctx;
     // The first run reads the whole list; later runs read changes only.
     // Seazona answers the whole list only to an empty `?lastModified=`; a
     // bare path is a 404.
-    const since = ctx.state?.cursorTime ? cursorSince(ctx, 0) : '';
-    const reply = since ? await client.get('clients/', { lastModified: since }) : await client.get('clients/?lastModified=');
-    const items = listOrFail(reply, 'client list', sync);
-    const result = await syncListWithDetail(ctx, {
-      mirror: 'clients',
+    return walkChangedList(ctx, {
       what: 'client',
-      items,
-      keyOf: (item) => text(item?.id),
-      idOf: (item) => text(item?.id),
-      signatureOf: clientListSignature,
-      detailPath: (id) => `clients/${encodeURIComponent(id)}`,
-      map: (item, detail) => mapClient(item, detail),
+      firstSince: cursorSince(ctx, ''),
+      // The guide says "modified after": step back a second so a row in the
+      // last row's second is not lost.
+      stepBackMs: 1000,
+      list: (since) => (since ? client.get('clients/', { lastModified: since }) : client.get('clients/?lastModified=')),
+      read: (items) => syncListWithDetail(ctx, {
+        mirror: 'clients',
+        what: 'client',
+        items,
+        keyOf: (item) => text(item?.id),
+        idOf: (item) => text(item?.id),
+        signatureOf: clientListSignature,
+        detailPath: (id) => `clients/${encodeURIComponent(id)}`,
+        map: (item, detail) => mapClient(item, detail),
+      }),
     });
-    return { ...result, cursorTime: result.anyFailed ? ctx.state?.cursorTime ?? null : at, since: since || null };
   },
 
+  history: (ctx) => runHistoryBackfill(ctx, { isLaneLevel, describeReadFailure, emptyResponse, LaneError }),
   products: (ctx) => syncReference(ctx, 'products'),
   users: (ctx) => syncReference(ctx, 'users'),
 };
 
 async function syncReference(ctx, resource) {
   const { db, client, sync, at } = ctx;
-  const list = listOrFail(await client.get(`${resource}/`), `${resource} list`, sync);
+  const list = listOrFail(await client.get(`${resource}/`), `${resource} list`);
+  // The whole list in one answer, with no paging and no way to read on.
+  if (list.length === sync.listCap) throw cannotGoOn(`${resource} list`, sync, 'the list has no paging');
   const records = list.map((item) => mapReference(resource, item)).filter((record) => record.id);
   const { stored } = await upsertReference(db, resource, records, { at });
   return { listCount: list.length, stored, cursorTime: at };
@@ -358,6 +542,7 @@ export async function runSeazonaSync(db, client, { settings, now = Date.now, onl
   for (const lane of wanted) {
     const state = states.get(lane) ?? null;
     const startMs = Number(now());
+    if (!force && !only.length && state?.status === 'stopped') continue;
     if (!force && !only.length && state?.nextRunAt && state.nextRunAt.getTime() > startMs) continue;
     const at = new Date(startMs);
     if (!(await claimLane(db, lane, { at, stuckMs: sync.stuckLaneMs }))) {
@@ -382,18 +567,20 @@ export async function runSeazonaSync(db, client, { settings, now = Date.now, onl
       const { cursor, cursorTime, anyFailed, ...reported } = result;
       results[lane] = { ok: true, ...reported };
     } catch (error) {
-      const delayMs = failureDelayMs(error, state, sync);
+      const { status, heldReason, delayMs } = laneOutcome(error, state, sync);
       const message = text(error?.message) || 'Seazona lane failed';
+      const code = heldReason ?? error?.code ?? 'seazona_lane_failed';
       await finishLane(db, lane, {
-        status: error?.held ? 'held' : 'failed',
+        status,
         lastAttemptAt: at,
-        nextRunAt: new Date(Number(now()) + delayMs),
+        nextRunAt: delayMs === null ? null : new Date(Number(now()) + delayMs),
+        ...(error?.progress?.cursor ? { cursor: error.progress.cursor } : {}),
         failureCount: (state?.failureCount ?? 0) + 1,
         lastError: message,
-        failureDetails: error?.details ?? null,
-        heldReason: error?.held ? error.code : null,
+        failureDetails: error?.details ?? (Number.isInteger(error?.statusCode) ? { httpStatus: error.statusCode, path: error.path ?? null } : null),
+        heldReason,
       });
-      results[lane] = { ok: false, error: message, code: error?.code ?? 'seazona_lane_failed', details: error?.details ?? null, retryAfterMs: delayMs };
+      results[lane] = { ok: false, status, error: message, code, details: error?.details ?? null, retryAfterMs: delayMs };
     }
   }
   return { ok: Object.values(results).every((result) => result.ok), results };
@@ -407,6 +594,7 @@ export async function readSeazonaSyncHealth(db, settings, { now = Date.now } = {
   const states = await readLaneStates(db);
   const quarantinedOrders = await readQuarantinedOrders(db, { at: new Date(nowMs), limit: MAX_REPORTED });
   const [{ deleted }] = await db.select({ deleted: count() }).from(seazonaOrders).where(isNotNull(seazonaOrders.deletedInSourceAt));
+  const quarantinedRecords = await readQuarantinedRecordCounts(db, { at: new Date(nowMs) });
   const lanes = LANES.map((lane) => {
     const state = states.get(lane);
     const { freshnessMs } = settings.sync.lanes[lane];
@@ -424,10 +612,11 @@ export async function readSeazonaSyncHealth(db, settings, { now = Date.now } = {
       heldReason: state?.heldReason ?? null,
       failureDetails: state?.failureDetails ?? null,
       ...(lane === 'orders_active' ? { quarantinedOrders } : {}),
+      ...(quarantinedRecords[lane] ? { quarantinedRecords: quarantinedRecords[lane] } : {}),
     };
   });
   return {
-    ok: lanes.every((lane) => lane.status !== 'failed' && lane.status !== 'held'),
+    ok: lanes.every((lane) => !['failed', 'held', 'stopped'].includes(lane.status)),
     lanes,
     deletedOrders: Number(deleted),
   };

package/server/integrations/seazona/migrations/0002_call_safety.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS "seazona_record_read_failures" (
+	"resource" text NOT NULL,
+	"record_key" text NOT NULL,
+	"failure_count" integer NOT NULL,
+	"last_attempt_at" timestamp with time zone NOT NULL,
+	"last_http_status" integer,
+	"last_failure_kind" text,
+	"quarantined_until" timestamp with time zone,
+	CONSTRAINT "seazona_record_read_failures_resource_record_key_pk" PRIMARY KEY("resource","record_key")
+);
+--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "circuit_open_until" timestamp with time zone DEFAULT 'epoch' NOT NULL;--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "circuit_opens" integer DEFAULT 0 NOT NULL;--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "halted_reason" text;--> statement-breakpoint
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "halted_at" timestamp with time zone;

package/server/integrations/seazona/migrations/0003_call_guard.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "seazona_api_governor" ADD COLUMN IF NOT EXISTS "stopped_resources" jsonb DEFAULT '{}'::jsonb NOT NULL;--> statement-breakpoint
+ALTER TABLE "seazona_api_requests" ADD COLUMN IF NOT EXISTS "resource" text;

package/server/integrations/seazona/migrations/0004_history_import.sql

@@ -0,0 +1,68 @@
+CREATE TABLE IF NOT EXISTS "seazona_history_batch_jobs" (
+	"batch_id" bigint NOT NULL,
+	"job_key" text NOT NULL,
+	"job" jsonb NOT NULL,
+	"record_hash" text NOT NULL,
+	"outcome" text,
+	"findings" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"mirror_order_id" text,
+	"mirror_invoice_number" text,
+	"client_id" text,
+	"decision" text,
+	"decided_by" text,
+	"decided_at" timestamp with time zone,
+	CONSTRAINT "seazona_history_batch_jobs_batch_id_job_key_pk" PRIMARY KEY("batch_id","job_key")
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "seazona_history_batches" (
+	"id" bigserial PRIMARY KEY NOT NULL,
+	"file_name" text,
+	"file_sha256" text NOT NULL,
+	"profile" text NOT NULL,
+	"columns" jsonb DEFAULT '{}'::jsonb NOT NULL,
+	"ignored_columns" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"status" text NOT NULL,
+	"counts" jsonb DEFAULT '{}'::jsonb NOT NULL,
+	"created_by" text NOT NULL,
+	"created_at" timestamp with time zone DEFAULT now() NOT NULL,
+	"reconciled_at" timestamp with time zone,
+	"applied_at" timestamp with time zone,
+	"applied_by" text
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "seazona_history_jobs" (
+	"job_key" text PRIMARY KEY NOT NULL,
+	"invoice_number" text,
+	"case_ref" text,
+	"client_id" text,
+	"company" text,
+	"doctor_name" text,
+	"status" text,
+	"job_date" text,
+	"due_date" text,
+	"shipped_date" text,
+	"lines" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	"subtotal_cents" bigint,
+	"total_cents" bigint,
+	"patient_reference" text,
+	"mirror_order_id" text,
+	"mirror_invoice_number" text,
+	"record_hash" text NOT NULL,
+	"batch_id" bigint NOT NULL,
+	"first_batch_id" bigint NOT NULL,
+	"first_imported_at" timestamp with time zone NOT NULL,
+	"last_changed_at" timestamp with time zone NOT NULL
+);
+--> statement-breakpoint
+CREATE TABLE IF NOT EXISTS "seazona_history_rows" (
+	"batch_id" bigint NOT NULL,
+	"row_number" integer NOT NULL,
+	"job_key" text,
+	"fields" jsonb NOT NULL,
+	"problems" jsonb DEFAULT '[]'::jsonb NOT NULL,
+	CONSTRAINT "seazona_history_rows_batch_id_row_number_pk" PRIMARY KEY("batch_id","row_number")
+);
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "seazona_history_batches_file" ON "seazona_history_batches" USING btree ("file_sha256");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "seazona_history_jobs_invoice" ON "seazona_history_jobs" USING btree ("invoice_number");--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "seazona_history_jobs_changed" ON "seazona_history_jobs" USING btree ("last_changed_at");