@arcforgelabs/flow 2026.10.29 → 2026.10.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@arcforgelabs/flow",
3
- "version": "2026.10.29",
3
+ "version": "2026.10.30",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@arcforgelabs/flow",
9
- "version": "2026.10.29",
9
+ "version": "2026.10.30",
10
10
  "license": "MIT",
11
11
  "dependencies": {
12
12
  "@clerk/backend": "3.23.2",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@arcforgelabs/flow",
3
- "version": "2026.10.29",
3
+ "version": "2026.10.30",
4
4
  "description": "Invoicing, bill entry and a financial dashboard for labs, with Seazona and Xero integrations.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -1,5 +1,5 @@
1
1
  import { and, eq, sql } from 'drizzle-orm';
2
- import { automations, billingObligations } from '../db/schema.js';
2
+ import { automations, billingObligations, invoices } from '../db/schema.js';
3
3
  import { FlowError, notFound } from './errors.js';
4
4
  import { autoDraftJobs } from './invoicing-rules.js';
5
5
  import { MAPPING_DEFAULTS, readMappingConfig, remapJobs } from './line-mapping.js';
@@ -17,6 +17,8 @@ import { setReady } from './ledger.js';
17
17
  // are picked from those the work system reported (workStatuses).
18
18
  // draft_when_ready: a job that is ready to invoice and passes the invoicing
19
19
  // rules (invoicing-rules.js) gets a draft invoice in Flow.
20
+ // ready_when_drafted: a draft the draft routine made goes straight to Ready
21
+ // on the Invoices page, until a person changes it or moves it back.
20
22
 
21
23
  const BATCH = 500;
22
24
 
@@ -82,6 +84,14 @@ export const AUTOMATIONS = Object.freeze([
82
84
  defaults: { enabled: true, config: {} },
83
85
  readConfig: () => ({}),
84
86
  },
87
+ {
88
+ key: 'ready_when_drafted',
89
+ place: 'invoicing',
90
+ name: 'Ready when drafted',
91
+ hint: 'An automatic draft goes straight to Ready. A draft a person changed stays in Drafts.',
92
+ defaults: { enabled: true, config: {} },
93
+ readConfig: () => ({}),
94
+ },
85
95
  ]);
86
96
 
87
97
  const byKey = new Map(AUTOMATIONS.map((a) => [a.key, a]));
@@ -138,6 +148,25 @@ async function markReadyFromWorkStatus(db, organisationId, { statuses }) {
138
148
  return { marked };
139
149
  }
140
150
 
151
+ // Marks ready each draft the draft routine made that no person has touched.
152
+ // A person's edit or "back to drafts" moves the draft's updatedAt on from its
153
+ // createdAt, so the routine leaves it to people from then on. A draft with a
154
+ // problem shows in Problem whatever its ready mark (sending.js, invoiceViews).
155
+ async function readyWhenDrafted(db, organisationId) {
156
+ const rows = await db.update(invoices).set({ readyAt: sql`now()`, readyBy: null })
157
+ .where(and(
158
+ eq(invoices.organisationId, organisationId),
159
+ eq(invoices.status, 'draft'),
160
+ sql`${invoices.readyAt} is null`,
161
+ sql`${invoices.updatedAt} = ${invoices.createdAt}`,
162
+ sql`exists (select 1 from billing_obligations o join obligation_events e on e.obligation_id = o.id
163
+ where o.organisation_id = ${invoices.organisationId} and o.invoice_id = ${invoices.id}
164
+ and e.kind = 'drafted' and e.actor = ${actorOf('draft_when_ready')})`,
165
+ ))
166
+ .returning({ id: invoices.id });
167
+ return { marked: rows.length };
168
+ }
169
+
141
170
  // Runs every routine that is on. `accounting` is the integration whose copy
142
171
  // the draft routine checks; without one, Flow's own records are the check.
143
172
  export async function runAutomations(db, organisationId, { accounting = null, now = new Date() } = {}) {
@@ -148,6 +177,8 @@ export async function runAutomations(db, organisationId, { accounting = null, no
148
177
  summary[automation.key] = await markReadyFromWorkStatus(db, organisationId, automation.config);
149
178
  } else if (automation.key === 'draft_when_ready') {
150
179
  summary[automation.key] = await autoDraftJobs(db, organisationId, { accounting, now, actor: actorOf(automation.key) });
180
+ } else if (automation.key === 'ready_when_drafted') {
181
+ summary[automation.key] = await readyWhenDrafted(db, organisationId);
151
182
  }
152
183
  }
153
184
  return summary;
@@ -9,7 +9,7 @@ import { readInvoicingSettings } from './invoicing-rules.js';
9
9
  import { observeJob, recordAccountingEvidence, setHold } from './ledger.js';
10
10
  import { loadMappingContext, mapLines } from './line-mapping.js';
11
11
  import { linkRecord, priorLinkFor } from './links.js';
12
- import { enqueueJob, runNextJob } from './queue.js';
12
+ import { deadHoldSince, deadSince, enqueueJob, runNextJob } from './queue.js';
13
13
 
14
14
  // The pipeline (docs/DATA_SPINE.md, issue 58). It joins what the integrations
15
15
  // copy into Flow, and never writes to another system:
@@ -448,6 +448,7 @@ export async function runPipelineOnce(db, integrations, { now = new Date() } = {
448
448
  // systems' copies and writes Flow's own records.
449
449
  export function createPipelineWorker({ db, integrations, log = console, timers = { setTimeout, clearTimeout }, everyMs = RUN_EVERY_MS, firstDelayMs = 15_000 }) {
450
450
  const workerId = `pipeline-${randomUUID()}`;
451
+ const startedAtMs = Date.now();
451
452
  let timer = null;
452
453
  let running = null;
453
454
  let stopped = true;
@@ -467,6 +468,9 @@ export function createPipelineWorker({ db, integrations, log = console, timers =
467
468
  if (!integrations.with('pullWorkItems').length && !integrations.with('findInvoices').length) return;
468
469
  const organisationId = await soleOrganisation(db);
469
470
  if (!organisationId) return;
471
+ // Issue #101: a run that gave up waits for a person, a restart or the
472
+ // next day; the timer never starts it again by itself.
473
+ if (await deadSince(db, organisationId, { kind: PIPELINE_KIND, dedupeKey: 'all', since: deadHoldSince(startedAtMs) })) return;
470
474
  await enqueueJob(db, organisationId, { kind: PIPELINE_KIND, dedupeKey: 'all' });
471
475
  const ran = await runNextJob(db, {
472
476
  workerId,
@@ -474,8 +478,10 @@ export function createPipelineWorker({ db, integrations, log = console, timers =
474
478
  handlers: { [PIPELINE_KIND]: () => runPipelineOnce(db, integrations) },
475
479
  });
476
480
  const line = ran ? `${ran.outcome}: ${JSON.stringify(ran.job.result ?? ran.job.lastError ?? {})}` : '';
477
- // Log a run only when its outcome differs from the last one.
478
- if (line && line !== last) log.info?.(`[pipeline] ${line}`);
481
+ // Log a run only when its outcome differs from the last one. A run
482
+ // that gave up is reported once to the error services.
483
+ if (ran?.outcome === 'dead') log.error?.('[pipeline] the pipeline run gave up after its last attempt');
484
+ if (line && line !== last) (ran.outcome === 'succeeded' ? log.info : log.warn)?.(`[pipeline] ${line}`);
479
485
  last = line || last;
480
486
  } catch (error) {
481
487
  log.error?.(`[pipeline] ${error?.code ?? 'error'}: ${error?.message ?? error}`);
@@ -38,6 +38,28 @@ export async function enqueueJob(db, organisationId, {
38
38
  return { job: open, queued: false };
39
39
  }
40
40
 
41
+ // Issue #101: the newest job of this kind and key that gave up after `since`,
42
+ // or null. A worker on a timer checks this before it queues the next run, so
43
+ // a job that always fails is not started again and again: it waits for a
44
+ // person (the review item), a restart on a new release, or the next day.
45
+ export async function deadSince(db, organisationId, { kind, dedupeKey = '', since }) {
46
+ const [row] = await db.select().from(backgroundJobs).where(and(
47
+ eq(backgroundJobs.organisationId, organisationId),
48
+ eq(backgroundJobs.kind, kind),
49
+ eq(backgroundJobs.dedupeKey, dedupeKey),
50
+ eq(backgroundJobs.state, 'dead'),
51
+ sql`${backgroundJobs.updatedAt} > ${since}`,
52
+ )).orderBy(sql`${backgroundJobs.updatedAt} desc`).limit(1);
53
+ return row ?? null;
54
+ }
55
+
56
+ // When a timer may queue the job again after it gave up: the later of this
57
+ // process's start and one day ago.
58
+ export const DEAD_HOLD_MS = 24 * 60 * 60 * 1000;
59
+ export function deadHoldSince(startedAtMs, nowMs = Date.now()) {
60
+ return new Date(Math.max(startedAtMs, nowMs - DEAD_HOLD_MS));
61
+ }
62
+
41
63
  async function giveUp(tx, job, error) {
42
64
  const [row] = await tx.update(backgroundJobs)
43
65
  .set({ state: 'dead', lockedBy: null, lockedUntil: null, lastError: error, updatedAt: sql`now()` })
@@ -4,7 +4,7 @@ import { invoiceLines, invoices } from '../db/schema.js';
4
4
  import { FlowError } from './errors.js';
5
5
  import { issueInvoice } from './invoices.js';
6
6
  import { soleOrganisation } from './pipeline.js';
7
- import { enqueueJob, runNextJob } from './queue.js';
7
+ import { deadHoldSince, deadSince, enqueueJob, runNextJob } from './queue.js';
8
8
 
9
9
  // Drafts, Problem, Ready and Send all ready (docs/INVOICE_SENDING.md).
10
10
  //
@@ -124,9 +124,10 @@ export async function markReady(db, organisationId, invoiceIds, { personId, acco
124
124
  return { marked: good.length, refused: [...problems].map(([invoiceId, found]) => ({ invoiceId, problems: found })) };
125
125
  }
126
126
 
127
- // Back to drafts: the drafts need another look before they go.
127
+ // Back to drafts: the drafts need another look before they go. The new
128
+ // updatedAt keeps "Ready when drafted" (automations.js) off them.
128
129
  export async function markNotReady(db, organisationId, invoiceIds) {
129
- const rows = await db.update(invoices).set({ readyAt: null, readyBy: null })
130
+ const rows = await db.update(invoices).set({ readyAt: null, readyBy: null, updatedAt: sql`now()` })
130
131
  .where(and(eq(invoices.organisationId, organisationId), eq(invoices.status, 'draft'), inArray(invoices.id, invoiceIds)))
131
132
  .returning({ id: invoices.id });
132
133
  return { moved: rows.length };
@@ -215,6 +216,7 @@ export async function sendQueued(db, organisationId, { accounting, limit = WORKE
215
216
  const error = result.error ?? { code: `send_${result.outcome}`, message: `Sending ended as ${result.outcome}.` };
216
217
  await settle(db, organisationId, result.invoiceId, { sendState: 'failed', sendError: { code: error.code, message: error.message } });
217
218
  summary.failed += 1;
219
+ summary.refused = [...new Set([...(summary.refused ?? []), error.code])];
218
220
  }
219
221
  }
220
222
  }
@@ -230,6 +232,7 @@ function settle(db, organisationId, invoiceId, fields) {
230
232
  // the job queue.
231
233
  export function createSendWorker({ db, integrations, log = console, timers = { setTimeout, clearTimeout }, everyMs = WORKER_EVERY_MS }) {
232
234
  const workerId = `send-${randomUUID()}`;
235
+ const startedAtMs = Date.now();
233
236
  let timer = null;
234
237
  let running = null;
235
238
  let stopped = true;
@@ -252,13 +255,22 @@ export function createSendWorker({ db, integrations, log = console, timers = { s
252
255
  const [waiting] = await db.select({ id: invoices.id }).from(invoices)
253
256
  .where(and(eq(invoices.organisationId, organisationId), eq(invoices.sendState, 'queued'))).limit(1);
254
257
  if (!waiting) return;
258
+ // Issue #101: a send run that gave up waits for a person, a restart or
259
+ // the next day; the timer never starts it again by itself.
260
+ if (await deadSince(db, organisationId, { kind: SEND_KIND, dedupeKey: 'all', since: deadHoldSince(startedAtMs) })) return;
255
261
  await enqueueJob(db, organisationId, { kind: SEND_KIND, dedupeKey: 'all' });
256
262
  const ran = await runNextJob(db, {
257
263
  workerId,
258
264
  leaseMs: LEASE_MS,
259
265
  handlers: { [SEND_KIND]: () => sendQueued(db, organisationId, { accounting }) },
260
266
  });
261
- if (ran) log.info?.(`[sending] ${ran.outcome}: ${JSON.stringify(ran.job.result ?? ran.job.lastError ?? {})}`);
267
+ if (!ran) return;
268
+ const line = `[sending] ${ran.outcome}: ${JSON.stringify(ran.job.result ?? ran.job.lastError ?? {})}`;
269
+ // An invoice the accounting system refused waits in Problem for a
270
+ // person; it is reported once, when it is refused.
271
+ if (ran.outcome === 'dead') log.error?.('[sending] the send run gave up after its last attempt');
272
+ if (ran.job.result?.failed) log.error?.('[sending] the accounting system refused an invoice');
273
+ (ran.outcome === 'succeeded' && !ran.job.result?.failed ? log.info : log.warn)?.(line);
262
274
  } catch (error) {
263
275
  log.error?.(`[sending] ${error?.code ?? 'error'}: ${error?.message ?? error}`);
264
276
  }
@@ -234,7 +234,9 @@ export function createRegistry({ log = console, env = process.env } = {}) {
234
234
  tablesReady.add(id);
235
235
  }
236
236
  if (adapter.start) {
237
- await adapter.start({ db: database, log, env: settingsEnv });
237
+ // A person changed the settings: an adapter may clear a stop that
238
+ // waits for a person (issue #101).
239
+ await adapter.start({ db: database, log, env: settingsEnv, reconfigured: true });
238
240
  running.push(adapter);
239
241
  }
240
242
  } catch (error) {
@@ -17,6 +17,8 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, m
17
17
  // A failed Seazona request. The message names the method, the path and the
18
18
  // status or failure kind; never the host, the query or a credential.
19
19
  // kind: http, timeout, network, invalid_json, replay_miss, governor_timeout.
20
+ // The governor's own SeazonaCallsStopped (kind calls_stopped) passes through
21
+ // unchanged: no call was made.
20
22
  export class SeazonaApiError extends Error {
21
23
  constructor(message, { statusCode = null, kind = 'http', path = null, retryAfterMs = null } = {}) {
22
24
  super(message);
@@ -93,6 +95,10 @@ export function createSeazonaClient({
93
95
  requestsPerMinute: rate.requestsPerMinute,
94
96
  maxWaitMs: rate.maxWaitMs,
95
97
  telemetryRetentionMs: rate.telemetryRetentionMs,
98
+ circuitFailures: rate.circuitFailures,
99
+ circuitWindowMs: rate.circuitWindowMs,
100
+ circuitPauseMs: rate.circuitPauseMs,
101
+ dailyFailureBudget: rate.dailyFailureBudget,
96
102
  now,
97
103
  sleepFn,
98
104
  });
@@ -10,9 +10,39 @@ import { MAX_REQUESTS_PER_MINUTE } from './settings.js';
10
10
  // admitted in the last minute, the minimum gap since the last admission has
11
11
  // passed, and no 429 cooldown is running. Admission takes a transaction-level
12
12
  // advisory lock per client, so two processes never admit the same slot.
13
+ //
14
+ // Issue #101: a vendor can cut off API access after many failed calls, even
15
+ // inside the rate limit. So the governor also stops calls outright:
16
+ // - a 401 or 403 halts every call until a person acts (the adapter restarts
17
+ // with new or saved settings, which calls resume());
18
+ // - `circuitFailures` failed calls inside `circuitWindowMs` open the circuit:
19
+ // no call for `circuitPauseMs`, then one probe call; a failed probe opens
20
+ // it again for twice as long, to at most 6 hours, and a success closes it;
21
+ // - `dailyFailureBudget` failed calls in one UTC day stop calls until the
22
+ // next day.
23
+ // A stopped call fails at once with SeazonaCallsStopped; it never waits.
13
24
 
14
25
  const WINDOW_MS = 60_000;
15
26
  const MAX_COOLDOWN_MS = 60_000;
27
+ const MAX_CIRCUIT_PAUSE_MS = 6 * 60 * 60_000;
28
+ const DAY_MS = 24 * 60 * 60_000;
29
+ const EPOCH = new Date(0);
30
+
31
+ // A call that failed in a way the vendor sees as a fault: no answer, a 5xx, a
32
+ // 429, or a 4xx other than 404 (a 404 on one record means it was deleted).
33
+ export function countsAsFailure({ statusCode = null, failureKind = null } = {}) {
34
+ if (failureKind) return true;
35
+ if (!Number.isInteger(statusCode)) return false;
36
+ return statusCode >= 500 || (statusCode >= 400 && statusCode !== 404);
37
+ }
38
+
39
+ const failedCall = sql`(${seazonaApiRequests.failureKind} is not null or ${seazonaApiRequests.statusCode} >= 500
40
+ or (${seazonaApiRequests.statusCode} >= 400 and ${seazonaApiRequests.statusCode} <> 404))`;
41
+
42
+ const startOfUtcDay = (ms) => {
43
+ const date = new Date(ms);
44
+ return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
45
+ };
16
46
 
17
47
  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
18
48
 
@@ -36,12 +66,34 @@ export class SeazonaGovernorTimeout extends Error {
36
66
  }
37
67
  }
38
68
 
69
+ // Calls are stopped: halted for a person, circuit open, or the day's failed
70
+ // calls are spent. `retryAfterMs` is null for a halt, which no wait clears.
71
+ export class SeazonaCallsStopped extends Error {
72
+ constructor(reason, retryAfterMs = null) {
73
+ const what = {
74
+ halted: 'Seazona calls are stopped until a person checks the API client (it was refused)',
75
+ circuit_open: 'Seazona calls are paused after repeated failures',
76
+ budget_spent: "Seazona calls are stopped for today: the day's failed calls are spent",
77
+ }[reason];
78
+ super(what);
79
+ this.name = 'SeazonaCallsStopped';
80
+ this.code = { halted: 'seazona_api_halted', circuit_open: 'seazona_api_circuit_open', budget_spent: 'seazona_api_failure_budget_spent' }[reason];
81
+ this.kind = 'calls_stopped';
82
+ this.reason = reason;
83
+ this.retryAfterMs = retryAfterMs;
84
+ }
85
+ }
86
+
39
87
  export function createSeazonaGovernor({
40
88
  db,
41
89
  clientKey,
42
90
  requestsPerMinute = MAX_REQUESTS_PER_MINUTE,
43
91
  maxWaitMs = 120_000,
44
92
  telemetryRetentionMs = 35 * 24 * 60 * 60_000,
93
+ circuitFailures = 5,
94
+ circuitWindowMs = 10 * 60_000,
95
+ circuitPauseMs = 5 * 60_000,
96
+ dailyFailureBudget = 50,
45
97
  now = Date.now,
46
98
  sleepFn = sleep,
47
99
  } = {}) {
@@ -59,6 +111,8 @@ export function createSeazonaGovernor({
59
111
  const nowAt = new Date(nowMs);
60
112
  await tx.insert(seazonaApiGovernor).values({ clientKey: key }).onConflictDoNothing();
61
113
  const [state] = await tx.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
114
+ const stopped = await stoppedReason(tx, state, nowMs);
115
+ if (stopped) return { admitted: false, stopped };
62
116
  const windowStart = new Date(nowMs - WINDOW_MS);
63
117
  const [{ admitted, oldestMs }] = await tx
64
118
  .select({ admitted: count(), oldestMs: sql`(extract(epoch from min(${seazonaApiRequests.admittedAt})) * 1000)::float8` })
@@ -84,12 +138,31 @@ export function createSeazonaGovernor({
84
138
  });
85
139
  }
86
140
 
141
+ async function failuresSince(tx, fromMs) {
142
+ const [{ failed }] = await tx.select({ failed: count() }).from(seazonaApiRequests)
143
+ .where(and(eq(seazonaApiRequests.clientKey, key), gte(seazonaApiRequests.admittedAt, new Date(fromMs)), failedCall));
144
+ return Number(failed);
145
+ }
146
+
147
+ // Why no call may go now, or null.
148
+ async function stoppedReason(tx, state, nowMs) {
149
+ if (state.haltedReason) return { reason: 'halted', retryAfterMs: null };
150
+ const openUntil = state.circuitOpenUntil.getTime();
151
+ if (openUntil > nowMs) return { reason: 'circuit_open', retryAfterMs: openUntil - nowMs };
152
+ const dayStart = startOfUtcDay(nowMs);
153
+ if (await failuresSince(tx, dayStart) >= dailyFailureBudget) {
154
+ return { reason: 'budget_spent', retryAfterMs: dayStart + DAY_MS - nowMs };
155
+ }
156
+ return null;
157
+ }
158
+
87
159
  // Waits for a slot and records the admission. Returns { requestId }.
88
160
  async function reserve({ workflow = 'seazona', isRetry = false } = {}) {
89
161
  let waitedMs = 0;
90
162
  for (;;) {
91
163
  const decision = await tryAdmit(workflow, isRetry);
92
164
  if (decision.admitted) return { requestId: decision.requestId, waitedMs };
165
+ if (decision.stopped) throw new SeazonaCallsStopped(decision.stopped.reason, decision.stopped.retryAfterMs);
93
166
  const delay = Math.max(1, Math.min(decision.waitMs, maxWaitMs));
94
167
  if (waitedMs + delay > maxWaitMs) throw new SeazonaGovernorTimeout(waitedMs, delay);
95
168
  waitedMs += delay;
@@ -97,25 +170,67 @@ export function createSeazonaGovernor({
97
170
  }
98
171
  }
99
172
 
100
- // Records how a request ended, and any cooldown a 429 asked for.
173
+ // Records how a request ended, any cooldown a 429 asked for, and what the
174
+ // ending does to the circuit. Answers { halted, opened } when this response
175
+ // stopped calls, so the caller can report it once.
101
176
  async function recordResponse({ requestId, path = null, statusCode = null, failureKind = null, durationMs = null, retryAfterMs = null } = {}) {
102
177
  const nowMs = Number(now());
103
- await db.update(seazonaApiRequests).set({
104
- requestPath: normaliseRequestPath(path),
105
- statusCode: Number.isInteger(statusCode) ? statusCode : null,
106
- failureKind: failureKind ? String(failureKind).slice(0, 40) : null,
107
- durationMs: Number.isFinite(durationMs) ? Math.max(0, Math.round(durationMs)) : null,
108
- }).where(and(eq(seazonaApiRequests.id, requestId), eq(seazonaApiRequests.clientKey, key)));
109
- if (Number.isFinite(retryAfterMs) && retryAfterMs > 0) {
110
- const until = new Date(nowMs + Math.min(retryAfterMs, MAX_COOLDOWN_MS));
111
- await db.update(seazonaApiGovernor)
112
- .set({ cooldownUntil: sql`greatest(${seazonaApiGovernor.cooldownUntil}, ${until})`, updatedAt: new Date(nowMs) })
178
+ const nowAt = new Date(nowMs);
179
+ const outcome = await db.transaction(async (tx) => {
180
+ await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${lockKey}))`);
181
+ await tx.update(seazonaApiRequests).set({
182
+ requestPath: normaliseRequestPath(path),
183
+ statusCode: Number.isInteger(statusCode) ? statusCode : null,
184
+ failureKind: failureKind ? String(failureKind).slice(0, 40) : null,
185
+ durationMs: Number.isFinite(durationMs) ? Math.max(0, Math.round(durationMs)) : null,
186
+ }).where(and(eq(seazonaApiRequests.id, requestId), eq(seazonaApiRequests.clientKey, key)));
187
+ if (Number.isFinite(retryAfterMs) && retryAfterMs > 0) {
188
+ const until = new Date(nowMs + Math.min(retryAfterMs, MAX_COOLDOWN_MS));
189
+ await tx.update(seazonaApiGovernor)
190
+ .set({ cooldownUntil: sql`greatest(${seazonaApiGovernor.cooldownUntil}, ${until})`, updatedAt: nowAt })
191
+ .where(eq(seazonaApiGovernor.clientKey, key));
192
+ }
193
+ const [state] = await tx.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
194
+ if (!state) return {};
195
+ if (!countsAsFailure({ statusCode, failureKind })) {
196
+ if (state.circuitOpens > 0) {
197
+ await tx.update(seazonaApiGovernor).set({ circuitOpens: 0, updatedAt: nowAt }).where(eq(seazonaApiGovernor.clientKey, key));
198
+ }
199
+ return {};
200
+ }
201
+ if (statusCode === 401 || statusCode === 403) {
202
+ if (state.haltedReason) return {};
203
+ await tx.update(seazonaApiGovernor).set({ haltedReason: 'auth', haltedAt: nowAt, updatedAt: nowAt })
204
+ .where(eq(seazonaApiGovernor.clientKey, key));
205
+ return { halted: 'auth' };
206
+ }
207
+ const openUntil = state.circuitOpenUntil.getTime();
208
+ // After a pause the next call is a probe: if it fails, open again at once.
209
+ const probeFailed = state.circuitOpens > 0 && openUntil <= nowMs;
210
+ const counted = probeFailed ? 0 : await failuresSince(tx, Math.max(nowMs - circuitWindowMs, openUntil));
211
+ if (!probeFailed && counted < circuitFailures) return {};
212
+ if (!probeFailed && openUntil > nowMs) return {};
213
+ const pauseMs = Math.min(MAX_CIRCUIT_PAUSE_MS, circuitPauseMs * 2 ** Math.min(state.circuitOpens, 10));
214
+ const until = new Date(nowMs + pauseMs);
215
+ await tx.update(seazonaApiGovernor)
216
+ .set({ circuitOpenUntil: until, circuitOpens: state.circuitOpens + 1, updatedAt: nowAt })
113
217
  .where(eq(seazonaApiGovernor.clientKey, key));
114
- }
218
+ return { opened: { until: until.toISOString(), pauseMs, opens: state.circuitOpens + 1 } };
219
+ });
115
220
  await db.delete(seazonaApiRequests).where(and(
116
221
  eq(seazonaApiRequests.clientKey, key),
117
222
  lt(seazonaApiRequests.admittedAt, new Date(nowMs - telemetryRetentionMs)),
118
223
  ));
224
+ return outcome;
225
+ }
226
+
227
+ // A person acted (new or saved settings, or the adapter turned on again):
228
+ // clear a halt and close the circuit. The day's failure budget stays.
229
+ async function resume() {
230
+ await db.insert(seazonaApiGovernor).values({ clientKey: key }).onConflictDoNothing();
231
+ await db.update(seazonaApiGovernor)
232
+ .set({ haltedReason: null, haltedAt: null, circuitOpenUntil: EPOCH, circuitOpens: 0, updatedAt: new Date(Number(now())) })
233
+ .where(eq(seazonaApiGovernor.clientKey, key));
119
234
  }
120
235
 
121
236
  // Counts for /health: the last minute, and the last day by status.
@@ -126,11 +241,21 @@ export function createSeazonaGovernor({
126
241
  .where(and(eq(seazonaApiRequests.clientKey, key), gte(seazonaApiRequests.admittedAt, new Date(nowMs - WINDOW_MS))));
127
242
  const day = await summarise({ from: new Date(nowMs - 24 * 60 * 60_000), to: new Date(nowMs) });
128
243
  const cooldownUntil = state && state.cooldownUntil.getTime() > nowMs ? state.cooldownUntil.toISOString() : null;
244
+ const openUntil = state?.circuitOpenUntil?.getTime() ?? 0;
245
+ const circuit = state?.haltedReason ? 'halted'
246
+ : openUntil > nowMs ? 'open'
247
+ : state?.circuitOpens > 0 ? 'half_open' : 'closed';
129
248
  return {
130
249
  limitPerMinute: limit,
131
250
  minIntervalMs,
132
251
  lastMinute: Number(lastMinute),
133
252
  cooldownUntil,
253
+ circuit,
254
+ circuitOpenUntil: openUntil > nowMs ? state.circuitOpenUntil.toISOString() : null,
255
+ haltedReason: state?.haltedReason ?? null,
256
+ haltedAt: state?.haltedAt?.toISOString() ?? null,
257
+ failedToday: await failuresSince(db, startOfUtcDay(nowMs)),
258
+ dailyFailureBudget,
134
259
  lastDay: { requests: day.requests, failed: day.failed, rateLimited: day.rateLimited, retries: day.retries },
135
260
  };
136
261
  }
@@ -170,5 +295,5 @@ export function createSeazonaGovernor({
170
295
  };
171
296
  }
172
297
 
173
- return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, status, summarise };
298
+ return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, resume, status, summarise };
174
299
  }
@@ -2,6 +2,7 @@ import { createSeazonaClient } from './client.js';
2
2
  import { listClients, listWork, readBillableWork } from './feed.js';
3
3
  import { readSeazonaSyncHealth, runSeazonaSync } from './lanes.js';
4
4
  import { createSeazonaScheduler } from './scheduler.js';
5
+ import { resetFailedLanes, resetStoppedLanes } from './store.js';
5
6
  import { describeSeazonaSettings, resolveSeazonaSettings } from './settings.js';
6
7
  import { seazonaSchema } from './tables.js';
7
8
 
@@ -58,9 +59,21 @@ export function createSeazonaAdapter({ env = null, fetchImpl = null, now = Date.
58
59
  schema: seazonaSchema,
59
60
  setup: SEAZONA_SETUP,
60
61
 
61
- async start({ db, log = console, env: settingsEnv }) {
62
+ // `reconfigured` is true when a person saved the settings or turned
63
+ // Seazona on (issue #101): that clears a halt after a refused API client,
64
+ // closes the circuit, and runs every failed lane again. A plain start (a
65
+ // restart or a new release) runs again only the lanes stopped by a
66
+ // refused path; a halt still waits for a person.
67
+ async start({ db, log = console, env: settingsEnv, reconfigured = false }) {
62
68
  const settings = requireSeazonaSettings(env ?? settingsEnv ?? process.env);
63
69
  const client = createSeazonaClient({ settings, db, fetchImpl: fetchImpl ?? undefined, workflow: 'seazona-sync', now });
70
+ const at = new Date(Number(now()));
71
+ if (reconfigured) {
72
+ await client.governor?.resume();
73
+ await resetFailedLanes(db, { at });
74
+ } else {
75
+ await resetStoppedLanes(db, { at });
76
+ }
64
77
  const run = (options = {}) => runSeazonaSync(db, client, { settings, now, ...options });
65
78
  // The background sync starts in live mode only. A sandbox or replay
66
79
  // installation syncs only when someone asks (sync-once.js).