@arcforgelabs/flow 2026.10.29 → 2026.10.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/server/core/automations.js +32 -1
- package/server/core/pipeline.js +9 -3
- package/server/core/queue.js +22 -0
- package/server/core/sending.js +16 -4
- package/server/integrations/port.js +3 -1
- package/server/integrations/seazona/client.js +6 -0
- package/server/integrations/seazona/governor.js +138 -13
- package/server/integrations/seazona/index.js +14 -1
- package/server/integrations/seazona/lanes.js +65 -12
- package/server/integrations/seazona/migrations/0002_call_safety.sql +15 -0
- package/server/integrations/seazona/migrations/meta/0002_snapshot.json +882 -0
- package/server/integrations/seazona/migrations/meta/_journal.json +7 -0
- package/server/integrations/seazona/scheduler.js +37 -1
- package/server/integrations/seazona/schema.js +22 -0
- package/server/integrations/seazona/settings.js +10 -0
- package/server/integrations/seazona/store.js +74 -0
- package/server/integrations/seazona/tables.js +1 -0
- package/server/integrations/xero/import.js +1 -0
- package/server/ui/monitoring.js +14 -0
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arcforgelabs/flow",
|
|
3
|
-
"version": "2026.10.
|
|
3
|
+
"version": "2026.10.30",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@arcforgelabs/flow",
|
|
9
|
-
"version": "2026.10.
|
|
9
|
+
"version": "2026.10.30",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@clerk/backend": "3.23.2",
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { and, eq, sql } from 'drizzle-orm';
|
|
2
|
-
import { automations, billingObligations } from '../db/schema.js';
|
|
2
|
+
import { automations, billingObligations, invoices } from '../db/schema.js';
|
|
3
3
|
import { FlowError, notFound } from './errors.js';
|
|
4
4
|
import { autoDraftJobs } from './invoicing-rules.js';
|
|
5
5
|
import { MAPPING_DEFAULTS, readMappingConfig, remapJobs } from './line-mapping.js';
|
|
@@ -17,6 +17,8 @@ import { setReady } from './ledger.js';
|
|
|
17
17
|
// are picked from those the work system reported (workStatuses).
|
|
18
18
|
// draft_when_ready: a job that is ready to invoice and passes the invoicing
|
|
19
19
|
// rules (invoicing-rules.js) gets a draft invoice in Flow.
|
|
20
|
+
// ready_when_drafted: a draft the draft routine made goes straight to Ready
|
|
21
|
+
// on the Invoices page, until a person changes it or moves it back.
|
|
20
22
|
|
|
21
23
|
const BATCH = 500;
|
|
22
24
|
|
|
@@ -82,6 +84,14 @@ export const AUTOMATIONS = Object.freeze([
|
|
|
82
84
|
defaults: { enabled: true, config: {} },
|
|
83
85
|
readConfig: () => ({}),
|
|
84
86
|
},
|
|
87
|
+
{
|
|
88
|
+
key: 'ready_when_drafted',
|
|
89
|
+
place: 'invoicing',
|
|
90
|
+
name: 'Ready when drafted',
|
|
91
|
+
hint: 'An automatic draft goes straight to Ready. A draft a person changed stays in Drafts.',
|
|
92
|
+
defaults: { enabled: true, config: {} },
|
|
93
|
+
readConfig: () => ({}),
|
|
94
|
+
},
|
|
85
95
|
]);
|
|
86
96
|
|
|
87
97
|
const byKey = new Map(AUTOMATIONS.map((a) => [a.key, a]));
|
|
@@ -138,6 +148,25 @@ async function markReadyFromWorkStatus(db, organisationId, { statuses }) {
|
|
|
138
148
|
return { marked };
|
|
139
149
|
}
|
|
140
150
|
|
|
151
|
+
// Marks ready each draft the draft routine made that no person has touched.
|
|
152
|
+
// A person's edit or "back to drafts" moves the draft's updatedAt on from its
|
|
153
|
+
// createdAt, so the routine leaves it to people from then on. A draft with a
|
|
154
|
+
// problem shows in Problem whatever its ready mark (sending.js, invoiceViews).
|
|
155
|
+
async function readyWhenDrafted(db, organisationId) {
|
|
156
|
+
const rows = await db.update(invoices).set({ readyAt: sql`now()`, readyBy: null })
|
|
157
|
+
.where(and(
|
|
158
|
+
eq(invoices.organisationId, organisationId),
|
|
159
|
+
eq(invoices.status, 'draft'),
|
|
160
|
+
sql`${invoices.readyAt} is null`,
|
|
161
|
+
sql`${invoices.updatedAt} = ${invoices.createdAt}`,
|
|
162
|
+
sql`exists (select 1 from billing_obligations o join obligation_events e on e.obligation_id = o.id
|
|
163
|
+
where o.organisation_id = ${invoices.organisationId} and o.invoice_id = ${invoices.id}
|
|
164
|
+
and e.kind = 'drafted' and e.actor = ${actorOf('draft_when_ready')})`,
|
|
165
|
+
))
|
|
166
|
+
.returning({ id: invoices.id });
|
|
167
|
+
return { marked: rows.length };
|
|
168
|
+
}
|
|
169
|
+
|
|
141
170
|
// Runs every routine that is on. `accounting` is the integration whose copy
|
|
142
171
|
// the draft routine checks; without one, Flow's own records are the check.
|
|
143
172
|
export async function runAutomations(db, organisationId, { accounting = null, now = new Date() } = {}) {
|
|
@@ -148,6 +177,8 @@ export async function runAutomations(db, organisationId, { accounting = null, no
|
|
|
148
177
|
summary[automation.key] = await markReadyFromWorkStatus(db, organisationId, automation.config);
|
|
149
178
|
} else if (automation.key === 'draft_when_ready') {
|
|
150
179
|
summary[automation.key] = await autoDraftJobs(db, organisationId, { accounting, now, actor: actorOf(automation.key) });
|
|
180
|
+
} else if (automation.key === 'ready_when_drafted') {
|
|
181
|
+
summary[automation.key] = await readyWhenDrafted(db, organisationId);
|
|
151
182
|
}
|
|
152
183
|
}
|
|
153
184
|
return summary;
|
package/server/core/pipeline.js
CHANGED
|
@@ -9,7 +9,7 @@ import { readInvoicingSettings } from './invoicing-rules.js';
|
|
|
9
9
|
import { observeJob, recordAccountingEvidence, setHold } from './ledger.js';
|
|
10
10
|
import { loadMappingContext, mapLines } from './line-mapping.js';
|
|
11
11
|
import { linkRecord, priorLinkFor } from './links.js';
|
|
12
|
-
import { enqueueJob, runNextJob } from './queue.js';
|
|
12
|
+
import { deadHoldSince, deadSince, enqueueJob, runNextJob } from './queue.js';
|
|
13
13
|
|
|
14
14
|
// The pipeline (docs/DATA_SPINE.md, issue 58). It joins what the integrations
|
|
15
15
|
// copy into Flow, and never writes to another system:
|
|
@@ -448,6 +448,7 @@ export async function runPipelineOnce(db, integrations, { now = new Date() } = {
|
|
|
448
448
|
// systems' copies and writes Flow's own records.
|
|
449
449
|
export function createPipelineWorker({ db, integrations, log = console, timers = { setTimeout, clearTimeout }, everyMs = RUN_EVERY_MS, firstDelayMs = 15_000 }) {
|
|
450
450
|
const workerId = `pipeline-${randomUUID()}`;
|
|
451
|
+
const startedAtMs = Date.now();
|
|
451
452
|
let timer = null;
|
|
452
453
|
let running = null;
|
|
453
454
|
let stopped = true;
|
|
@@ -467,6 +468,9 @@ export function createPipelineWorker({ db, integrations, log = console, timers =
|
|
|
467
468
|
if (!integrations.with('pullWorkItems').length && !integrations.with('findInvoices').length) return;
|
|
468
469
|
const organisationId = await soleOrganisation(db);
|
|
469
470
|
if (!organisationId) return;
|
|
471
|
+
// Issue #101: a run that gave up waits for a person, a restart or the
|
|
472
|
+
// next day; the timer never starts it again by itself.
|
|
473
|
+
if (await deadSince(db, organisationId, { kind: PIPELINE_KIND, dedupeKey: 'all', since: deadHoldSince(startedAtMs) })) return;
|
|
470
474
|
await enqueueJob(db, organisationId, { kind: PIPELINE_KIND, dedupeKey: 'all' });
|
|
471
475
|
const ran = await runNextJob(db, {
|
|
472
476
|
workerId,
|
|
@@ -474,8 +478,10 @@ export function createPipelineWorker({ db, integrations, log = console, timers =
|
|
|
474
478
|
handlers: { [PIPELINE_KIND]: () => runPipelineOnce(db, integrations) },
|
|
475
479
|
});
|
|
476
480
|
const line = ran ? `${ran.outcome}: ${JSON.stringify(ran.job.result ?? ran.job.lastError ?? {})}` : '';
|
|
477
|
-
// Log a run only when its outcome differs from the last one.
|
|
478
|
-
|
|
481
|
+
// Log a run only when its outcome differs from the last one. A run
|
|
482
|
+
// that gave up is reported once to the error services.
|
|
483
|
+
if (ran?.outcome === 'dead') log.error?.('[pipeline] the pipeline run gave up after its last attempt');
|
|
484
|
+
if (line && line !== last) (ran.outcome === 'succeeded' ? log.info : log.warn)?.(`[pipeline] ${line}`);
|
|
479
485
|
last = line || last;
|
|
480
486
|
} catch (error) {
|
|
481
487
|
log.error?.(`[pipeline] ${error?.code ?? 'error'}: ${error?.message ?? error}`);
|
package/server/core/queue.js
CHANGED
|
@@ -38,6 +38,28 @@ export async function enqueueJob(db, organisationId, {
|
|
|
38
38
|
return { job: open, queued: false };
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
// Issue #101: the newest job of this kind and key that gave up after `since`,
|
|
42
|
+
// or null. A worker on a timer checks this before it queues the next run, so
|
|
43
|
+
// a job that always fails is not started again and again: it waits for a
|
|
44
|
+
// person (the review item), a restart on a new release, or the next day.
|
|
45
|
+
export async function deadSince(db, organisationId, { kind, dedupeKey = '', since }) {
|
|
46
|
+
const [row] = await db.select().from(backgroundJobs).where(and(
|
|
47
|
+
eq(backgroundJobs.organisationId, organisationId),
|
|
48
|
+
eq(backgroundJobs.kind, kind),
|
|
49
|
+
eq(backgroundJobs.dedupeKey, dedupeKey),
|
|
50
|
+
eq(backgroundJobs.state, 'dead'),
|
|
51
|
+
sql`${backgroundJobs.updatedAt} > ${since}`,
|
|
52
|
+
)).orderBy(sql`${backgroundJobs.updatedAt} desc`).limit(1);
|
|
53
|
+
return row ?? null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// When a timer may queue the job again after it gave up: the later of this
|
|
57
|
+
// process's start and one day ago.
|
|
58
|
+
export const DEAD_HOLD_MS = 24 * 60 * 60 * 1000;
|
|
59
|
+
export function deadHoldSince(startedAtMs, nowMs = Date.now()) {
|
|
60
|
+
return new Date(Math.max(startedAtMs, nowMs - DEAD_HOLD_MS));
|
|
61
|
+
}
|
|
62
|
+
|
|
41
63
|
async function giveUp(tx, job, error) {
|
|
42
64
|
const [row] = await tx.update(backgroundJobs)
|
|
43
65
|
.set({ state: 'dead', lockedBy: null, lockedUntil: null, lastError: error, updatedAt: sql`now()` })
|
package/server/core/sending.js
CHANGED
|
@@ -4,7 +4,7 @@ import { invoiceLines, invoices } from '../db/schema.js';
|
|
|
4
4
|
import { FlowError } from './errors.js';
|
|
5
5
|
import { issueInvoice } from './invoices.js';
|
|
6
6
|
import { soleOrganisation } from './pipeline.js';
|
|
7
|
-
import { enqueueJob, runNextJob } from './queue.js';
|
|
7
|
+
import { deadHoldSince, deadSince, enqueueJob, runNextJob } from './queue.js';
|
|
8
8
|
|
|
9
9
|
// Drafts, Problem, Ready and Send all ready (docs/INVOICE_SENDING.md).
|
|
10
10
|
//
|
|
@@ -124,9 +124,10 @@ export async function markReady(db, organisationId, invoiceIds, { personId, acco
|
|
|
124
124
|
return { marked: good.length, refused: [...problems].map(([invoiceId, found]) => ({ invoiceId, problems: found })) };
|
|
125
125
|
}
|
|
126
126
|
|
|
127
|
-
// Back to drafts: the drafts need another look before they go.
|
|
127
|
+
// Back to drafts: the drafts need another look before they go. The new
|
|
128
|
+
// updatedAt keeps "Ready when drafted" (automations.js) off them.
|
|
128
129
|
export async function markNotReady(db, organisationId, invoiceIds) {
|
|
129
|
-
const rows = await db.update(invoices).set({ readyAt: null, readyBy: null })
|
|
130
|
+
const rows = await db.update(invoices).set({ readyAt: null, readyBy: null, updatedAt: sql`now()` })
|
|
130
131
|
.where(and(eq(invoices.organisationId, organisationId), eq(invoices.status, 'draft'), inArray(invoices.id, invoiceIds)))
|
|
131
132
|
.returning({ id: invoices.id });
|
|
132
133
|
return { moved: rows.length };
|
|
@@ -215,6 +216,7 @@ export async function sendQueued(db, organisationId, { accounting, limit = WORKE
|
|
|
215
216
|
const error = result.error ?? { code: `send_${result.outcome}`, message: `Sending ended as ${result.outcome}.` };
|
|
216
217
|
await settle(db, organisationId, result.invoiceId, { sendState: 'failed', sendError: { code: error.code, message: error.message } });
|
|
217
218
|
summary.failed += 1;
|
|
219
|
+
summary.refused = [...new Set([...(summary.refused ?? []), error.code])];
|
|
218
220
|
}
|
|
219
221
|
}
|
|
220
222
|
}
|
|
@@ -230,6 +232,7 @@ function settle(db, organisationId, invoiceId, fields) {
|
|
|
230
232
|
// the job queue.
|
|
231
233
|
export function createSendWorker({ db, integrations, log = console, timers = { setTimeout, clearTimeout }, everyMs = WORKER_EVERY_MS }) {
|
|
232
234
|
const workerId = `send-${randomUUID()}`;
|
|
235
|
+
const startedAtMs = Date.now();
|
|
233
236
|
let timer = null;
|
|
234
237
|
let running = null;
|
|
235
238
|
let stopped = true;
|
|
@@ -252,13 +255,22 @@ export function createSendWorker({ db, integrations, log = console, timers = { s
|
|
|
252
255
|
const [waiting] = await db.select({ id: invoices.id }).from(invoices)
|
|
253
256
|
.where(and(eq(invoices.organisationId, organisationId), eq(invoices.sendState, 'queued'))).limit(1);
|
|
254
257
|
if (!waiting) return;
|
|
258
|
+
// Issue #101: a send run that gave up waits for a person, a restart or
|
|
259
|
+
// the next day; the timer never starts it again by itself.
|
|
260
|
+
if (await deadSince(db, organisationId, { kind: SEND_KIND, dedupeKey: 'all', since: deadHoldSince(startedAtMs) })) return;
|
|
255
261
|
await enqueueJob(db, organisationId, { kind: SEND_KIND, dedupeKey: 'all' });
|
|
256
262
|
const ran = await runNextJob(db, {
|
|
257
263
|
workerId,
|
|
258
264
|
leaseMs: LEASE_MS,
|
|
259
265
|
handlers: { [SEND_KIND]: () => sendQueued(db, organisationId, { accounting }) },
|
|
260
266
|
});
|
|
261
|
-
if (ran)
|
|
267
|
+
if (!ran) return;
|
|
268
|
+
const line = `[sending] ${ran.outcome}: ${JSON.stringify(ran.job.result ?? ran.job.lastError ?? {})}`;
|
|
269
|
+
// An invoice the accounting system refused waits in Problem for a
|
|
270
|
+
// person; it is reported once, when it is refused.
|
|
271
|
+
if (ran.outcome === 'dead') log.error?.('[sending] the send run gave up after its last attempt');
|
|
272
|
+
if (ran.job.result?.failed) log.error?.('[sending] the accounting system refused an invoice');
|
|
273
|
+
(ran.outcome === 'succeeded' && !ran.job.result?.failed ? log.info : log.warn)?.(line);
|
|
262
274
|
} catch (error) {
|
|
263
275
|
log.error?.(`[sending] ${error?.code ?? 'error'}: ${error?.message ?? error}`);
|
|
264
276
|
}
|
|
@@ -234,7 +234,9 @@ export function createRegistry({ log = console, env = process.env } = {}) {
|
|
|
234
234
|
tablesReady.add(id);
|
|
235
235
|
}
|
|
236
236
|
if (adapter.start) {
|
|
237
|
-
|
|
237
|
+
// A person changed the settings: an adapter may clear a stop that
|
|
238
|
+
// waits for a person (issue #101).
|
|
239
|
+
await adapter.start({ db: database, log, env: settingsEnv, reconfigured: true });
|
|
238
240
|
running.push(adapter);
|
|
239
241
|
}
|
|
240
242
|
} catch (error) {
|
|
@@ -17,6 +17,8 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, m
|
|
|
17
17
|
// A failed Seazona request. The message names the method, the path and the
|
|
18
18
|
// status or failure kind; never the host, the query or a credential.
|
|
19
19
|
// kind: http, timeout, network, invalid_json, replay_miss, governor_timeout.
|
|
20
|
+
// The governor's own SeazonaCallsStopped (kind calls_stopped) passes through
|
|
21
|
+
// unchanged: no call was made.
|
|
20
22
|
export class SeazonaApiError extends Error {
|
|
21
23
|
constructor(message, { statusCode = null, kind = 'http', path = null, retryAfterMs = null } = {}) {
|
|
22
24
|
super(message);
|
|
@@ -93,6 +95,10 @@ export function createSeazonaClient({
|
|
|
93
95
|
requestsPerMinute: rate.requestsPerMinute,
|
|
94
96
|
maxWaitMs: rate.maxWaitMs,
|
|
95
97
|
telemetryRetentionMs: rate.telemetryRetentionMs,
|
|
98
|
+
circuitFailures: rate.circuitFailures,
|
|
99
|
+
circuitWindowMs: rate.circuitWindowMs,
|
|
100
|
+
circuitPauseMs: rate.circuitPauseMs,
|
|
101
|
+
dailyFailureBudget: rate.dailyFailureBudget,
|
|
96
102
|
now,
|
|
97
103
|
sleepFn,
|
|
98
104
|
});
|
|
@@ -10,9 +10,39 @@ import { MAX_REQUESTS_PER_MINUTE } from './settings.js';
|
|
|
10
10
|
// admitted in the last minute, the minimum gap since the last admission has
|
|
11
11
|
// passed, and no 429 cooldown is running. Admission takes a transaction-level
|
|
12
12
|
// advisory lock per client, so two processes never admit the same slot.
|
|
13
|
+
//
|
|
14
|
+
// Issue #101: a vendor can cut off API access after many failed calls, even
|
|
15
|
+
// inside the rate limit. So the governor also stops calls outright:
|
|
16
|
+
// - a 401 or 403 halts every call until a person acts (the adapter restarts
|
|
17
|
+
// with new or saved settings, which calls resume());
|
|
18
|
+
// - `circuitFailures` failed calls inside `circuitWindowMs` open the circuit:
|
|
19
|
+
// no call for `circuitPauseMs`, then one probe call; a failed probe opens
|
|
20
|
+
// it again for twice as long, to at most 6 hours, and a success closes it;
|
|
21
|
+
// - `dailyFailureBudget` failed calls in one UTC day stop calls until the
|
|
22
|
+
// next day.
|
|
23
|
+
// A stopped call fails at once with SeazonaCallsStopped; it never waits.
|
|
13
24
|
|
|
14
25
|
const WINDOW_MS = 60_000;
|
|
15
26
|
const MAX_COOLDOWN_MS = 60_000;
|
|
27
|
+
const MAX_CIRCUIT_PAUSE_MS = 6 * 60 * 60_000;
|
|
28
|
+
const DAY_MS = 24 * 60 * 60_000;
|
|
29
|
+
const EPOCH = new Date(0);
|
|
30
|
+
|
|
31
|
+
// A call that failed in a way the vendor sees as a fault: no answer, a 5xx, a
|
|
32
|
+
// 429, or a 4xx other than 404 (a 404 on one record means it was deleted).
|
|
33
|
+
export function countsAsFailure({ statusCode = null, failureKind = null } = {}) {
|
|
34
|
+
if (failureKind) return true;
|
|
35
|
+
if (!Number.isInteger(statusCode)) return false;
|
|
36
|
+
return statusCode >= 500 || (statusCode >= 400 && statusCode !== 404);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const failedCall = sql`(${seazonaApiRequests.failureKind} is not null or ${seazonaApiRequests.statusCode} >= 500
|
|
40
|
+
or (${seazonaApiRequests.statusCode} >= 400 and ${seazonaApiRequests.statusCode} <> 404))`;
|
|
41
|
+
|
|
42
|
+
const startOfUtcDay = (ms) => {
|
|
43
|
+
const date = new Date(ms);
|
|
44
|
+
return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
|
|
45
|
+
};
|
|
16
46
|
|
|
17
47
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
|
18
48
|
|
|
@@ -36,12 +66,34 @@ export class SeazonaGovernorTimeout extends Error {
|
|
|
36
66
|
}
|
|
37
67
|
}
|
|
38
68
|
|
|
69
|
+
// Calls are stopped: halted for a person, circuit open, or the day's failed
|
|
70
|
+
// calls are spent. `retryAfterMs` is null for a halt, which no wait clears.
|
|
71
|
+
export class SeazonaCallsStopped extends Error {
|
|
72
|
+
constructor(reason, retryAfterMs = null) {
|
|
73
|
+
const what = {
|
|
74
|
+
halted: 'Seazona calls are stopped until a person checks the API client (it was refused)',
|
|
75
|
+
circuit_open: 'Seazona calls are paused after repeated failures',
|
|
76
|
+
budget_spent: "Seazona calls are stopped for today: the day's failed calls are spent",
|
|
77
|
+
}[reason];
|
|
78
|
+
super(what);
|
|
79
|
+
this.name = 'SeazonaCallsStopped';
|
|
80
|
+
this.code = { halted: 'seazona_api_halted', circuit_open: 'seazona_api_circuit_open', budget_spent: 'seazona_api_failure_budget_spent' }[reason];
|
|
81
|
+
this.kind = 'calls_stopped';
|
|
82
|
+
this.reason = reason;
|
|
83
|
+
this.retryAfterMs = retryAfterMs;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
39
87
|
export function createSeazonaGovernor({
|
|
40
88
|
db,
|
|
41
89
|
clientKey,
|
|
42
90
|
requestsPerMinute = MAX_REQUESTS_PER_MINUTE,
|
|
43
91
|
maxWaitMs = 120_000,
|
|
44
92
|
telemetryRetentionMs = 35 * 24 * 60 * 60_000,
|
|
93
|
+
circuitFailures = 5,
|
|
94
|
+
circuitWindowMs = 10 * 60_000,
|
|
95
|
+
circuitPauseMs = 5 * 60_000,
|
|
96
|
+
dailyFailureBudget = 50,
|
|
45
97
|
now = Date.now,
|
|
46
98
|
sleepFn = sleep,
|
|
47
99
|
} = {}) {
|
|
@@ -59,6 +111,8 @@ export function createSeazonaGovernor({
|
|
|
59
111
|
const nowAt = new Date(nowMs);
|
|
60
112
|
await tx.insert(seazonaApiGovernor).values({ clientKey: key }).onConflictDoNothing();
|
|
61
113
|
const [state] = await tx.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
|
|
114
|
+
const stopped = await stoppedReason(tx, state, nowMs);
|
|
115
|
+
if (stopped) return { admitted: false, stopped };
|
|
62
116
|
const windowStart = new Date(nowMs - WINDOW_MS);
|
|
63
117
|
const [{ admitted, oldestMs }] = await tx
|
|
64
118
|
.select({ admitted: count(), oldestMs: sql`(extract(epoch from min(${seazonaApiRequests.admittedAt})) * 1000)::float8` })
|
|
@@ -84,12 +138,31 @@ export function createSeazonaGovernor({
|
|
|
84
138
|
});
|
|
85
139
|
}
|
|
86
140
|
|
|
141
|
+
async function failuresSince(tx, fromMs) {
|
|
142
|
+
const [{ failed }] = await tx.select({ failed: count() }).from(seazonaApiRequests)
|
|
143
|
+
.where(and(eq(seazonaApiRequests.clientKey, key), gte(seazonaApiRequests.admittedAt, new Date(fromMs)), failedCall));
|
|
144
|
+
return Number(failed);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Why no call may go now, or null.
|
|
148
|
+
async function stoppedReason(tx, state, nowMs) {
|
|
149
|
+
if (state.haltedReason) return { reason: 'halted', retryAfterMs: null };
|
|
150
|
+
const openUntil = state.circuitOpenUntil.getTime();
|
|
151
|
+
if (openUntil > nowMs) return { reason: 'circuit_open', retryAfterMs: openUntil - nowMs };
|
|
152
|
+
const dayStart = startOfUtcDay(nowMs);
|
|
153
|
+
if (await failuresSince(tx, dayStart) >= dailyFailureBudget) {
|
|
154
|
+
return { reason: 'budget_spent', retryAfterMs: dayStart + DAY_MS - nowMs };
|
|
155
|
+
}
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
|
|
87
159
|
// Waits for a slot and records the admission. Returns { requestId }.
|
|
88
160
|
async function reserve({ workflow = 'seazona', isRetry = false } = {}) {
|
|
89
161
|
let waitedMs = 0;
|
|
90
162
|
for (;;) {
|
|
91
163
|
const decision = await tryAdmit(workflow, isRetry);
|
|
92
164
|
if (decision.admitted) return { requestId: decision.requestId, waitedMs };
|
|
165
|
+
if (decision.stopped) throw new SeazonaCallsStopped(decision.stopped.reason, decision.stopped.retryAfterMs);
|
|
93
166
|
const delay = Math.max(1, Math.min(decision.waitMs, maxWaitMs));
|
|
94
167
|
if (waitedMs + delay > maxWaitMs) throw new SeazonaGovernorTimeout(waitedMs, delay);
|
|
95
168
|
waitedMs += delay;
|
|
@@ -97,25 +170,67 @@ export function createSeazonaGovernor({
|
|
|
97
170
|
}
|
|
98
171
|
}
|
|
99
172
|
|
|
100
|
-
// Records how a request ended,
|
|
173
|
+
// Records how a request ended, any cooldown a 429 asked for, and what the
|
|
174
|
+
// ending does to the circuit. Answers { halted, opened } when this response
|
|
175
|
+
// stopped calls, so the caller can report it once.
|
|
101
176
|
async function recordResponse({ requestId, path = null, statusCode = null, failureKind = null, durationMs = null, retryAfterMs = null } = {}) {
|
|
102
177
|
const nowMs = Number(now());
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
178
|
+
const nowAt = new Date(nowMs);
|
|
179
|
+
const outcome = await db.transaction(async (tx) => {
|
|
180
|
+
await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${lockKey}))`);
|
|
181
|
+
await tx.update(seazonaApiRequests).set({
|
|
182
|
+
requestPath: normaliseRequestPath(path),
|
|
183
|
+
statusCode: Number.isInteger(statusCode) ? statusCode : null,
|
|
184
|
+
failureKind: failureKind ? String(failureKind).slice(0, 40) : null,
|
|
185
|
+
durationMs: Number.isFinite(durationMs) ? Math.max(0, Math.round(durationMs)) : null,
|
|
186
|
+
}).where(and(eq(seazonaApiRequests.id, requestId), eq(seazonaApiRequests.clientKey, key)));
|
|
187
|
+
if (Number.isFinite(retryAfterMs) && retryAfterMs > 0) {
|
|
188
|
+
const until = new Date(nowMs + Math.min(retryAfterMs, MAX_COOLDOWN_MS));
|
|
189
|
+
await tx.update(seazonaApiGovernor)
|
|
190
|
+
.set({ cooldownUntil: sql`greatest(${seazonaApiGovernor.cooldownUntil}, ${until})`, updatedAt: nowAt })
|
|
191
|
+
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
192
|
+
}
|
|
193
|
+
const [state] = await tx.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
|
|
194
|
+
if (!state) return {};
|
|
195
|
+
if (!countsAsFailure({ statusCode, failureKind })) {
|
|
196
|
+
if (state.circuitOpens > 0) {
|
|
197
|
+
await tx.update(seazonaApiGovernor).set({ circuitOpens: 0, updatedAt: nowAt }).where(eq(seazonaApiGovernor.clientKey, key));
|
|
198
|
+
}
|
|
199
|
+
return {};
|
|
200
|
+
}
|
|
201
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
202
|
+
if (state.haltedReason) return {};
|
|
203
|
+
await tx.update(seazonaApiGovernor).set({ haltedReason: 'auth', haltedAt: nowAt, updatedAt: nowAt })
|
|
204
|
+
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
205
|
+
return { halted: 'auth' };
|
|
206
|
+
}
|
|
207
|
+
const openUntil = state.circuitOpenUntil.getTime();
|
|
208
|
+
// After a pause the next call is a probe: if it fails, open again at once.
|
|
209
|
+
const probeFailed = state.circuitOpens > 0 && openUntil <= nowMs;
|
|
210
|
+
const counted = probeFailed ? 0 : await failuresSince(tx, Math.max(nowMs - circuitWindowMs, openUntil));
|
|
211
|
+
if (!probeFailed && counted < circuitFailures) return {};
|
|
212
|
+
if (!probeFailed && openUntil > nowMs) return {};
|
|
213
|
+
const pauseMs = Math.min(MAX_CIRCUIT_PAUSE_MS, circuitPauseMs * 2 ** Math.min(state.circuitOpens, 10));
|
|
214
|
+
const until = new Date(nowMs + pauseMs);
|
|
215
|
+
await tx.update(seazonaApiGovernor)
|
|
216
|
+
.set({ circuitOpenUntil: until, circuitOpens: state.circuitOpens + 1, updatedAt: nowAt })
|
|
113
217
|
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
114
|
-
|
|
218
|
+
return { opened: { until: until.toISOString(), pauseMs, opens: state.circuitOpens + 1 } };
|
|
219
|
+
});
|
|
115
220
|
await db.delete(seazonaApiRequests).where(and(
|
|
116
221
|
eq(seazonaApiRequests.clientKey, key),
|
|
117
222
|
lt(seazonaApiRequests.admittedAt, new Date(nowMs - telemetryRetentionMs)),
|
|
118
223
|
));
|
|
224
|
+
return outcome;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// A person acted (new or saved settings, or the adapter turned on again):
|
|
228
|
+
// clear a halt and close the circuit. The day's failure budget stays.
|
|
229
|
+
async function resume() {
|
|
230
|
+
await db.insert(seazonaApiGovernor).values({ clientKey: key }).onConflictDoNothing();
|
|
231
|
+
await db.update(seazonaApiGovernor)
|
|
232
|
+
.set({ haltedReason: null, haltedAt: null, circuitOpenUntil: EPOCH, circuitOpens: 0, updatedAt: new Date(Number(now())) })
|
|
233
|
+
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
119
234
|
}
|
|
120
235
|
|
|
121
236
|
// Counts for /health: the last minute, and the last day by status.
|
|
@@ -126,11 +241,21 @@ export function createSeazonaGovernor({
|
|
|
126
241
|
.where(and(eq(seazonaApiRequests.clientKey, key), gte(seazonaApiRequests.admittedAt, new Date(nowMs - WINDOW_MS))));
|
|
127
242
|
const day = await summarise({ from: new Date(nowMs - 24 * 60 * 60_000), to: new Date(nowMs) });
|
|
128
243
|
const cooldownUntil = state && state.cooldownUntil.getTime() > nowMs ? state.cooldownUntil.toISOString() : null;
|
|
244
|
+
const openUntil = state?.circuitOpenUntil?.getTime() ?? 0;
|
|
245
|
+
const circuit = state?.haltedReason ? 'halted'
|
|
246
|
+
: openUntil > nowMs ? 'open'
|
|
247
|
+
: state?.circuitOpens > 0 ? 'half_open' : 'closed';
|
|
129
248
|
return {
|
|
130
249
|
limitPerMinute: limit,
|
|
131
250
|
minIntervalMs,
|
|
132
251
|
lastMinute: Number(lastMinute),
|
|
133
252
|
cooldownUntil,
|
|
253
|
+
circuit,
|
|
254
|
+
circuitOpenUntil: openUntil > nowMs ? state.circuitOpenUntil.toISOString() : null,
|
|
255
|
+
haltedReason: state?.haltedReason ?? null,
|
|
256
|
+
haltedAt: state?.haltedAt?.toISOString() ?? null,
|
|
257
|
+
failedToday: await failuresSince(db, startOfUtcDay(nowMs)),
|
|
258
|
+
dailyFailureBudget,
|
|
134
259
|
lastDay: { requests: day.requests, failed: day.failed, rateLimited: day.rateLimited, retries: day.retries },
|
|
135
260
|
};
|
|
136
261
|
}
|
|
@@ -170,5 +295,5 @@ export function createSeazonaGovernor({
|
|
|
170
295
|
};
|
|
171
296
|
}
|
|
172
297
|
|
|
173
|
-
return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, status, summarise };
|
|
298
|
+
return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, resume, status, summarise };
|
|
174
299
|
}
|
|
@@ -2,6 +2,7 @@ import { createSeazonaClient } from './client.js';
|
|
|
2
2
|
import { listClients, listWork, readBillableWork } from './feed.js';
|
|
3
3
|
import { readSeazonaSyncHealth, runSeazonaSync } from './lanes.js';
|
|
4
4
|
import { createSeazonaScheduler } from './scheduler.js';
|
|
5
|
+
import { resetFailedLanes, resetStoppedLanes } from './store.js';
|
|
5
6
|
import { describeSeazonaSettings, resolveSeazonaSettings } from './settings.js';
|
|
6
7
|
import { seazonaSchema } from './tables.js';
|
|
7
8
|
|
|
@@ -58,9 +59,21 @@ export function createSeazonaAdapter({ env = null, fetchImpl = null, now = Date.
|
|
|
58
59
|
schema: seazonaSchema,
|
|
59
60
|
setup: SEAZONA_SETUP,
|
|
60
61
|
|
|
61
|
-
|
|
62
|
+
// `reconfigured` is true when a person saved the settings or turned
|
|
63
|
+
// Seazona on (issue #101): that clears a halt after a refused API client,
|
|
64
|
+
// closes the circuit, and runs every failed lane again. A plain start (a
|
|
65
|
+
// restart or a new release) runs again only the lanes stopped by a
|
|
66
|
+
// refused path; a halt still waits for a person.
|
|
67
|
+
async start({ db, log = console, env: settingsEnv, reconfigured = false }) {
|
|
62
68
|
const settings = requireSeazonaSettings(env ?? settingsEnv ?? process.env);
|
|
63
69
|
const client = createSeazonaClient({ settings, db, fetchImpl: fetchImpl ?? undefined, workflow: 'seazona-sync', now });
|
|
70
|
+
const at = new Date(Number(now()));
|
|
71
|
+
if (reconfigured) {
|
|
72
|
+
await client.governor?.resume();
|
|
73
|
+
await resetFailedLanes(db, { at });
|
|
74
|
+
} else {
|
|
75
|
+
await resetStoppedLanes(db, { at });
|
|
76
|
+
}
|
|
64
77
|
const run = (options = {}) => runSeazonaSync(db, client, { settings, now, ...options });
|
|
65
78
|
// The background sync starts in live mode only. A sandbox or replay
|
|
66
79
|
// installation syncs only when someone asks (sync-once.js).
|