@arcforgelabs/flow 2026.10.29 → 2026.10.32-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +2 -0
  2. package/npm-shrinkwrap.json +188 -2
  3. package/package.json +2 -1
  4. package/server/api/alerts.js +49 -0
  5. package/server/api/invoicing.js +110 -3
  6. package/server/app.js +29 -2
  7. package/server/core/alerts.js +157 -0
  8. package/server/core/automations.js +32 -1
  9. package/server/core/history.js +65 -0
  10. package/server/core/invoice-pdf.js +266 -0
  11. package/server/core/invoicing-rules.js +12 -0
  12. package/server/core/job-board.js +25 -3
  13. package/server/core/line-mapping.js +36 -10
  14. package/server/core/pipeline.js +84 -6
  15. package/server/core/price-list.js +348 -0
  16. package/server/core/queue.js +22 -0
  17. package/server/core/sending.js +16 -4
  18. package/server/db/commands.js +42 -0
  19. package/server/db/doctor.js +168 -0
  20. package/server/db/migrations/0014_price_list.sql +34 -0
  21. package/server/db/migrations/0015_alert_dismissals.sql +17 -0
  22. package/server/db/migrations/meta/0014_snapshot.json +3773 -0
  23. package/server/db/migrations/meta/0015_snapshot.json +3854 -0
  24. package/server/db/migrations/meta/_journal.json +14 -0
  25. package/server/db/schema.js +44 -0
  26. package/server/flow-server.js +20 -1
  27. package/server/index.js +2 -1
  28. package/server/integrations/call-guard.js +362 -0
  29. package/server/integrations/port.js +23 -2
  30. package/server/integrations/seazona/client.js +6 -0
  31. package/server/integrations/seazona/feed.js +23 -1
  32. package/server/integrations/seazona/governor.js +74 -21
  33. package/server/integrations/seazona/history/backfill.js +208 -0
  34. package/server/integrations/seazona/history/csv.js +65 -0
  35. package/server/integrations/seazona/history/index.js +584 -0
  36. package/server/integrations/seazona/history/read.js +196 -0
  37. package/server/integrations/seazona/index.js +51 -2
  38. package/server/integrations/seazona/lanes.js +251 -62
  39. package/server/integrations/seazona/migrations/0002_call_safety.sql +15 -0
  40. package/server/integrations/seazona/migrations/0003_call_guard.sql +2 -0
  41. package/server/integrations/seazona/migrations/0004_history_import.sql +68 -0
  42. package/server/integrations/seazona/migrations/meta/0002_snapshot.json +882 -0
  43. package/server/integrations/seazona/migrations/meta/0003_snapshot.json +895 -0
  44. package/server/integrations/seazona/migrations/meta/0004_snapshot.json +1323 -0
  45. package/server/integrations/seazona/migrations/meta/_journal.json +21 -0
  46. package/server/integrations/seazona/routes.js +145 -0
  47. package/server/integrations/seazona/scheduler.js +37 -1
  48. package/server/integrations/seazona/schema.js +116 -0
  49. package/server/integrations/seazona/settings.js +24 -3
  50. package/server/integrations/seazona/store.js +74 -0
  51. package/server/integrations/seazona/sync-once.js +1 -1
  52. package/server/integrations/seazona/tables.js +5 -0
  53. package/server/integrations/xero/client.js +27 -7
  54. package/server/integrations/xero/drafts.js +46 -8
  55. package/server/integrations/xero/import.js +125 -16
  56. package/server/integrations/xero/index.js +36 -7
  57. package/server/integrations/xero/lookup.js +26 -0
  58. package/server/integrations/xero/migrations/0006_xero_call_guard.sql +26 -0
  59. package/server/integrations/xero/migrations/meta/_journal.json +7 -0
  60. package/server/integrations/xero/mirror.js +11 -2
  61. package/server/integrations/xero/oauth.js +4 -1
  62. package/server/integrations/xero/reconcile.js +173 -0
  63. package/server/integrations/xero/routes.js +35 -0
  64. package/server/integrations/xero/runtime.js +45 -4
  65. package/server/release-channel.js +3 -3
  66. package/server/ui/alerts.js +77 -0
  67. package/server/ui/app-page.js +1 -1
  68. package/server/ui/app.js +48 -14
  69. package/server/ui/clerk-appearance.js +42 -9
  70. package/server/ui/flow.css +31 -2
  71. package/server/ui/invoicing-settings.js +1 -1
  72. package/server/ui/invoicing.js +121 -8
  73. package/server/ui/line-mappings.js +3 -2
  74. package/server/ui/monitoring.js +14 -0
  75. package/server/ui/price-list.js +247 -0
  76. package/server/ui/routes.js +1 -1
@@ -15,6 +15,27 @@
15
15
  "when": 1792330268730,
16
16
  "tag": "0001_mirror",
17
17
  "breakpoints": true
18
+ },
19
+ {
20
+ "idx": 2,
21
+ "version": "7",
22
+ "when": 1792448378477,
23
+ "tag": "0002_call_safety",
24
+ "breakpoints": true
25
+ },
26
+ {
27
+ "idx": 3,
28
+ "version": "7",
29
+ "when": 1792452091297,
30
+ "tag": "0003_call_guard",
31
+ "breakpoints": true
32
+ },
33
+ {
34
+ "idx": 4,
35
+ "version": "7",
36
+ "when": 1792453111948,
37
+ "tag": "0004_history_import",
38
+ "breakpoints": true
18
39
  }
19
40
  ]
20
41
  }
@@ -0,0 +1,145 @@
1
+ import express from 'express';
2
+ import { FlowError } from '../../core/errors.js';
3
+ import { listHistoryJobs, releaseHistoryJob } from '../../core/history.js';
4
+ import { soleOrganisation } from '../../core/pipeline.js';
5
+ import {
6
+ applyBatch,
7
+ cancelBatch,
8
+ decideJobs,
9
+ describeColumns,
10
+ historySummary,
11
+ listBatches,
12
+ readBatch,
13
+ reconcileBatch,
14
+ stageBatch,
15
+ } from './history/index.js';
16
+ import { readBackfill, startBackfill, stopBackfill } from './history/backfill.js';
17
+
18
+ // The Seazona adapter's routes, mounted at /api/seazona (server/integrations/port.js).
19
+ // The history import (docs/SEAZONA.md, "History import"):
20
+ //
21
+ // GET /history any signed-in person: totals and batches
22
+ // GET /history/columns any signed-in person: the headers Flow reads
23
+ // POST /history/batches owner or admin: upload a CSV export
24
+ // (text/csv body, ?fileName=; or JSON
25
+ // { csv, fileName, columns })
26
+ // GET /history/batches/:id?outcome=&limit=&offset= any signed-in person
27
+ // POST /history/batches/:id/reconcile owner or admin: match again
28
+ // POST /history/batches/:id/decisions owner or admin: { decisions: [{ jobKey, decision, clientId? }] }
29
+ // POST /history/batches/:id/apply owner or admin: write the batch into Flow's history
30
+ // POST /history/batches/:id/cancel owner or admin
31
+ // POST /history/backfill owner or admin: { from, through? } reads
32
+ // past orders from the Seazona API, a few a run
33
+ // POST /history/backfill/stop owner or admin
34
+ // GET /history/ledger?state=&limit=&offset= any signed-in person: held
35
+ // history jobs by accounting state
36
+ // (state: one state, or needs_person)
37
+ // POST /history/ledger/:obligationId/release owner or admin: the job may draft
38
+ //
39
+ // None of these call Seazona.
40
+
41
+ export const MAX_UPLOAD = '25mb';
42
+
43
+ function sendError(res, error, log) {
44
+ const status = error instanceof FlowError ? (error.status ?? 400) : 500;
45
+ if (status >= 500) log.error?.(`[seazona] history import: ${error.code ?? 'error'}: ${error.message}`);
46
+ const body = { ok: false, code: error.code ?? 'seazona_failed', error: status === 500 ? 'Something went wrong with the import.' : error.message };
47
+ if (error.details !== undefined) body.details = error.details;
48
+ return res.status(status).json(body);
49
+ }
50
+
51
+ const handle = (log, fn) => async (req, res) => {
52
+ try {
53
+ await fn(req, res);
54
+ } catch (error) {
55
+ sendError(res, error, log);
56
+ }
57
+ };
58
+
59
+ export function createSeazonaRouter({ db, personAuth, settings, log = console }) {
60
+ const router = express.Router();
61
+ // Sign-in off: every route answers 503, as the Xero routes do.
62
+ const signedIn = personAuth
63
+ ? [personAuth.requireSession]
64
+ : [(_req, res) => res.status(503).json({ ok: false, code: 'sign_in_not_configured', error: 'Sign-in is not configured on this installation (FLOW_AUTH is unset).' })];
65
+ const manager = personAuth ? [...signedIn, personAuth.requireRole('owner', 'admin')] : signedIn;
66
+ const actor = (req) => (req.person?.id ? `person:${req.person.id}` : 'person:unknown');
67
+ const json = express.json({ limit: MAX_UPLOAD });
68
+ const csv = express.text({ type: ['text/csv', 'text/plain', 'application/csv'], limit: MAX_UPLOAD });
69
+
70
+ if (!db) {
71
+ router.use((req, res) => res.status(503).json({ ok: false, code: 'no_database', error: 'Flow has no database' }));
72
+ return router;
73
+ }
74
+
75
+ router.get('/history', ...signedIn, handle(log, async (req, res) => {
76
+ res.json({ ok: true, history: await historySummary(db), backfill: await readBackfill(db), batches: await listBatches(db) });
77
+ }));
78
+
79
+ router.get('/history/columns', ...signedIn, (req, res) => res.json({ ok: true, profile: 'invoice_lines', columns: describeColumns() }));
80
+
81
+ router.post('/history/batches', ...manager, csv, json, handle(log, async (req, res) => {
82
+ const body = typeof req.body === 'string' ? { csv: req.body, fileName: req.query.fileName } : (req.body ?? {});
83
+ const batch = await stageBatch(db, {
84
+ csv: body.csv,
85
+ fileName: body.fileName ?? null,
86
+ columns: body.columns ?? {},
87
+ createdBy: actor(req),
88
+ keepPatientReference: Boolean(settings?.keepPatientReference),
89
+ });
90
+ log.info?.(`[seazona] history import: batch ${batch.id} ${batch.duplicate ? 'already staged' : 'staged'} by ${actor(req)}: ${JSON.stringify(batch.counts)}`);
91
+ res.status(batch.duplicate ? 200 : 201).json({ ok: true, batch });
92
+ }));
93
+
94
+ router.get('/history/batches/:id', ...signedIn, handle(log, async (req, res) => {
95
+ const { outcome = null, limit, offset } = req.query;
96
+ res.json({ ok: true, batch: await readBatch(db, req.params.id, { outcome: outcome || null, limit, offset }) });
97
+ }));
98
+
99
+ router.post('/history/batches/:id/reconcile', ...manager, handle(log, async (req, res) => {
100
+ res.json({ ok: true, batch: await reconcileBatch(db, req.params.id) });
101
+ }));
102
+
103
+ router.post('/history/batches/:id/decisions', ...manager, json, handle(log, async (req, res) => {
104
+ res.json({ ok: true, batch: await decideJobs(db, req.params.id, req.body?.decisions, { by: actor(req) }) });
105
+ }));
106
+
107
+ router.post('/history/batches/:id/apply', ...manager, handle(log, async (req, res) => {
108
+ const batch = await applyBatch(db, req.params.id, { by: actor(req) });
109
+ log.info?.(`[seazona] history import: batch ${batch.id} applied by ${actor(req)}: ${batch.applied} new, ${batch.updated} updated`);
110
+ res.json({ ok: true, batch });
111
+ }));
112
+
113
+ router.post('/history/batches/:id/cancel', ...manager, handle(log, async (req, res) => {
114
+ res.json({ ok: true, batch: await cancelBatch(db, req.params.id) });
115
+ }));
116
+
117
+ const organisation = async () => {
118
+ const id = await soleOrganisation(db);
119
+ if (!id) throw Object.assign(new FlowError('invalid', 'the installation needs exactly one organisation'), { status: 409 });
120
+ return id;
121
+ };
122
+
123
+ router.post('/history/backfill', ...manager, json, handle(log, async (req, res) => {
124
+ const backfill = await startBackfill(db, { from: req.body?.from, through: req.body?.through ?? null, createdBy: actor(req), settings });
125
+ log.info?.(`[seazona] history backfill from ${backfill.from} to ${backfill.through} started by ${actor(req)}`);
126
+ res.status(201).json({ ok: true, backfill });
127
+ }));
128
+
129
+ router.post('/history/backfill/stop', ...manager, handle(log, async (req, res) => {
130
+ res.json({ ok: true, backfill: await stopBackfill(db) });
131
+ }));
132
+
133
+ router.get('/history/ledger', ...signedIn, handle(log, async (req, res) => {
134
+ const { state = null, limit, offset } = req.query;
135
+ res.json({ ok: true, ...(await listHistoryJobs(db, await organisation(), { state: state || null, limit, offset })) });
136
+ }));
137
+
138
+ router.post('/history/ledger/:obligationId/release', ...manager, handle(log, async (req, res) => {
139
+ const row = await releaseHistoryJob(db, await organisation(), req.params.obligationId, { actor: actor(req) });
140
+ log.info?.(`[seazona] history import: job ${row.sourceJobRef} released by ${actor(req)}`);
141
+ res.json({ ok: true, job: { id: row.id, holdState: row.holdState } });
142
+ }));
143
+
144
+ return router;
145
+ }
@@ -4,12 +4,20 @@ import { readLaneStates } from './store.js';
4
4
  // The background sync, for live mode only (issue #10: index.js never creates
5
5
  // it in sandbox or replay). It runs the lanes that are due, then sleeps until
6
6
  // the next lane is due, between 15 seconds and 30 minutes. One run at a time.
7
+ // A stopped lane (issue #101) is not due until a person or a release resets it.
8
+ //
9
+ // Issue #101: each held failure is reported once to the error services
10
+ // (log.error, server/core/monitoring.js): when a lane starts failing or its
11
+ // failure code changes, and when records are quarantined. A repeat of the
12
+ // same failure is a warning in the host log only. The report text is stable,
13
+ // so the error service groups repeats.
7
14
 
8
15
  const MIN_DELAY_MS = 15_000;
9
16
  const MAX_DELAY_MS = 30 * 60_000;
10
17
 
11
18
  export function createSeazonaScheduler({
12
19
  db,
20
+ settings = null,
13
21
  run,
14
22
  now = Date.now,
15
23
  log = console,
@@ -21,11 +29,39 @@ export function createSeazonaScheduler({
21
29
  let stopped = false;
22
30
  let nextRunAt = null;
23
31
  let last = null;
32
+ // lane -> the failure code last reported, while that lane keeps failing.
33
+ const reported = new Map();
34
+ const quarantineAt = settings?.sync?.orderQuarantineFailures ?? null;
35
+
36
+ function newlyQuarantined(outcome) {
37
+ if (Number.isInteger(outcome.newlyQuarantined)) return outcome.newlyQuarantined;
38
+ const failures = outcome.failureDetails?.failures ?? [];
39
+ return failures.filter((f) => f.quarantinedUntil && f.failureCount === quarantineAt).length;
40
+ }
41
+
42
+ function report(results) {
43
+ for (const [lane, outcome] of Object.entries(results)) {
44
+ if (outcome.ok) {
45
+ reported.delete(lane);
46
+ const count = newlyQuarantined(outcome);
47
+ if (count) log.error?.(`[seazona] ${lane}: records quarantined after repeated failed reads`);
48
+ continue;
49
+ }
50
+ if (reported.get(lane) === outcome.code) {
51
+ log.warn?.(`[seazona] ${lane}: ${outcome.error}`);
52
+ continue;
53
+ }
54
+ reported.set(lane, outcome.code);
55
+ log.error?.(`[seazona] ${lane} lane ${outcome.status === 'stopped' ? 'stopped' : 'failed'}: ${outcome.code}`);
56
+ log.warn?.(`[seazona] ${lane}: ${outcome.error}`);
57
+ }
58
+ }
24
59
 
25
60
  async function nextDelayMs() {
26
61
  const states = await readLaneStates(db);
27
62
  let soonest = Infinity;
28
63
  for (const lane of LANES) {
64
+ if (states.get(lane)?.status === 'stopped') continue;
29
65
  const at = states.get(lane)?.nextRunAt?.getTime();
30
66
  soonest = Math.min(soonest, Number.isFinite(at) ? at - Number(now()) : 0);
31
67
  }
@@ -48,7 +84,7 @@ export function createSeazonaScheduler({
48
84
  const result = await run();
49
85
  const failed = Object.entries(result.results).filter(([, lane]) => !lane.ok);
50
86
  last = { at: startedAt, ok: result.ok, lanes: Object.keys(result.results), failed: failed.map(([lane]) => lane) };
51
- for (const [lane, outcome] of failed) log.warn?.(`[seazona] ${lane}: ${outcome.error}`);
87
+ report(result.results);
52
88
  } catch (error) {
53
89
  last = { at: startedAt, ok: false, error: error?.message || String(error) };
54
90
  log.error?.(`[seazona] sync run failed: ${last.error}`);
@@ -10,10 +10,20 @@ const at = (name) => timestamp(name, { withTimezone: true });
10
10
 
11
11
  // One row per API client: when the next request may go, and any cooldown a
12
12
  // 429 asked for. Every process that shares the database shares this row.
13
+ // Issue #101: the circuit breaker (open until, and how many times in a row it
14
+ // opened) and a stop that waits for a person (`halted_reason`, for example
15
+ // `auth` after a 401).
13
16
  export const seazonaApiGovernor = pgTable('seazona_api_governor', {
14
17
  clientKey: text('client_key').primaryKey(),
15
18
  nextEligibleAt: at('next_eligible_at').notNull().default(sql`'epoch'`),
16
19
  cooldownUntil: at('cooldown_until').notNull().default(sql`'epoch'`),
20
+ circuitOpenUntil: at('circuit_open_until').notNull().default(sql`'epoch'`),
21
+ circuitOpens: integer('circuit_opens').notNull().default(0),
22
+ haltedReason: text('halted_reason'),
23
+ haltedAt: at('halted_at'),
24
+ // The call guard's stopped resources (../call-guard.js). Seazona's guard
25
+ // halts on any 401 or 403, so this stays empty; the shared guard needs it.
26
+ stoppedResources: jsonb('stopped_resources').notNull().default({}),
17
27
  updatedAt: at('updated_at').notNull().defaultNow(),
18
28
  });
19
29
 
@@ -26,6 +36,7 @@ export const seazonaApiRequests = pgTable('seazona_api_requests', {
26
36
  clientKey: text('client_key').notNull(),
27
37
  workflow: text('workflow').notNull(),
28
38
  admittedAt: at('admitted_at').notNull(),
39
+ resource: text('resource'),
29
40
  requestPath: text('request_path'),
30
41
  statusCode: integer('status_code'),
31
42
  failureKind: text('failure_kind'),
@@ -155,3 +166,108 @@ export const seazonaOrderReadFailures = pgTable('seazona_order_read_failures', {
155
166
  lastError: text('last_error'),
156
167
  quarantinedUntil: at('quarantined_until'),
157
168
  });
169
+
170
+ // Issue #101: each failed detail read of an invoice or a client, per record.
171
+ // Orders keep their own table (issue #12). After a few failures in a row the
172
+ // record is quarantined: the lane skips it and its cursor moves on.
173
+ export const seazonaRecordReadFailures = pgTable('seazona_record_read_failures', {
174
+ resource: text('resource').notNull(),
175
+ recordKey: text('record_key').notNull(),
176
+ failureCount: integer('failure_count').notNull(),
177
+ lastAttemptAt: at('last_attempt_at').notNull(),
178
+ lastHttpStatus: integer('last_http_status'),
179
+ lastFailureKind: text('last_failure_kind'),
180
+ quarantinedUntil: at('quarantined_until'),
181
+ }, (t) => [
182
+ primaryKey({ columns: [t.resource, t.recordKey] }),
183
+ ]);
184
+
185
+ // History import (docs/SEAZONA.md, "History import"). Seazona's own CSV
186
+ // export brings in the years before the API sync started, with no API call.
187
+ // A batch is one uploaded file. Its rows are kept as Flow read them (only the
188
+ // columns Flow knows; never an unknown column, and no patient field unless
189
+ // SEAZONA_KEEP_PATIENT_REFERENCE is on). Its jobs are the rows grouped into
190
+ // work, each with what reconciliation found and what a person decided.
191
+ // Applying a batch copies its jobs into `seazona_history_jobs`, Flow's own
192
+ // record of past work. A batch is never deleted; a cancelled one stays.
193
+ export const seazonaHistoryBatches = pgTable('seazona_history_batches', {
194
+ id: bigserial('id', { mode: 'number' }).primaryKey(),
195
+ fileName: text('file_name'),
196
+ // The file's sha256: the same file uploaded twice finds its first batch.
197
+ fileSha256: text('file_sha256').notNull(),
198
+ profile: text('profile').notNull(),
199
+ // Flow field -> the file's column header.
200
+ columns: jsonb('columns').notNull().default({}),
201
+ // Headers Flow does not know. Only the names are kept, never the values.
202
+ ignoredColumns: jsonb('ignored_columns').notNull().default([]),
203
+ // staged -> reconciled -> applied, or cancelled.
204
+ status: text('status').notNull(),
205
+ counts: jsonb('counts').notNull().default({}),
206
+ createdBy: text('created_by').notNull(),
207
+ createdAt: at('created_at').notNull().defaultNow(),
208
+ reconciledAt: at('reconciled_at'),
209
+ appliedAt: at('applied_at'),
210
+ appliedBy: text('applied_by'),
211
+ }, (t) => [
212
+ index('seazona_history_batches_file').on(t.fileSha256),
213
+ ]);
214
+
215
+ export const seazonaHistoryRows = pgTable('seazona_history_rows', {
216
+ batchId: bigint('batch_id', { mode: 'number' }).notNull(),
217
+ rowNumber: integer('row_number').notNull(),
218
+ jobKey: text('job_key'),
219
+ fields: jsonb('fields').notNull(),
220
+ // Why the row cannot be read, for example `missing_date`. Empty when fine.
221
+ problems: jsonb('problems').notNull().default([]),
222
+ }, (t) => [
223
+ primaryKey({ columns: [t.batchId, t.rowNumber] }),
224
+ ]);
225
+
226
+ export const seazonaHistoryBatchJobs = pgTable('seazona_history_batch_jobs', {
227
+ batchId: bigint('batch_id', { mode: 'number' }).notNull(),
228
+ jobKey: text('job_key').notNull(),
229
+ job: jsonb('job').notNull(),
230
+ recordHash: text('record_hash').notNull(),
231
+ // What reconciliation found: new, same, changed, needs_person.
232
+ outcome: text('outcome'),
233
+ // Each finding: { code, blocking, detail }.
234
+ findings: jsonb('findings').notNull().default([]),
235
+ // What Seazona's API copy holds for the same work, when Flow found it.
236
+ mirrorOrderId: text('mirror_order_id'),
237
+ mirrorInvoiceNumber: text('mirror_invoice_number'),
238
+ clientId: text('client_id'),
239
+ // A person's choice: import, skip, or null (follow the outcome).
240
+ decision: text('decision'),
241
+ decidedBy: text('decided_by'),
242
+ decidedAt: at('decided_at'),
243
+ }, (t) => [
244
+ primaryKey({ columns: [t.batchId, t.jobKey] }),
245
+ ]);
246
+
247
+ export const seazonaHistoryJobs = pgTable('seazona_history_jobs', {
248
+ jobKey: text('job_key').primaryKey(),
249
+ invoiceNumber: text('invoice_number'),
250
+ caseRef: text('case_ref'),
251
+ clientId: text('client_id'),
252
+ company: text('company'),
253
+ doctorName: text('doctor_name'),
254
+ status: text('status'),
255
+ jobDate: text('job_date'),
256
+ dueDate: text('due_date'),
257
+ shippedDate: text('shipped_date'),
258
+ lines: jsonb('lines').notNull().default([]),
259
+ subtotalCents: cents('subtotal_cents'),
260
+ totalCents: cents('total_cents'),
261
+ patientReference: text('patient_reference'),
262
+ mirrorOrderId: text('mirror_order_id'),
263
+ mirrorInvoiceNumber: text('mirror_invoice_number'),
264
+ recordHash: text('record_hash').notNull(),
265
+ // The batch that last wrote the row, and the first one.
266
+ batchId: bigint('batch_id', { mode: 'number' }).notNull(),
267
+ firstBatchId: bigint('first_batch_id', { mode: 'number' }).notNull(),
268
+ firstImportedAt: at('first_imported_at').notNull(),
269
+ lastChangedAt: at('last_changed_at').notNull(),
270
+ }, (t) => [
271
+ index('seazona_history_jobs_invoice').on(t.invoiceNumber),
272
+ index('seazona_history_jobs_changed').on(t.lastChangedAt),
273
+ ]);
@@ -75,6 +75,16 @@ export function resolveSeazonaSettings(env = process.env) {
75
75
  maxWaitMs: clampInt(env.SEAZONA_API_MAX_WAIT_MS, 120_000, 1_000, 15 * 60_000),
76
76
  // Long enough to answer a vendor's question about last month.
77
77
  telemetryRetentionMs: clampInt(env.SEAZONA_API_TELEMETRY_RETENTION_DAYS, 35, 1, 90) * DAY_MS,
78
+ // Issue #101: a vendor can cut off API access after many failed calls,
79
+ // even inside the rate limit. This many failed calls inside the window
80
+ // open the circuit: no call goes out for the pause, which doubles each
81
+ // time the probe after it fails, to at most 6 hours.
82
+ circuitFailures: clampInt(env.SEAZONA_CIRCUIT_FAILURES, 5, 1, 100),
83
+ circuitWindowMs: clampInt(env.SEAZONA_CIRCUIT_WINDOW_MS, 10 * 60_000, 60_000, DAY_MS),
84
+ circuitPauseMs: clampInt(env.SEAZONA_CIRCUIT_PAUSE_MS, 5 * 60_000, 60_000, 6 * 60 * 60_000),
85
+ // Failed calls allowed in one UTC day. When they are spent, calls stop
86
+ // until the next day.
87
+ dailyFailureBudget: clampInt(env.SEAZONA_DAILY_FAILURE_BUDGET, 50, 1, 10_000),
78
88
  };
79
89
 
80
90
  const MINUTE = 60_000;
@@ -89,19 +99,30 @@ export function resolveSeazonaSettings(env = process.env) {
89
99
  clients: { cadenceMs: ms('SEAZONA_CLIENT_CADENCE_MS', 30 * MINUTE, MINUTE, DAY_MS), freshnessMs: 60 * MINUTE },
90
100
  products: { cadenceMs: ms('SEAZONA_REFERENCE_CADENCE_MS', DAY_MS, MINUTE, 30 * DAY_MS), freshnessMs: 2 * DAY_MS },
91
101
  users: { cadenceMs: ms('SEAZONA_REFERENCE_CADENCE_MS', DAY_MS, MINUTE, 30 * DAY_MS), freshnessMs: 2 * DAY_MS },
102
+ // The history backfill (docs/SEAZONA.md, "History import"). It calls
103
+ // Seazona only while a person has started one.
104
+ history: { cadenceMs: ms('SEAZONA_HISTORY_CADENCE_MS', 2 * MINUTE, MINUTE, DAY_MS), freshnessMs: 30 * DAY_MS },
92
105
  },
106
+ // Calls the history backfill makes per run, at most: a small share of
107
+ // the rate limit, so the live lanes always have room.
108
+ historyCallsPerRun: clampInt(env.SEAZONA_HISTORY_CALLS_PER_RUN, 10, 1, 25),
109
+ historyPageSize: clampInt(env.SEAZONA_HISTORY_PAGE_SIZE, 500, 1, 500),
93
110
  // Re-read this much before a lastModified cursor, so a change written
94
111
  // while the last run was reading is not missed.
95
112
  cursorOverlapMs: 5 * MINUTE,
96
113
  // The recent-order lane reads orders placed in the last N days, a few
97
- // days per run (GET orders/?ordered= matches one UTC day).
114
+ // days per run, page by page (GET orders/?ordered=&page=&pageSize=).
98
115
  orderLookbackDays: clampInt(env.SEAZONA_ORDER_LOOKBACK_DAYS, 30, 1, 3650),
99
116
  orderDaysPerRun: clampInt(env.SEAZONA_ORDER_DAYS_PER_RUN, 6, 1, 31),
100
117
  // The first invoice run reads from the start of today (UTC) minus this.
101
118
  invoiceLookbackDays: clampInt(env.SEAZONA_INVOICE_LOOKBACK_DAYS, 0, 0, 3650),
102
- // Seazona documents no paging. A list exactly this long may be cut off,
103
- // so the lane holds its cursor instead of trusting it.
119
+ // A list this long may be cut off, so the lane reads on (issue #125):
120
+ // the next part of a changed list from its last row, the next order
121
+ // page. Also the order page size (at most 500).
104
122
  listCap: clampInt(env.SEAZONA_LIST_CAP, 50, 1, 10_000),
123
+ // A lane makes at most this many list calls a run; the next run goes on
124
+ // from where it stopped.
125
+ listPagesPerRun: clampInt(env.SEAZONA_LIST_PAGES_PER_RUN, 10, 1, 100),
105
126
  // The active-order lane re-reads this many open orders per run, oldest first.
106
127
  activeOrdersPerRun: clampInt(env.SEAZONA_ACTIVE_ORDERS_PER_RUN, 25, 1, 500),
107
128
  // Issue #12: after this many failed reads in a row an order waits this long.
@@ -6,6 +6,7 @@ import {
6
6
  seazonaLaneState,
7
7
  seazonaOrderReadFailures,
8
8
  seazonaOrders,
9
+ seazonaRecordReadFailures,
9
10
  seazonaReference,
10
11
  } from './schema.js';
11
12
 
@@ -192,6 +193,60 @@ export async function readQuarantinedOrders(db, { at, limit = 25 }) {
192
193
  }));
193
194
  }
194
195
 
196
+ // Issue #101: failed detail reads of invoices and clients, per record.
197
+ // Answers Map(key -> row) for the keys that are quarantined at `at`.
198
+ export async function readQuarantinedRecords(db, resource, keys, { at }) {
199
+ if (!keys.length) return new Map();
200
+ const rows = await db.select().from(seazonaRecordReadFailures).where(and(
201
+ eq(seazonaRecordReadFailures.resource, resource),
202
+ inArray(seazonaRecordReadFailures.recordKey, keys),
203
+ gt(seazonaRecordReadFailures.quarantinedUntil, at),
204
+ ));
205
+ return new Map(rows.map((row) => [row.recordKey, row]));
206
+ }
207
+
208
+ export async function clearRecordFailures(db, resource, keys) {
209
+ if (!keys.length) return;
210
+ await db.delete(seazonaRecordReadFailures).where(and(
211
+ eq(seazonaRecordReadFailures.resource, resource),
212
+ inArray(seazonaRecordReadFailures.recordKey, keys),
213
+ ));
214
+ }
215
+
216
+ // Records one failed read, as recordOrderFailure does for orders. Answers
217
+ // { failureCount, quarantinedUntil }.
218
+ export async function recordRecordFailure(db, resource, key, failure, { at, quarantineFailures, quarantineMs }) {
219
+ const until = new Date(at.getTime() + quarantineMs);
220
+ const [row] = await db.insert(seazonaRecordReadFailures).values({
221
+ resource,
222
+ recordKey: key,
223
+ failureCount: 1,
224
+ lastAttemptAt: at,
225
+ lastHttpStatus: failure.httpStatus,
226
+ lastFailureKind: failure.kind,
227
+ quarantinedUntil: quarantineFailures <= 1 ? until : null,
228
+ }).onConflictDoUpdate({
229
+ target: [seazonaRecordReadFailures.resource, seazonaRecordReadFailures.recordKey],
230
+ set: {
231
+ failureCount: sql`${seazonaRecordReadFailures.failureCount} + 1`,
232
+ lastAttemptAt: at,
233
+ lastHttpStatus: failure.httpStatus,
234
+ lastFailureKind: failure.kind,
235
+ quarantinedUntil: sql`case when ${seazonaRecordReadFailures.failureCount} + 1 >= ${quarantineFailures}
236
+ then ${until}::timestamptz else null end`,
237
+ },
238
+ }).returning();
239
+ return { failureCount: row.failureCount, quarantinedUntil: row.quarantinedUntil?.toISOString() ?? null };
240
+ }
241
+
242
+ export async function readQuarantinedRecordCounts(db, { at }) {
243
+ const rows = await db.select({ resource: seazonaRecordReadFailures.resource, records: sql`count(*)::int` })
244
+ .from(seazonaRecordReadFailures)
245
+ .where(gt(seazonaRecordReadFailures.quarantinedUntil, at))
246
+ .groupBy(seazonaRecordReadFailures.resource);
247
+ return Object.fromEntries(rows.map((row) => [row.resource, row.records]));
248
+ }
249
+
195
250
  // Lane state.
196
251
 
197
252
  export async function readLaneStates(db) {
@@ -214,6 +269,25 @@ export async function claimLane(db, lane, { at, stuckMs }) {
214
269
  return claimed.length > 0;
215
270
  }
216
271
 
272
+ // Issue #101: a lane stopped for a person or a release runs again. Answers
273
+ // the lanes it reset.
274
+ export async function resetStoppedLanes(db, { at }) {
275
+ const rows = await db.update(seazonaLaneState)
276
+ .set({ status: 'idle', nextRunAt: at, failureCount: 0, heldReason: null, updatedAt: at })
277
+ .where(eq(seazonaLaneState.status, 'stopped'))
278
+ .returning({ lane: seazonaLaneState.lane });
279
+ return rows.map((row) => row.lane);
280
+ }
281
+
282
+ // A person acted: every lane that failed or stopped runs at the next tick.
283
+ export async function resetFailedLanes(db, { at }) {
284
+ const rows = await db.update(seazonaLaneState)
285
+ .set({ status: 'idle', nextRunAt: at, failureCount: 0, heldReason: null, updatedAt: at })
286
+ .where(inArray(seazonaLaneState.status, ['stopped', 'failed', 'held']))
287
+ .returning({ lane: seazonaLaneState.lane });
288
+ return rows.map((row) => row.lane);
289
+ }
290
+
217
291
  export async function finishLane(db, lane, patch) {
218
292
  await db.update(seazonaLaneState).set({ ...patch, updatedAt: patch.lastAttemptAt ?? new Date() }).where(eq(seazonaLaneState.lane, lane));
219
293
  }
@@ -6,7 +6,7 @@
6
6
  // DATABASE_URL=postgres://… FLOW_INTEGRATIONS=seazona SEAZONA_MODE=sandbox … \
7
7
  // node server/integrations/seazona/sync-once.js [lane …]
8
8
  //
9
- // Lanes: invoices, orders_recent, orders_active, clients, products, users.
9
+ // Lanes: invoices, orders_recent, orders_active, clients, products, users, history.
10
10
  // None named: all of them.
11
11
  import { openDatabase } from '../../db/index.js';
12
12
  import { resolveEnabledIntegrations } from '../index.js';
@@ -13,5 +13,10 @@ export const seazonaSchema = Object.freeze({
13
13
  'seazona_reference',
14
14
  'seazona_lane_state',
15
15
  'seazona_order_read_failures',
16
+ 'seazona_record_read_failures',
17
+ 'seazona_history_batches',
18
+ 'seazona_history_rows',
19
+ 'seazona_history_batch_jobs',
20
+ 'seazona_history_jobs',
16
21
  ]),
17
22
  });
@@ -2,9 +2,11 @@ import { FlowError } from '../../core/errors.js';
2
2
  import { XERO_ENDPOINTS } from './settings.js';
3
3
 
4
4
  // The Xero Accounting API client. Every call goes through the rate-limit
5
- // governor (governor.js). A 429 is retried after the wait Xero asks for,
6
- // except a DayLimit 429, which opens the circuit and fails fast. Tokens are
7
- // sent only in the Authorization header and never logged or put in errors.
5
+ // governor (governor.js) and, when one is given, the call guard
6
+ // (../call-guard.js): a call the guard has stopped fails at once, and every
7
+ // call's ending is recorded there. A 429 is retried after the wait Xero asks
8
+ // for, except a DayLimit 429, which opens the circuit and fails fast. Tokens
9
+ // are sent only in the Authorization header and never logged or put in errors.
8
10
 
9
11
  export const DEFAULT_TIMEOUT_MS = 15_000;
10
12
  export const DEFAULT_ATTEMPTS = 2;
@@ -86,6 +88,14 @@ function quote(value) {
86
88
 
87
89
  const first = (list) => (Array.isArray(list) && list.length ? list[0] : null);
88
90
 
91
+ // The resource a call reads or writes, for the call guard: the first path
92
+ // segment (Invoices, Payments, Contacts). A 401 or 403 that a fresh token
93
+ // does not clear stops that resource only, so a scope the connection lacks
94
+ // (for example payments) does not stop the rest.
95
+ export function resourceOf(path) {
96
+ return String(path).replace(/^\/+/, '').split(/[/?#]/)[0] || null;
97
+ }
98
+
89
99
  // `accessToken()` returns a current access token; `invalidateAccessToken()`
90
100
  // drops a cached one after Xero answers 401, and the request is tried once
91
101
  // more with a fresh token.
@@ -95,6 +105,7 @@ export function createXeroClient({
95
105
  invalidateAccessToken = () => {},
96
106
  fetch: fetchImpl = globalThis.fetch,
97
107
  governor,
108
+ guard = null,
98
109
  apiBase = XERO_ENDPOINTS.api,
99
110
  timeoutMs = DEFAULT_TIMEOUT_MS,
100
111
  attempts = DEFAULT_ATTEMPTS,
@@ -106,10 +117,12 @@ export function createXeroClient({
106
117
  const base = apiBase.endsWith('/') ? apiBase : `${apiBase}/`;
107
118
  let lastRateLimit = null;
108
119
 
109
- async function send(method, url, headers, bodyText) {
120
+ // `call` is { resource, path } for the guard.
121
+ async function send(method, url, headers, bodyText, call) {
110
122
  await governor.acquire(tenantId);
111
123
  const controller = new AbortController();
112
124
  const timer = setTimeout(() => controller.abort(), timeoutMs);
125
+ const startedAt = Date.now();
113
126
  try {
114
127
  let response;
115
128
  try {
@@ -117,6 +130,7 @@ export function createXeroClient({
117
130
  } catch (error) {
118
131
  if (error?.code === 'xero_replay_no_match') throw error;
119
132
  const timedOut = error?.name === 'AbortError';
133
+ await guard?.record(tenantId, { ...call, failureKind: timedOut ? 'timeout' : 'network', durationMs: Date.now() - startedAt });
120
134
  throw xeroError(timedOut ? 'xero_timeout' : 'xero_unreachable',
121
135
  timedOut ? `Xero did not answer within ${timeoutMs} ms.` : 'Flow could not reach Xero.',
122
136
  { status: timedOut ? 504 : 503 });
@@ -124,7 +138,7 @@ export function createXeroClient({
124
138
  const text = await response.text().catch(() => '');
125
139
  lastRateLimit = readRateLimitHeaders(response);
126
140
  governor.observe(tenantId, lastRateLimit);
127
- return { response, text };
141
+ return { response, text, durationMs: Date.now() - startedAt };
128
142
  } finally {
129
143
  clearTimeout(timer);
130
144
  governor.release(tenantId);
@@ -139,8 +153,10 @@ export function createXeroClient({
139
153
  // Serialised once, so a retry sends the same bytes under the same
140
154
  // Idempotency-Key.
141
155
  const bodyText = body === undefined ? undefined : JSON.stringify(body);
156
+ const call = { resource: resourceOf(path), path: url.pathname };
142
157
  let refreshed = false;
143
158
  for (let attempt = 1; ; attempt += 1) {
159
+ await guard?.check(tenantId, { resource: call.resource });
144
160
  const headers = {
145
161
  ...extra,
146
162
  Authorization: `Bearer ${await accessToken()}`,
@@ -148,12 +164,16 @@ export function createXeroClient({
148
164
  'xero-tenant-id': tenantId,
149
165
  ...(bodyText === undefined ? {} : { 'Content-Type': 'application/json' }),
150
166
  };
151
- const { response, text } = await send(method, url, headers, bodyText);
167
+ const { response, text, durationMs } = await send(method, url, headers, bodyText, call);
152
168
  const payload = parseJson(text);
169
+ // A first 401 usually means the access token expired: it is not counted,
170
+ // and the call is tried once more with a fresh token.
171
+ const expiredToken = response.status === 401 && !refreshed;
172
+ if (!expiredToken) await guard?.record(tenantId, { ...call, statusCode: response.status, durationMs });
153
173
  if (response.ok) return payload ?? {};
154
174
 
155
175
  const error = apiError(response, payload);
156
- if (response.status === 401 && !refreshed) {
176
+ if (expiredToken) {
157
177
  refreshed = true;
158
178
  invalidateAccessToken();
159
179
  attempt -= 1;