@arcforgelabs/flow 2026.10.27 → 2026.10.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +2 -2
- package/server/api/invoicing.js +96 -40
- package/server/app.js +3 -1
- package/server/core/automations.js +46 -1
- package/server/core/invoice-rules.js +4 -2
- package/server/core/invoices.js +4 -2
- package/server/core/job-board.js +130 -0
- package/server/core/ledger.js +12 -5
- package/server/core/line-mapping.js +337 -0
- package/server/core/pipeline.js +18 -6
- package/server/core/queue.js +22 -0
- package/server/core/sending.js +20 -7
- package/server/db/migrate-all.js +26 -0
- package/server/db/migrations/0010_invoicing_rules.sql +34 -13
- package/server/db/migrations/0011_invoice_rules.sql +11 -5
- package/server/db/migrations/0012_invoice_sending.sql +18 -9
- package/server/db/migrations/0013_line_mappings.sql +35 -0
- package/server/db/migrations/meta/0013_snapshot.json +3568 -0
- package/server/db/migrations/meta/_journal.json +7 -0
- package/server/db/schema.js +43 -0
- package/server/import/at-start.js +64 -2
- package/server/index.js +5 -1
- package/server/integrations/port.js +4 -1
- package/server/integrations/seazona/client.js +6 -0
- package/server/integrations/seazona/feed.js +62 -4
- package/server/integrations/seazona/governor.js +138 -13
- package/server/integrations/seazona/index.js +21 -2
- package/server/integrations/seazona/lanes.js +69 -13
- package/server/integrations/seazona/migrations/0000_api_governor.sql +3 -3
- package/server/integrations/seazona/migrations/0001_mirror.sql +9 -9
- package/server/integrations/seazona/migrations/0002_call_safety.sql +15 -0
- package/server/integrations/seazona/migrations/meta/0002_snapshot.json +882 -0
- package/server/integrations/seazona/migrations/meta/_journal.json +7 -0
- package/server/integrations/seazona/scheduler.js +37 -1
- package/server/integrations/seazona/schema.js +22 -0
- package/server/integrations/seazona/settings.js +10 -0
- package/server/integrations/seazona/store.js +74 -0
- package/server/integrations/seazona/tables.js +1 -0
- package/server/integrations/xero/drafts.js +4 -1
- package/server/integrations/xero/import.js +1 -0
- package/server/integrations/xero/lookup.js +6 -2
- package/server/integrations/xero/migrations/0000_xero_connections.sql +1 -1
- package/server/integrations/xero/migrations/0001_xero_catalogue.sql +5 -5
- package/server/integrations/xero/migrations/0002_xero_invoice_pushes.sql +2 -2
- package/server/integrations/xero/migrations/0003_xero_payment_cursors.sql +1 -1
- package/server/integrations/xero/migrations/0004_xero_mirror.sql +7 -7
- package/server/integrations/xero/migrations/0005_xero_send_modes.sql +12 -6
- package/server/ui/app-page.js +6 -4
- package/server/ui/app.js +111 -8
- package/server/ui/clerk-appearance.js +21 -0
- package/server/ui/flow.css +155 -30
- package/server/ui/invoicing-settings.js +2 -0
- package/server/ui/job-board.js +155 -0
- package/server/ui/jobs.js +220 -91
- package/server/ui/line-mappings.js +60 -0
- package/server/ui/monitoring.js +29 -7
- package/server/ui/routes.js +1 -1
- package/server/ui/search.js +145 -0
package/server/db/schema.js
CHANGED
|
@@ -59,6 +59,9 @@ export const priceItems = pgTable('price_items', {
|
|
|
59
59
|
organisationId: organisationId().references(() => organisations.id),
|
|
60
60
|
code: text('code').notNull(),
|
|
61
61
|
name: text('name').notNull(),
|
|
62
|
+
// The accounting system's longer text for the item, when it has one. Line
|
|
63
|
+
// mapping (server/core/line-mapping.js) compares a work line with it too.
|
|
64
|
+
description: text('description'),
|
|
62
65
|
unitAmountCents: integer('unit_amount_cents').notNull(),
|
|
63
66
|
taxCode: text('tax_code').notNull().default('GST'),
|
|
64
67
|
archivedAt: timestamp('archived_at', { withTimezone: true }),
|
|
@@ -120,6 +123,9 @@ export const invoiceLines = pgTable('invoice_lines', {
|
|
|
120
123
|
quantity: integer('quantity').notNull(),
|
|
121
124
|
unitAmountCents: integer('unit_amount_cents').notNull(),
|
|
122
125
|
taxCode: text('tax_code').notNull(),
|
|
126
|
+
// A line with no price item may name the accounting account it books to,
|
|
127
|
+
// for example a surcharge (server/core/line-mapping.js).
|
|
128
|
+
accountCode: text('account_code'),
|
|
123
129
|
lineAmountCents: integer('line_amount_cents').notNull(),
|
|
124
130
|
taxCents: integer('tax_cents').notNull(),
|
|
125
131
|
}, (t) => [
|
|
@@ -674,3 +680,40 @@ export const automations = pgTable('automations', {
|
|
|
674
680
|
}, (t) => [
|
|
675
681
|
primaryKey({ name: 'automations_pk', columns: [t.organisationId, t.key] }),
|
|
676
682
|
]);
|
|
683
|
+
|
|
684
|
+
// Line mappings (docs/LINE_MAPPING.md): how a work system's line label finds
|
|
685
|
+
// the lab's price item, as rows the lab edits. `kind` says which step of the
|
|
686
|
+
// line mapping routine uses the row:
|
|
687
|
+
// alias label (exact, or the start of it) -> price item code
|
|
688
|
+
// price_alias label and unit price -> price item code
|
|
689
|
+
// saved label and unit price -> price item code, chosen by a person
|
|
690
|
+
// account label -> a line with no price item, booked to an account
|
|
691
|
+
// `label` is stored in the lookup form (lineLookupText), so it matches in any
|
|
692
|
+
// case, spacing or punctuation.
|
|
693
|
+
export const LINE_MAPPING_KINDS = ['alias', 'price_alias', 'saved', 'account'];
|
|
694
|
+
export const lineMappings = pgTable('line_mappings', {
|
|
695
|
+
id: id(),
|
|
696
|
+
organisationId: organisationId().references(() => organisations.id),
|
|
697
|
+
kind: text('kind').notNull(),
|
|
698
|
+
match: text('match').notNull().default('exact'),
|
|
699
|
+
label: text('label').notNull(),
|
|
700
|
+
unitAmountCents: integer('unit_amount_cents'),
|
|
701
|
+
itemCode: text('item_code'),
|
|
702
|
+
accountCode: text('account_code'),
|
|
703
|
+
taxCode: text('tax_code'),
|
|
704
|
+
description: text('description'),
|
|
705
|
+
enabled: boolean('enabled').notNull().default(true),
|
|
706
|
+
// Who made it: a person, a setup profile, or the earlier codebase's rules.
|
|
707
|
+
origin: text('origin').notNull().default('person'),
|
|
708
|
+
updatedBy: uuid('updated_by').references(() => people.id, { onDelete: 'set null' }),
|
|
709
|
+
createdAt: createdAt(),
|
|
710
|
+
updatedAt: updatedAt(),
|
|
711
|
+
}, (t) => [
|
|
712
|
+
uniqueIndex('line_mappings_unique').on(t.organisationId, t.kind, t.match, t.label, sql`coalesce(${t.unitAmountCents}, -1)`),
|
|
713
|
+
check('line_mappings_kind_valid', sql`${t.kind} in ('alias', 'price_alias', 'saved', 'account')`),
|
|
714
|
+
check('line_mappings_match_valid', sql`${t.match} in ('exact', 'starts_with')`),
|
|
715
|
+
check('line_mappings_tax_code_valid', sql`${t.taxCode} is null or ${t.taxCode} in ('GST', 'GST_FREE')`),
|
|
716
|
+
check('line_mappings_target', sql`(${t.kind} = 'account' and ${t.accountCode} is not null) or (${t.kind} <> 'account' and ${t.itemCode} is not null)`),
|
|
717
|
+
check('line_mappings_price', sql`${t.kind} not in ('price_alias', 'saved') or ${t.unitAmountCents} is not null`),
|
|
718
|
+
check('line_mappings_label_valid', sql`length(${t.label}) between 1 and 200`),
|
|
719
|
+
]);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { sql } from 'drizzle-orm';
|
|
2
|
-
import { organisations, priorLinks } from '../db/schema.js';
|
|
1
|
+
import { and, eq, isNull, sql } from 'drizzle-orm';
|
|
2
|
+
import { lineMappings, organisations, priceItems, priorLinks } from '../db/schema.js';
|
|
3
|
+
import { lineLookupText, remapJobs } from '../core/line-mapping.js';
|
|
3
4
|
import { applyImport, planImport, readEarlierFlow, summarisePlan } from './earlier-flow.js';
|
|
4
5
|
|
|
5
6
|
// The import from the earlier codebase, run once by the server itself. On a
|
|
@@ -118,3 +119,64 @@ export async function importEarlierLinksAtStart({ db, pool, settings, log = cons
|
|
|
118
119
|
log.info?.(`[import] links from schema "${settings.schema}": ${JSON.stringify(counts)}`);
|
|
119
120
|
return counts;
|
|
120
121
|
}
|
|
122
|
+
|
|
123
|
+
// Carries the earlier app's line mapping knowledge into Flow, at every start
|
|
124
|
+
// while the import settings are set, in the same read-only way:
|
|
125
|
+
// - each label and price a person matched to an item becomes a "saved" line
|
|
126
|
+
// mapping (server/core/line-mapping.js), unless the lab already has one;
|
|
127
|
+
// - each price item with no description takes the accounting item's own.
|
|
128
|
+
// Answers counts, or null when there is nothing to read.
|
|
129
|
+
const EARLIER_RULES = 'xero_item_match_rules';
|
|
130
|
+
const EARLIER_ITEMS = 'xero_catalog_items';
|
|
131
|
+
|
|
132
|
+
export async function importEarlierMappingsAtStart({ db, pool, settings, log = console }) {
|
|
133
|
+
if (!settings) return null;
|
|
134
|
+
const { rows: labs } = await db.execute(sql`select id, name from ${organisations}`);
|
|
135
|
+
const organisation = labs.find((lab) => lab.name === settings.organisationName) ?? (labs.length === 1 ? labs[0] : null);
|
|
136
|
+
if (!organisation) return null;
|
|
137
|
+
|
|
138
|
+
let rules = [];
|
|
139
|
+
let items = [];
|
|
140
|
+
const client = await pool.connect();
|
|
141
|
+
try {
|
|
142
|
+
await client.query('BEGIN TRANSACTION READ ONLY');
|
|
143
|
+
await client.query(`SET LOCAL search_path TO "${settings.schema}"`);
|
|
144
|
+
const present = async (table) => (await client.query('select to_regclass($1) is not null as present', [table])).rows[0].present;
|
|
145
|
+
if (await present(EARLIER_RULES)) {
|
|
146
|
+
({ rows: rules } = await client.query(`select source_lookup_text, unit_amount_cents, xero_item_code from ${EARLIER_RULES}`));
|
|
147
|
+
}
|
|
148
|
+
if (await present(EARLIER_ITEMS)) {
|
|
149
|
+
({ rows: items } = await client.query(`select item_code, description from ${EARLIER_ITEMS} where coalesce(description, '') <> ''`));
|
|
150
|
+
}
|
|
151
|
+
await client.query('COMMIT');
|
|
152
|
+
} catch (error) {
|
|
153
|
+
await client.query('ROLLBACK').catch(() => {});
|
|
154
|
+
throw new Error(`could not read mappings from schema "${settings.schema}": ${error.message}`);
|
|
155
|
+
} finally {
|
|
156
|
+
client.release();
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const values = rules
|
|
160
|
+
.map((row) => ({
|
|
161
|
+
organisationId: organisation.id,
|
|
162
|
+
kind: 'saved',
|
|
163
|
+
match: 'exact',
|
|
164
|
+
label: lineLookupText(row.source_lookup_text),
|
|
165
|
+
unitAmountCents: Number(row.unit_amount_cents),
|
|
166
|
+
itemCode: String(row.xero_item_code ?? '').trim(),
|
|
167
|
+
origin: `earlier:${EARLIER_RULES}`,
|
|
168
|
+
}))
|
|
169
|
+
.filter((v) => v.label && v.label.length <= 200 && v.itemCode && Number.isInteger(v.unitAmountCents) && v.unitAmountCents >= 0);
|
|
170
|
+
const added = values.length ? (await db.insert(lineMappings).values(values).onConflictDoNothing().returning({ id: lineMappings.id })).length : 0;
|
|
171
|
+
let described = 0;
|
|
172
|
+
for (const item of items) {
|
|
173
|
+
const updated = await db.update(priceItems).set({ description: String(item.description).trim().slice(0, 1000) })
|
|
174
|
+
.where(and(eq(priceItems.organisationId, organisation.id), eq(priceItems.code, String(item.item_code).trim()), isNull(priceItems.description)))
|
|
175
|
+
.returning({ id: priceItems.id });
|
|
176
|
+
described += updated.length;
|
|
177
|
+
}
|
|
178
|
+
if (added) await remapJobs(db, organisation.id);
|
|
179
|
+
const counts = { rules: values.length, added, described };
|
|
180
|
+
log.info?.(`[import] mappings from schema "${settings.schema}": ${JSON.stringify(counts)}`);
|
|
181
|
+
return counts;
|
|
182
|
+
}
|
package/server/index.js
CHANGED
|
@@ -5,7 +5,9 @@ import { seedFirstOwner } from './auth/people.js';
|
|
|
5
5
|
import { createPersonAuth } from './auth/session.js';
|
|
6
6
|
import { resolveSignInSettings } from './auth/settings.js';
|
|
7
7
|
import { migrateDatabase, openDatabase } from './db/index.js';
|
|
8
|
-
import {
|
|
8
|
+
import {
|
|
9
|
+
importEarlierAtStart, importEarlierLinksAtStart, importEarlierMappingsAtStart, resolveImportAtStart,
|
|
10
|
+
} from './import/at-start.js';
|
|
9
11
|
import { createMonitoredLog } from './core/monitoring.js';
|
|
10
12
|
import { createPipelineWorker } from './core/pipeline.js';
|
|
11
13
|
import { createSendWorker } from './core/sending.js';
|
|
@@ -66,6 +68,8 @@ try {
|
|
|
66
68
|
// failure is logged and the start goes on.
|
|
67
69
|
await importEarlierLinksAtStart({ db: database.db, pool: database.pool, settings: importAtStart })
|
|
68
70
|
.catch((error) => console.error(`[import] ${error.message}`));
|
|
71
|
+
await importEarlierMappingsAtStart({ db: database.db, pool: database.pool, settings: importAtStart })
|
|
72
|
+
.catch((error) => console.error(`[import] ${error.message}`));
|
|
69
73
|
// What an owner set in Settings, Integrations, with the host's own
|
|
70
74
|
// environment over it (server/integrations/setup.js).
|
|
71
75
|
// Every part of Flow logs here: each error also goes to the installation's
|
|
@@ -18,6 +18,7 @@ export const CAPABILITIES = Object.freeze([
|
|
|
18
18
|
'pushBill', // send an approved bill; the Flow record id is the idempotency key
|
|
19
19
|
'pullPayments', // payments since a cursor, matched to invoices and bills
|
|
20
20
|
'checkInvoice', // does the accounting system hold an invoice for a billable job? (the ledger's liveCheck)
|
|
21
|
+
'listWork', // the work system's jobs from its copy in Flow, open and lately finished, for the Jobs board
|
|
21
22
|
'listCustomers', // the system's customers, by id and name, for the identity map (Seazona: clients; Xero: contacts)
|
|
22
23
|
'findInvoices', // sales invoices in the accounting system's copy, for the matcher; never a live call
|
|
23
24
|
'health', // connection state for Settings
|
|
@@ -233,7 +234,9 @@ export function createRegistry({ log = console, env = process.env } = {}) {
|
|
|
233
234
|
tablesReady.add(id);
|
|
234
235
|
}
|
|
235
236
|
if (adapter.start) {
|
|
236
|
-
|
|
237
|
+
// A person changed the settings: an adapter may clear a stop that
|
|
238
|
+
// waits for a person (issue #101).
|
|
239
|
+
await adapter.start({ db: database, log, env: settingsEnv, reconfigured: true });
|
|
237
240
|
running.push(adapter);
|
|
238
241
|
}
|
|
239
242
|
} catch (error) {
|
|
@@ -17,6 +17,8 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, m
|
|
|
17
17
|
// A failed Seazona request. The message names the method, the path and the
|
|
18
18
|
// status or failure kind; never the host, the query or a credential.
|
|
19
19
|
// kind: http, timeout, network, invalid_json, replay_miss, governor_timeout.
|
|
20
|
+
// The governor's own SeazonaCallsStopped (kind calls_stopped) passes through
|
|
21
|
+
// unchanged: no call was made.
|
|
20
22
|
export class SeazonaApiError extends Error {
|
|
21
23
|
constructor(message, { statusCode = null, kind = 'http', path = null, retryAfterMs = null } = {}) {
|
|
22
24
|
super(message);
|
|
@@ -93,6 +95,10 @@ export function createSeazonaClient({
|
|
|
93
95
|
requestsPerMinute: rate.requestsPerMinute,
|
|
94
96
|
maxWaitMs: rate.maxWaitMs,
|
|
95
97
|
telemetryRetentionMs: rate.telemetryRetentionMs,
|
|
98
|
+
circuitFailures: rate.circuitFailures,
|
|
99
|
+
circuitWindowMs: rate.circuitWindowMs,
|
|
100
|
+
circuitPauseMs: rate.circuitPauseMs,
|
|
101
|
+
dailyFailureBudget: rate.dailyFailureBudget,
|
|
96
102
|
now,
|
|
97
103
|
sleepFn,
|
|
98
104
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { eq, sql } from 'drizzle-orm';
|
|
1
|
+
import { and, desc, eq, gte, isNull, or, sql } from 'drizzle-orm';
|
|
2
2
|
import { ledgerLinesFor, toLedgerHold, toLedgerJob } from './ledger.js';
|
|
3
|
-
import { orderStatusLabel, parseSeazonaDate } from './map.js';
|
|
3
|
+
import { isFinalStatus, orderStatusLabel, parseSeazonaDate } from './map.js';
|
|
4
4
|
import { seazonaClients, seazonaInvoices, seazonaOrders } from './schema.js';
|
|
5
5
|
|
|
6
6
|
// The billing feed behind the port's pullWorkItems: every order that changed
|
|
@@ -104,12 +104,70 @@ export async function readBillableWork(db, { cursor = null, limit = 200 } = {})
|
|
|
104
104
|
}
|
|
105
105
|
|
|
106
106
|
// Seazona clients, by id and the company name an invoice bills. A client
|
|
107
|
-
// with no company name is listed by its full name.
|
|
107
|
+
// with no company name is listed by its full name. A client the client list
|
|
108
|
+
// never gave (the list can be closed to an API client) is listed by the
|
|
109
|
+
// company name its orders carry, so its jobs still find their customer.
|
|
108
110
|
export async function listClients(db) {
|
|
109
111
|
const rows = await db.select({ id: seazonaClients.id, company: seazonaClients.company, fullName: seazonaClients.fullName, active: seazonaClients.active })
|
|
110
112
|
.from(seazonaClients);
|
|
111
|
-
|
|
113
|
+
const listed = rows
|
|
112
114
|
.filter((r) => r.active !== false)
|
|
113
115
|
.map((r) => ({ id: r.id, name: (r.company || r.fullName || '').trim() }))
|
|
114
116
|
.filter((r) => r.name);
|
|
117
|
+
const known = new Set(rows.map((r) => r.id));
|
|
118
|
+
const fromOrders = await db.selectDistinctOn([seazonaOrders.clientId], { id: seazonaOrders.clientId, company: seazonaOrders.company })
|
|
119
|
+
.from(seazonaOrders)
|
|
120
|
+
.where(sql`${seazonaOrders.clientId} is not null and coalesce(${seazonaOrders.company}, '') <> '' and ${seazonaOrders.deletedInSourceAt} is null`)
|
|
121
|
+
.orderBy(seazonaOrders.clientId, sql`${seazonaOrders.lastChangedAt} desc`);
|
|
122
|
+
return [
|
|
123
|
+
...listed,
|
|
124
|
+
...fromOrders.filter((r) => !known.has(r.id)).map((r) => ({ id: r.id, name: r.company.trim() })),
|
|
125
|
+
];
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// A final order (shipped, delivered, cancelled; map.js isFinalStatus) stays on
|
|
129
|
+
// the Jobs board this long after it last changed.
|
|
130
|
+
export const WORK_FINISHED_DAYS = 30;
|
|
131
|
+
const WORK_LIMIT = 2000;
|
|
132
|
+
const FINAL_SQL = sql`coalesce(${seazonaOrders.status}, '') ~* '(shipped|delivered|cancelled|canceled|void|closed|archived|deleted)'`;
|
|
133
|
+
|
|
134
|
+
// The orders behind the port's listWork, for the Jobs board: every open
|
|
135
|
+
// order, and each final order that changed in the last WORK_FINISHED_DAYS
|
|
136
|
+
// days, newest change first. An order deleted in Seazona is never listed.
|
|
137
|
+
// The client name is the Seazona client's company (or full name), else the
|
|
138
|
+
// order's own company. `statuses` lists each status once, with the first time
|
|
139
|
+
// Flow saw an order in it, so the board can order its columns.
|
|
140
|
+
export async function listWork(db, { now = Date.now(), finishedDays = WORK_FINISHED_DAYS, limit = WORK_LIMIT } = {}) {
|
|
141
|
+
const since = new Date(now - finishedDays * 86_400_000);
|
|
142
|
+
const rows = await db.select({ order: seazonaOrders, company: seazonaClients.company, fullName: seazonaClients.fullName })
|
|
143
|
+
.from(seazonaOrders)
|
|
144
|
+
.leftJoin(seazonaClients, eq(seazonaClients.id, seazonaOrders.clientId))
|
|
145
|
+
.where(and(isNull(seazonaOrders.deletedInSourceAt), or(sql`not (${FINAL_SQL})`, gte(seazonaOrders.lastChangedAt, since))))
|
|
146
|
+
.orderBy(desc(seazonaOrders.lastChangedAt), seazonaOrders.id)
|
|
147
|
+
.limit(limit);
|
|
148
|
+
const firstSeen = new Map();
|
|
149
|
+
const items = rows.map(({ order, company, fullName }) => {
|
|
150
|
+
const status = orderStatusLabel(order) || null;
|
|
151
|
+
const seen = firstSeen.get(status);
|
|
152
|
+
if (status && (!seen || order.firstSeenAt < seen)) firstSeen.set(status, order.firstSeenAt);
|
|
153
|
+
return {
|
|
154
|
+
sourceSystem: 'seazona',
|
|
155
|
+
sourceJobRef: order.id,
|
|
156
|
+
// The Seazona order id is the case number. Never patient data.
|
|
157
|
+
caseRef: order.id,
|
|
158
|
+
status,
|
|
159
|
+
clientId: order.clientId,
|
|
160
|
+
clientName: (company || order.company || fullName || '').trim() || null,
|
|
161
|
+
practitioner: order.doctorName,
|
|
162
|
+
department: order.department,
|
|
163
|
+
orderedOn: parseSeazonaDate(order.orderedText),
|
|
164
|
+
dueOn: parseSeazonaDate(order.dueText),
|
|
165
|
+
lines: (order.lines ?? []).map((l) => ({ code: l.code ?? null, description: l.description, amountCents: l.amountCents ?? null })),
|
|
166
|
+
totalCents: order.subtotalCents,
|
|
167
|
+
};
|
|
168
|
+
});
|
|
169
|
+
return {
|
|
170
|
+
items,
|
|
171
|
+
statuses: [...firstSeen].map(([label, at]) => ({ label, final: isFinalStatus(label), firstSeenAt: at.toISOString() })),
|
|
172
|
+
};
|
|
115
173
|
}
|
|
@@ -10,9 +10,39 @@ import { MAX_REQUESTS_PER_MINUTE } from './settings.js';
|
|
|
10
10
|
// admitted in the last minute, the minimum gap since the last admission has
|
|
11
11
|
// passed, and no 429 cooldown is running. Admission takes a transaction-level
|
|
12
12
|
// advisory lock per client, so two processes never admit the same slot.
|
|
13
|
+
//
|
|
14
|
+
// Issue #101: a vendor can cut off API access after many failed calls, even
|
|
15
|
+
// inside the rate limit. So the governor also stops calls outright:
|
|
16
|
+
// - a 401 or 403 halts every call until a person acts (the adapter restarts
|
|
17
|
+
// with new or saved settings, which calls resume());
|
|
18
|
+
// - `circuitFailures` failed calls inside `circuitWindowMs` open the circuit:
|
|
19
|
+
// no call for `circuitPauseMs`, then one probe call; a failed probe opens
|
|
20
|
+
// it again for twice as long, to at most 6 hours, and a success closes it;
|
|
21
|
+
// - `dailyFailureBudget` failed calls in one UTC day stop calls until the
|
|
22
|
+
// next day.
|
|
23
|
+
// A stopped call fails at once with SeazonaCallsStopped; it never waits.
|
|
13
24
|
|
|
14
25
|
const WINDOW_MS = 60_000;
|
|
15
26
|
const MAX_COOLDOWN_MS = 60_000;
|
|
27
|
+
const MAX_CIRCUIT_PAUSE_MS = 6 * 60 * 60_000;
|
|
28
|
+
const DAY_MS = 24 * 60 * 60_000;
|
|
29
|
+
const EPOCH = new Date(0);
|
|
30
|
+
|
|
31
|
+
// A call that failed in a way the vendor sees as a fault: no answer, a 5xx, a
|
|
32
|
+
// 429, or a 4xx other than 404 (a 404 on one record means it was deleted).
|
|
33
|
+
export function countsAsFailure({ statusCode = null, failureKind = null } = {}) {
|
|
34
|
+
if (failureKind) return true;
|
|
35
|
+
if (!Number.isInteger(statusCode)) return false;
|
|
36
|
+
return statusCode >= 500 || (statusCode >= 400 && statusCode !== 404);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const failedCall = sql`(${seazonaApiRequests.failureKind} is not null or ${seazonaApiRequests.statusCode} >= 500
|
|
40
|
+
or (${seazonaApiRequests.statusCode} >= 400 and ${seazonaApiRequests.statusCode} <> 404))`;
|
|
41
|
+
|
|
42
|
+
const startOfUtcDay = (ms) => {
|
|
43
|
+
const date = new Date(ms);
|
|
44
|
+
return Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate());
|
|
45
|
+
};
|
|
16
46
|
|
|
17
47
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
|
|
18
48
|
|
|
@@ -36,12 +66,34 @@ export class SeazonaGovernorTimeout extends Error {
|
|
|
36
66
|
}
|
|
37
67
|
}
|
|
38
68
|
|
|
69
|
+
// Calls are stopped: halted for a person, circuit open, or the day's failed
|
|
70
|
+
// calls are spent. `retryAfterMs` is null for a halt, which no wait clears.
|
|
71
|
+
export class SeazonaCallsStopped extends Error {
|
|
72
|
+
constructor(reason, retryAfterMs = null) {
|
|
73
|
+
const what = {
|
|
74
|
+
halted: 'Seazona calls are stopped until a person checks the API client (it was refused)',
|
|
75
|
+
circuit_open: 'Seazona calls are paused after repeated failures',
|
|
76
|
+
budget_spent: "Seazona calls are stopped for today: the day's failed calls are spent",
|
|
77
|
+
}[reason];
|
|
78
|
+
super(what);
|
|
79
|
+
this.name = 'SeazonaCallsStopped';
|
|
80
|
+
this.code = { halted: 'seazona_api_halted', circuit_open: 'seazona_api_circuit_open', budget_spent: 'seazona_api_failure_budget_spent' }[reason];
|
|
81
|
+
this.kind = 'calls_stopped';
|
|
82
|
+
this.reason = reason;
|
|
83
|
+
this.retryAfterMs = retryAfterMs;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
39
87
|
export function createSeazonaGovernor({
|
|
40
88
|
db,
|
|
41
89
|
clientKey,
|
|
42
90
|
requestsPerMinute = MAX_REQUESTS_PER_MINUTE,
|
|
43
91
|
maxWaitMs = 120_000,
|
|
44
92
|
telemetryRetentionMs = 35 * 24 * 60 * 60_000,
|
|
93
|
+
circuitFailures = 5,
|
|
94
|
+
circuitWindowMs = 10 * 60_000,
|
|
95
|
+
circuitPauseMs = 5 * 60_000,
|
|
96
|
+
dailyFailureBudget = 50,
|
|
45
97
|
now = Date.now,
|
|
46
98
|
sleepFn = sleep,
|
|
47
99
|
} = {}) {
|
|
@@ -59,6 +111,8 @@ export function createSeazonaGovernor({
|
|
|
59
111
|
const nowAt = new Date(nowMs);
|
|
60
112
|
await tx.insert(seazonaApiGovernor).values({ clientKey: key }).onConflictDoNothing();
|
|
61
113
|
const [state] = await tx.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
|
|
114
|
+
const stopped = await stoppedReason(tx, state, nowMs);
|
|
115
|
+
if (stopped) return { admitted: false, stopped };
|
|
62
116
|
const windowStart = new Date(nowMs - WINDOW_MS);
|
|
63
117
|
const [{ admitted, oldestMs }] = await tx
|
|
64
118
|
.select({ admitted: count(), oldestMs: sql`(extract(epoch from min(${seazonaApiRequests.admittedAt})) * 1000)::float8` })
|
|
@@ -84,12 +138,31 @@ export function createSeazonaGovernor({
|
|
|
84
138
|
});
|
|
85
139
|
}
|
|
86
140
|
|
|
141
|
+
async function failuresSince(tx, fromMs) {
|
|
142
|
+
const [{ failed }] = await tx.select({ failed: count() }).from(seazonaApiRequests)
|
|
143
|
+
.where(and(eq(seazonaApiRequests.clientKey, key), gte(seazonaApiRequests.admittedAt, new Date(fromMs)), failedCall));
|
|
144
|
+
return Number(failed);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Why no call may go now, or null.
|
|
148
|
+
async function stoppedReason(tx, state, nowMs) {
|
|
149
|
+
if (state.haltedReason) return { reason: 'halted', retryAfterMs: null };
|
|
150
|
+
const openUntil = state.circuitOpenUntil.getTime();
|
|
151
|
+
if (openUntil > nowMs) return { reason: 'circuit_open', retryAfterMs: openUntil - nowMs };
|
|
152
|
+
const dayStart = startOfUtcDay(nowMs);
|
|
153
|
+
if (await failuresSince(tx, dayStart) >= dailyFailureBudget) {
|
|
154
|
+
return { reason: 'budget_spent', retryAfterMs: dayStart + DAY_MS - nowMs };
|
|
155
|
+
}
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
|
|
87
159
|
// Waits for a slot and records the admission. Returns { requestId }.
|
|
88
160
|
async function reserve({ workflow = 'seazona', isRetry = false } = {}) {
|
|
89
161
|
let waitedMs = 0;
|
|
90
162
|
for (;;) {
|
|
91
163
|
const decision = await tryAdmit(workflow, isRetry);
|
|
92
164
|
if (decision.admitted) return { requestId: decision.requestId, waitedMs };
|
|
165
|
+
if (decision.stopped) throw new SeazonaCallsStopped(decision.stopped.reason, decision.stopped.retryAfterMs);
|
|
93
166
|
const delay = Math.max(1, Math.min(decision.waitMs, maxWaitMs));
|
|
94
167
|
if (waitedMs + delay > maxWaitMs) throw new SeazonaGovernorTimeout(waitedMs, delay);
|
|
95
168
|
waitedMs += delay;
|
|
@@ -97,25 +170,67 @@ export function createSeazonaGovernor({
|
|
|
97
170
|
}
|
|
98
171
|
}
|
|
99
172
|
|
|
100
|
-
// Records how a request ended,
|
|
173
|
+
// Records how a request ended, any cooldown a 429 asked for, and what the
|
|
174
|
+
// ending does to the circuit. Answers { halted, opened } when this response
|
|
175
|
+
// stopped calls, so the caller can report it once.
|
|
101
176
|
async function recordResponse({ requestId, path = null, statusCode = null, failureKind = null, durationMs = null, retryAfterMs = null } = {}) {
|
|
102
177
|
const nowMs = Number(now());
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
178
|
+
const nowAt = new Date(nowMs);
|
|
179
|
+
const outcome = await db.transaction(async (tx) => {
|
|
180
|
+
await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${lockKey}))`);
|
|
181
|
+
await tx.update(seazonaApiRequests).set({
|
|
182
|
+
requestPath: normaliseRequestPath(path),
|
|
183
|
+
statusCode: Number.isInteger(statusCode) ? statusCode : null,
|
|
184
|
+
failureKind: failureKind ? String(failureKind).slice(0, 40) : null,
|
|
185
|
+
durationMs: Number.isFinite(durationMs) ? Math.max(0, Math.round(durationMs)) : null,
|
|
186
|
+
}).where(and(eq(seazonaApiRequests.id, requestId), eq(seazonaApiRequests.clientKey, key)));
|
|
187
|
+
if (Number.isFinite(retryAfterMs) && retryAfterMs > 0) {
|
|
188
|
+
const until = new Date(nowMs + Math.min(retryAfterMs, MAX_COOLDOWN_MS));
|
|
189
|
+
await tx.update(seazonaApiGovernor)
|
|
190
|
+
.set({ cooldownUntil: sql`greatest(${seazonaApiGovernor.cooldownUntil}, ${until})`, updatedAt: nowAt })
|
|
191
|
+
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
192
|
+
}
|
|
193
|
+
const [state] = await tx.select().from(seazonaApiGovernor).where(eq(seazonaApiGovernor.clientKey, key));
|
|
194
|
+
if (!state) return {};
|
|
195
|
+
if (!countsAsFailure({ statusCode, failureKind })) {
|
|
196
|
+
if (state.circuitOpens > 0) {
|
|
197
|
+
await tx.update(seazonaApiGovernor).set({ circuitOpens: 0, updatedAt: nowAt }).where(eq(seazonaApiGovernor.clientKey, key));
|
|
198
|
+
}
|
|
199
|
+
return {};
|
|
200
|
+
}
|
|
201
|
+
if (statusCode === 401 || statusCode === 403) {
|
|
202
|
+
if (state.haltedReason) return {};
|
|
203
|
+
await tx.update(seazonaApiGovernor).set({ haltedReason: 'auth', haltedAt: nowAt, updatedAt: nowAt })
|
|
204
|
+
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
205
|
+
return { halted: 'auth' };
|
|
206
|
+
}
|
|
207
|
+
const openUntil = state.circuitOpenUntil.getTime();
|
|
208
|
+
// After a pause the next call is a probe: if it fails, open again at once.
|
|
209
|
+
const probeFailed = state.circuitOpens > 0 && openUntil <= nowMs;
|
|
210
|
+
const counted = probeFailed ? 0 : await failuresSince(tx, Math.max(nowMs - circuitWindowMs, openUntil));
|
|
211
|
+
if (!probeFailed && counted < circuitFailures) return {};
|
|
212
|
+
if (!probeFailed && openUntil > nowMs) return {};
|
|
213
|
+
const pauseMs = Math.min(MAX_CIRCUIT_PAUSE_MS, circuitPauseMs * 2 ** Math.min(state.circuitOpens, 10));
|
|
214
|
+
const until = new Date(nowMs + pauseMs);
|
|
215
|
+
await tx.update(seazonaApiGovernor)
|
|
216
|
+
.set({ circuitOpenUntil: until, circuitOpens: state.circuitOpens + 1, updatedAt: nowAt })
|
|
113
217
|
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
114
|
-
|
|
218
|
+
return { opened: { until: until.toISOString(), pauseMs, opens: state.circuitOpens + 1 } };
|
|
219
|
+
});
|
|
115
220
|
await db.delete(seazonaApiRequests).where(and(
|
|
116
221
|
eq(seazonaApiRequests.clientKey, key),
|
|
117
222
|
lt(seazonaApiRequests.admittedAt, new Date(nowMs - telemetryRetentionMs)),
|
|
118
223
|
));
|
|
224
|
+
return outcome;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// A person acted (new or saved settings, or the adapter turned on again):
|
|
228
|
+
// clear a halt and close the circuit. The day's failure budget stays.
|
|
229
|
+
async function resume() {
|
|
230
|
+
await db.insert(seazonaApiGovernor).values({ clientKey: key }).onConflictDoNothing();
|
|
231
|
+
await db.update(seazonaApiGovernor)
|
|
232
|
+
.set({ haltedReason: null, haltedAt: null, circuitOpenUntil: EPOCH, circuitOpens: 0, updatedAt: new Date(Number(now())) })
|
|
233
|
+
.where(eq(seazonaApiGovernor.clientKey, key));
|
|
119
234
|
}
|
|
120
235
|
|
|
121
236
|
// Counts for /health: the last minute, and the last day by status.
|
|
@@ -126,11 +241,21 @@ export function createSeazonaGovernor({
|
|
|
126
241
|
.where(and(eq(seazonaApiRequests.clientKey, key), gte(seazonaApiRequests.admittedAt, new Date(nowMs - WINDOW_MS))));
|
|
127
242
|
const day = await summarise({ from: new Date(nowMs - 24 * 60 * 60_000), to: new Date(nowMs) });
|
|
128
243
|
const cooldownUntil = state && state.cooldownUntil.getTime() > nowMs ? state.cooldownUntil.toISOString() : null;
|
|
244
|
+
const openUntil = state?.circuitOpenUntil?.getTime() ?? 0;
|
|
245
|
+
const circuit = state?.haltedReason ? 'halted'
|
|
246
|
+
: openUntil > nowMs ? 'open'
|
|
247
|
+
: state?.circuitOpens > 0 ? 'half_open' : 'closed';
|
|
129
248
|
return {
|
|
130
249
|
limitPerMinute: limit,
|
|
131
250
|
minIntervalMs,
|
|
132
251
|
lastMinute: Number(lastMinute),
|
|
133
252
|
cooldownUntil,
|
|
253
|
+
circuit,
|
|
254
|
+
circuitOpenUntil: openUntil > nowMs ? state.circuitOpenUntil.toISOString() : null,
|
|
255
|
+
haltedReason: state?.haltedReason ?? null,
|
|
256
|
+
haltedAt: state?.haltedAt?.toISOString() ?? null,
|
|
257
|
+
failedToday: await failuresSince(db, startOfUtcDay(nowMs)),
|
|
258
|
+
dailyFailureBudget,
|
|
134
259
|
lastDay: { requests: day.requests, failed: day.failed, rateLimited: day.rateLimited, retries: day.retries },
|
|
135
260
|
};
|
|
136
261
|
}
|
|
@@ -170,5 +295,5 @@ export function createSeazonaGovernor({
|
|
|
170
295
|
};
|
|
171
296
|
}
|
|
172
297
|
|
|
173
|
-
return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, status, summarise };
|
|
298
|
+
return { clientKey: key, limitPerMinute: limit, minIntervalMs, reserve, recordResponse, resume, status, summarise };
|
|
174
299
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { createSeazonaClient } from './client.js';
|
|
2
|
-
import { listClients, readBillableWork } from './feed.js';
|
|
2
|
+
import { listClients, listWork, readBillableWork } from './feed.js';
|
|
3
3
|
import { readSeazonaSyncHealth, runSeazonaSync } from './lanes.js';
|
|
4
4
|
import { createSeazonaScheduler } from './scheduler.js';
|
|
5
|
+
import { resetFailedLanes, resetStoppedLanes } from './store.js';
|
|
5
6
|
import { describeSeazonaSettings, resolveSeazonaSettings } from './settings.js';
|
|
6
7
|
import { seazonaSchema } from './tables.js';
|
|
7
8
|
|
|
@@ -58,9 +59,21 @@ export function createSeazonaAdapter({ env = null, fetchImpl = null, now = Date.
|
|
|
58
59
|
schema: seazonaSchema,
|
|
59
60
|
setup: SEAZONA_SETUP,
|
|
60
61
|
|
|
61
|
-
|
|
62
|
+
// `reconfigured` is true when a person saved the settings or turned
|
|
63
|
+
// Seazona on (issue #101): that clears a halt after a refused API client,
|
|
64
|
+
// closes the circuit, and runs every failed lane again. A plain start (a
|
|
65
|
+
// restart or a new release) runs again only the lanes stopped by a
|
|
66
|
+
// refused path; a halt still waits for a person.
|
|
67
|
+
async start({ db, log = console, env: settingsEnv, reconfigured = false }) {
|
|
62
68
|
const settings = requireSeazonaSettings(env ?? settingsEnv ?? process.env);
|
|
63
69
|
const client = createSeazonaClient({ settings, db, fetchImpl: fetchImpl ?? undefined, workflow: 'seazona-sync', now });
|
|
70
|
+
const at = new Date(Number(now()));
|
|
71
|
+
if (reconfigured) {
|
|
72
|
+
await client.governor?.resume();
|
|
73
|
+
await resetFailedLanes(db, { at });
|
|
74
|
+
} else {
|
|
75
|
+
await resetStoppedLanes(db, { at });
|
|
76
|
+
}
|
|
64
77
|
const run = (options = {}) => runSeazonaSync(db, client, { settings, now, ...options });
|
|
65
78
|
// The background sync starts in live mode only. A sandbox or replay
|
|
66
79
|
// installation syncs only when someone asks (sync-once.js).
|
|
@@ -105,6 +118,12 @@ export function createSeazonaAdapter({ env = null, fetchImpl = null, now = Date.
|
|
|
105
118
|
async listCustomers() {
|
|
106
119
|
return listClients(current().db);
|
|
107
120
|
},
|
|
121
|
+
|
|
122
|
+
// Open orders and orders finished lately, with their statuses, for the
|
|
123
|
+
// Jobs board. Reads Flow's copy only.
|
|
124
|
+
async listWork() {
|
|
125
|
+
return listWork(current().db, { now: now() });
|
|
126
|
+
},
|
|
108
127
|
},
|
|
109
128
|
};
|
|
110
129
|
}
|