@arcforgelabs/flow 2026.10.30 → 2026.10.32-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/npm-shrinkwrap.json +188 -2
- package/package.json +2 -1
- package/server/api/alerts.js +49 -0
- package/server/api/invoicing.js +110 -3
- package/server/app.js +29 -2
- package/server/core/alerts.js +157 -0
- package/server/core/history.js +65 -0
- package/server/core/invoice-pdf.js +266 -0
- package/server/core/invoicing-rules.js +12 -0
- package/server/core/job-board.js +25 -3
- package/server/core/line-mapping.js +36 -10
- package/server/core/pipeline.js +75 -3
- package/server/core/price-list.js +348 -0
- package/server/db/commands.js +42 -0
- package/server/db/doctor.js +168 -0
- package/server/db/migrations/0014_price_list.sql +34 -0
- package/server/db/migrations/0015_alert_dismissals.sql +17 -0
- package/server/db/migrations/meta/0014_snapshot.json +3773 -0
- package/server/db/migrations/meta/0015_snapshot.json +3854 -0
- package/server/db/migrations/meta/_journal.json +14 -0
- package/server/db/schema.js +44 -0
- package/server/flow-server.js +20 -1
- package/server/index.js +2 -1
- package/server/integrations/call-guard.js +362 -0
- package/server/integrations/port.js +20 -1
- package/server/integrations/seazona/feed.js +23 -1
- package/server/integrations/seazona/governor.js +50 -122
- package/server/integrations/seazona/history/backfill.js +208 -0
- package/server/integrations/seazona/history/csv.js +65 -0
- package/server/integrations/seazona/history/index.js +584 -0
- package/server/integrations/seazona/history/read.js +196 -0
- package/server/integrations/seazona/index.js +37 -1
- package/server/integrations/seazona/lanes.js +186 -50
- package/server/integrations/seazona/migrations/0003_call_guard.sql +2 -0
- package/server/integrations/seazona/migrations/0004_history_import.sql +68 -0
- package/server/integrations/seazona/migrations/meta/0003_snapshot.json +895 -0
- package/server/integrations/seazona/migrations/meta/0004_snapshot.json +1323 -0
- package/server/integrations/seazona/migrations/meta/_journal.json +14 -0
- package/server/integrations/seazona/routes.js +145 -0
- package/server/integrations/seazona/schema.js +94 -0
- package/server/integrations/seazona/settings.js +14 -3
- package/server/integrations/seazona/sync-once.js +1 -1
- package/server/integrations/seazona/tables.js +4 -0
- package/server/integrations/xero/client.js +27 -7
- package/server/integrations/xero/drafts.js +46 -8
- package/server/integrations/xero/import.js +125 -17
- package/server/integrations/xero/index.js +36 -7
- package/server/integrations/xero/lookup.js +26 -0
- package/server/integrations/xero/migrations/0006_xero_call_guard.sql +26 -0
- package/server/integrations/xero/migrations/meta/_journal.json +7 -0
- package/server/integrations/xero/mirror.js +11 -2
- package/server/integrations/xero/oauth.js +4 -1
- package/server/integrations/xero/reconcile.js +173 -0
- package/server/integrations/xero/routes.js +35 -0
- package/server/integrations/xero/runtime.js +45 -4
- package/server/release-channel.js +3 -3
- package/server/ui/alerts.js +77 -0
- package/server/ui/app-page.js +1 -1
- package/server/ui/app.js +48 -14
- package/server/ui/clerk-appearance.js +42 -9
- package/server/ui/flow.css +31 -2
- package/server/ui/invoicing-settings.js +1 -1
- package/server/ui/invoicing.js +121 -8
- package/server/ui/line-mappings.js +3 -2
- package/server/ui/price-list.js +247 -0
- package/server/ui/routes.js +1 -1
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { sql } from 'drizzle-orm';
|
|
3
|
-
import { enqueueJob, runNextJob } from '../../core/queue.js';
|
|
3
|
+
import { deadHoldSince, deadSince, enqueueJob, runNextJob } from '../../core/queue.js';
|
|
4
|
+
import { isCallsStopped } from '../call-guard.js';
|
|
4
5
|
import { lastSync, syncCatalogue } from './catalogue.js';
|
|
6
|
+
import { mirrorState } from './lookup.js';
|
|
5
7
|
import { MIRROR_RESOURCES, mirrorXero } from './mirror.js';
|
|
8
|
+
import { checkSentInvoices } from './reconcile.js';
|
|
6
9
|
|
|
7
|
-
// The history import. When an owner connects Xero,
|
|
8
|
-
// organisation's invoices, credit notes and payments into the
|
|
9
|
-
// (mirror.js) as a background job, so nobody runs a script by hand. The
|
|
10
|
-
// reads in slices; a slice that stops at the day's budget or its page
|
|
11
|
-
// queues the next slice.
|
|
10
|
+
// The history import, then the reads of changes. When an owner connects Xero,
|
|
11
|
+
// Flow copies the organisation's invoices, credit notes and payments into the
|
|
12
|
+
// mirror (mirror.js) as a background job, so nobody runs a script by hand. The
|
|
13
|
+
// job reads in slices; a slice that stops at the day's budget or its page
|
|
14
|
+
// limit queues the next slice. When every resource is read to its end, the
|
|
15
|
+
// same job is queued again for REFRESH_MS later and reads only what changed
|
|
16
|
+
// since its cursor (issue #131), so the copy stays fresh (lookup.js). It only
|
|
17
|
+
// reads from Xero.
|
|
12
18
|
|
|
13
19
|
export const IMPORT_KIND = 'xero.import';
|
|
14
20
|
const IMPORT_KEY = 'history';
|
|
@@ -18,6 +24,10 @@ const LEASE_MS = 30 * 60 * 1000;
|
|
|
18
24
|
// Xero's day limit resets on a rolling day; an hour is a cheap re-check.
|
|
19
25
|
const BUDGET_WAIT_MS = 60 * 60 * 1000;
|
|
20
26
|
const IDLE_MS = 30 * 1000;
|
|
27
|
+
// How often a finished copy reads Xero's changes again. Each read costs about
|
|
28
|
+
// one call per resource when little changed: 96 a day for each of invoices and
|
|
29
|
+
// credit notes, inside the reserve the mirror leaves (XERO_MIRROR_RESERVE_FRACTION).
|
|
30
|
+
export const REFRESH_MS = 15 * 60 * 1000;
|
|
21
31
|
// Contacts and items are read again once a day, so the matcher can link each
|
|
22
32
|
// customer (server/core/pipeline.js) without a person pressing Sync.
|
|
23
33
|
const CATALOGUE_EVERY_MS = 24 * 60 * 60 * 1000;
|
|
@@ -44,12 +54,33 @@ export async function organisationsToImport(db, mode) {
|
|
|
44
54
|
return rows.map((r) => r.organisation_id);
|
|
45
55
|
}
|
|
46
56
|
|
|
57
|
+
// The next read for one organisation: queued now unless one is open already.
|
|
58
|
+
export async function queueChangesRead(db, organisationId, { runAfter = null } = {}) {
|
|
59
|
+
return queueHistoryImport(db, organisationId, { reason: 'changes', runAfter });
|
|
60
|
+
}
|
|
61
|
+
|
|
47
62
|
// One slice of the import for one job. Answers what it read and whether more
|
|
48
63
|
// remains (`more`: 'budget' or 'max_pages').
|
|
64
|
+
// While the call guard stops Xero calls the slice does not fail, so it does
|
|
65
|
+
// not spend the job's attempts: it answers `more: 'stopped'` with the wait,
|
|
66
|
+
// or no wait for a stop only a person clears (connecting queues the import).
|
|
49
67
|
export async function runImportSlice(db, runtime, job, { maxPages = SLICE_PAGES } = {}) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
68
|
+
let results;
|
|
69
|
+
try {
|
|
70
|
+
results = await mirrorXero(db, runtime, job.organisationId, { maxPages });
|
|
71
|
+
} catch (error) {
|
|
72
|
+
if (!isCallsStopped(error)) throw error;
|
|
73
|
+
return { results: [], more: error.retryAfterMs === null ? null : 'stopped', retryAfterMs: error.retryAfterMs, stoppedBy: error.code };
|
|
74
|
+
}
|
|
75
|
+
// A refused resource waits for a person; it never queues another slice.
|
|
76
|
+
const stopped = results.find((r) => !['done', 'refused'].includes(r.stopped))?.stopped ?? null;
|
|
77
|
+
// Each invoice Flow sent, against what the copy now holds (issue #132):
|
|
78
|
+
// an edit, void or delete in Xero opens a review item. No Xero calls.
|
|
79
|
+
const invoices = results.find((r) => r.resource === 'invoices');
|
|
80
|
+
const sent = invoices && (invoices.inserted || invoices.changed || job.payload?.reason !== 'changes')
|
|
81
|
+
? await checkSentInvoices(db, runtime, job.organisationId)
|
|
82
|
+
: null;
|
|
83
|
+
return { results, more: stopped, ...(sent ? { sent } : {}) };
|
|
53
84
|
}
|
|
54
85
|
|
|
55
86
|
// Claims and runs one due import job, then queues the next slice when one is
|
|
@@ -61,33 +92,106 @@ export async function runImportOnce(db, runtime, { workerId = `xero-import-${ran
|
|
|
61
92
|
handlers: { [IMPORT_KIND]: (job) => runImportSlice(db, runtime, job, { maxPages }) },
|
|
62
93
|
});
|
|
63
94
|
if (ran?.outcome === 'succeeded' && ran.job.result?.more) {
|
|
64
|
-
const
|
|
95
|
+
const { more, retryAfterMs } = ran.job.result;
|
|
96
|
+
const wait = more === 'budget' ? BUDGET_WAIT_MS : more === 'stopped' ? Math.max(IDLE_MS, Number(retryAfterMs) || BUDGET_WAIT_MS) : 0;
|
|
65
97
|
await queueHistoryImport(db, ran.job.organisationId, { reason: ran.job.result.more, runAfter: new Date(Number(now()) + wait) });
|
|
98
|
+
} else if (ran?.outcome === 'succeeded' && !ran.job.result?.stoppedBy) {
|
|
99
|
+
// Read to the end: read the changes again later. A stop only a person
|
|
100
|
+
// clears queues nothing; connecting or saving the settings queues it.
|
|
101
|
+
await queueChangesRead(db, ran.job.organisationId, { runAfter: new Date(Number(now()) + REFRESH_MS) });
|
|
66
102
|
}
|
|
67
103
|
return ran;
|
|
68
104
|
}
|
|
69
105
|
|
|
106
|
+
// Failed catalogue reads in a row since the last good one, and the newest
|
|
107
|
+
// failure's code.
|
|
108
|
+
async function failuresInARow(db, organisationId) {
|
|
109
|
+
const { rows } = await db.execute(sql`
|
|
110
|
+
select count(*)::int as failures,
|
|
111
|
+
(array_agg(error_code order by started_at desc))[1] as last_code
|
|
112
|
+
from xero_sync_runs
|
|
113
|
+
where organisation_id = ${organisationId} and status = 'failed'
|
|
114
|
+
and started_at > coalesce((select max(started_at) from xero_sync_runs
|
|
115
|
+
where organisation_id = ${organisationId} and status = 'succeeded'), 'epoch')`);
|
|
116
|
+
return { failures: rows[0]?.failures ?? 0, lastCode: rows[0]?.last_code ?? null };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// The wait after `failures` failed reads in a row: an hour, doubling, at most
|
|
120
|
+
// a day (issue #111: never every hour for ever).
|
|
121
|
+
export function catalogueRetryMs(failures) {
|
|
122
|
+
return Math.min(CATALOGUE_EVERY_MS, BUDGET_WAIT_MS * 2 ** Math.max(0, failures - 1));
|
|
123
|
+
}
|
|
124
|
+
|
|
70
125
|
// Reads the contacts and items of each connected organisation whose last good
|
|
71
|
-
// read is older than a day. A
|
|
126
|
+
// read is older than a day. A failed read waits longer each time (above). A
|
|
127
|
+
// failure is reported once with log.error; the same failure again is a
|
|
128
|
+
// warning. While the call guard stops Xero calls, no read is tried.
|
|
72
129
|
export async function refreshStaleCatalogues(db, runtime, { now = Date.now, log = console } = {}) {
|
|
73
|
-
const { rows } = await db.execute(sql`select organisation_id from xero_connections where mode = ${runtime.settings.mode}`);
|
|
74
|
-
for (const { organisation_id: organisationId } of rows) {
|
|
130
|
+
const { rows } = await db.execute(sql`select organisation_id, tenant_id from xero_connections where mode = ${runtime.settings.mode}`);
|
|
131
|
+
for (const { organisation_id: organisationId, tenant_id: tenantId } of rows) {
|
|
75
132
|
const last = await lastSync(db, organisationId);
|
|
76
133
|
const at = last?.finishedAt ?? last?.startedAt;
|
|
77
|
-
|
|
134
|
+
const { failures, lastCode } = last?.status === 'failed' ? await failuresInARow(db, organisationId) : { failures: 0, lastCode: null };
|
|
135
|
+
if (last && at && Number(now()) - new Date(at) < (last.status === 'succeeded' ? CATALOGUE_EVERY_MS : catalogueRetryMs(failures))) continue;
|
|
136
|
+
try {
|
|
137
|
+
await runtime.guardFor(db).check(tenantId);
|
|
138
|
+
} catch (error) {
|
|
139
|
+
if (isCallsStopped(error)) continue;
|
|
140
|
+
throw error;
|
|
141
|
+
}
|
|
78
142
|
try {
|
|
79
143
|
const run = await syncCatalogue(db, runtime, organisationId);
|
|
80
144
|
log.info?.(`[xero] contacts and items read for organisation ${organisationId}: ${run.contactCount} contacts, ${run.itemCount} items`);
|
|
81
145
|
} catch (error) {
|
|
82
|
-
|
|
146
|
+
const code = error?.code ?? 'error';
|
|
147
|
+
const line = `[xero] reading contacts and items for organisation ${organisationId} failed: ${code}; next try in ${Math.round(catalogueRetryMs(failures + 1) / 60_000)} minutes`;
|
|
148
|
+
// The guard reported a stop already; a repeat of the same failure is a warning.
|
|
149
|
+
if (isCallsStopped(error) || (failures > 0 && code === lastCode)) log.warn?.(line);
|
|
150
|
+
else log.error?.(line);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Keeps a read queued for each connected organisation. A read that gave up
|
|
156
|
+
// (dead) waits for a person, a restart or the next day (issue #101), and
|
|
157
|
+
// while the call guard stops Xero calls no read is queued.
|
|
158
|
+
export async function keepReadsQueued(db, runtime, { startedAtMs = Date.now(), now = Date.now } = {}) {
|
|
159
|
+
const { rows } = await db.execute(sql`select organisation_id, tenant_id from xero_connections where mode = ${runtime.settings.mode}`);
|
|
160
|
+
for (const { organisation_id: organisationId, tenant_id: tenantId } of rows) {
|
|
161
|
+
if (await deadSince(db, organisationId, { kind: IMPORT_KIND, dedupeKey: IMPORT_KEY, since: deadHoldSince(startedAtMs, Number(now())) })) continue;
|
|
162
|
+
try {
|
|
163
|
+
await runtime.guardFor(db).check(tenantId);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if (isCallsStopped(error)) continue;
|
|
166
|
+
throw error;
|
|
167
|
+
}
|
|
168
|
+
await queueChangesRead(db, organisationId);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Reports a copy that went stale once with log.error, and its recovery with
|
|
173
|
+
// log.info. `stale` is the set of organisations already reported.
|
|
174
|
+
export async function reportStaleCopies(db, runtime, { stale = new Set(), now = Date.now, log = console } = {}) {
|
|
175
|
+
const { rows } = await db.execute(sql`select organisation_id from xero_connections where mode = ${runtime.settings.mode}`);
|
|
176
|
+
for (const { organisation_id: organisationId } of rows) {
|
|
177
|
+
const state = await mirrorState(db, organisationId, { now: new Date(Number(now())) });
|
|
178
|
+
if (!state.complete) continue;
|
|
179
|
+
if (!state.fresh && !stale.has(organisationId)) {
|
|
180
|
+
stale.add(organisationId);
|
|
181
|
+
log.error?.(`[xero] the copy of Xero for organisation ${organisationId} is out of date (last full read ${state.lastPassAt?.toISOString() ?? 'never'}); drafts by rule wait until it is read again`);
|
|
182
|
+
} else if (state.fresh && stale.delete(organisationId)) {
|
|
183
|
+
log.info?.(`[xero] the copy of Xero for organisation ${organisationId} is up to date again`);
|
|
83
184
|
}
|
|
84
185
|
}
|
|
186
|
+
return stale;
|
|
85
187
|
}
|
|
86
188
|
|
|
87
189
|
// The background worker: runs due slices one at a time, then waits. Started
|
|
88
190
|
// by the adapter outside replay mode; stop() waits for a slice in flight.
|
|
89
191
|
export function createImportWorker({ db, runtime, log = console, timers = { setTimeout, clearTimeout }, firstDelayMs = 10_000 }) {
|
|
90
192
|
const workerId = `xero-import-${randomUUID()}`;
|
|
193
|
+
const startedAtMs = Date.now();
|
|
194
|
+
const stale = new Set();
|
|
91
195
|
let timer = null;
|
|
92
196
|
let running = null;
|
|
93
197
|
let stopped = true;
|
|
@@ -105,12 +209,14 @@ export function createImportWorker({ db, runtime, log = console, timers = { setT
|
|
|
105
209
|
running = (async () => {
|
|
106
210
|
try {
|
|
107
211
|
await refreshStaleCatalogues(db, runtime, { log });
|
|
212
|
+
await keepReadsQueued(db, runtime, { startedAtMs });
|
|
213
|
+
await reportStaleCopies(db, runtime, { stale, log });
|
|
108
214
|
const ran = await runImportOnce(db, runtime, { workerId });
|
|
109
215
|
if (ran) {
|
|
110
216
|
busy = true;
|
|
111
217
|
const read = (ran.job.result?.results ?? []).map((r) => `${r.resource} ${r.read}`).join(', ');
|
|
112
|
-
if (ran.outcome === 'dead') log.error?.('[xero]
|
|
113
|
-
log.info?.(`[xero]
|
|
218
|
+
if (ran.outcome === 'dead') log.error?.('[xero] reading Xero gave up after its last attempt');
|
|
219
|
+
log.info?.(`[xero] read ${ran.outcome} for organisation ${ran.job.organisationId}${read ? `: ${read}` : ''}${ran.job.result?.more ? `; more (${ran.job.result.more})` : ''}`);
|
|
114
220
|
}
|
|
115
221
|
} catch (error) {
|
|
116
222
|
log.error?.(`[xero] import worker: ${error?.code ?? 'error'}: ${error?.message ?? error}`);
|
|
@@ -127,6 +233,8 @@ export function createImportWorker({ db, runtime, log = console, timers = { setT
|
|
|
127
233
|
for (const organisationId of await organisationsToImport(db, runtime.settings.mode)) {
|
|
128
234
|
await queueHistoryImport(db, organisationId, { reason: 'start' });
|
|
129
235
|
}
|
|
236
|
+
// A finished copy reads its changes again from the first tick on
|
|
237
|
+
// (keepReadsQueued), so an upgrade or a restart needs no step.
|
|
130
238
|
schedule(firstDelayMs);
|
|
131
239
|
},
|
|
132
240
|
async stop() {
|
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
import { fileURLToPath } from 'node:url';
|
|
2
2
|
import { sql } from 'drizzle-orm';
|
|
3
|
-
import {
|
|
3
|
+
import { guardAlerts } from '../call-guard.js';
|
|
4
|
+
import { checkDrafts, documentFacts, pushInvoice, pushInvoices } from './drafts.js';
|
|
4
5
|
import { readConnection } from './connections.js';
|
|
5
6
|
import { createImportWorker } from './import.js';
|
|
6
7
|
import { readPayments, xeroLiveCheck } from './ledger.js';
|
|
7
|
-
import { findInvoices, listCustomers } from './lookup.js';
|
|
8
|
+
import { findInvoices, listCustomers, listPriceItems, mirrorState } from './lookup.js';
|
|
8
9
|
import { createXeroRouter } from './routes.js';
|
|
9
|
-
import { createXeroRuntime, requireXeroSettings } from './runtime.js';
|
|
10
|
+
import { createXeroRuntime, requireXeroSettings, XERO_CALL_GUARD } from './runtime.js';
|
|
10
11
|
import { writesAllowed } from './settings.js';
|
|
11
12
|
|
|
12
13
|
// The Xero adapter (docs/XERO.md). FLOW_INTEGRATIONS=xero turns it on and
|
|
13
14
|
// XERO_MODE says which Xero it acts on. Its tables are created only while it
|
|
14
15
|
// is on (docs/INTEGRATIONS.md).
|
|
15
16
|
|
|
16
|
-
export const XERO_TABLES = Object.freeze(['xero_connections', 'xero_contacts', 'xero_items', 'xero_sync_runs', 'xero_invoice_pushes', 'xero_payment_cursors', 'xero_invoices', 'xero_credit_notes', 'xero_payments', 'xero_mirror_cursors']);
|
|
17
|
+
export const XERO_TABLES = Object.freeze(['xero_connections', 'xero_contacts', 'xero_items', 'xero_sync_runs', 'xero_invoice_pushes', 'xero_payment_cursors', 'xero_invoices', 'xero_credit_notes', 'xero_payments', 'xero_mirror_cursors', 'xero_call_guard', 'xero_calls']);
|
|
17
18
|
|
|
18
19
|
// What an owner enters in Settings, Integrations (server/integrations/setup.js).
|
|
19
20
|
// The refresh token's key comes from Flow's secrets key.
|
|
@@ -61,10 +62,16 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
|
|
|
61
62
|
tables: [...XERO_TABLES],
|
|
62
63
|
},
|
|
63
64
|
setup: XERO_SETUP,
|
|
64
|
-
|
|
65
|
+
// Every Xero call goes through this call guard (docs/INTEGRATIONS.md,
|
|
66
|
+
// "Outbound calls").
|
|
67
|
+
callGuard: XERO_CALL_GUARD,
|
|
68
|
+
// `reconfigured` is true when a person saved the settings or turned Xero
|
|
69
|
+
// on: that clears a halt, stopped resources and an open circuit.
|
|
70
|
+
async start({ db, log, env, reconfigured = false }) {
|
|
65
71
|
database = db;
|
|
66
72
|
if (!given) runtime = createXeroRuntime({ settings: requireXeroSettings(env ?? process.env) });
|
|
67
73
|
const { settings } = current();
|
|
74
|
+
if (reconfigured) await current().guardFor(db).resume();
|
|
68
75
|
log.info?.(`[xero] mode ${settings.mode}; writes ${writesAllowed(settings) ? 'on' : 'off (XERO_ALLOW_LIVE_WRITES)'}; redirect URI ${settings.redirectUri ?? 'not set (FLOW_PUBLIC_ORIGIN)'}`);
|
|
69
76
|
// The history import (import.js) runs in the background outside replay
|
|
70
77
|
// mode. Replay tests run its slices directly.
|
|
@@ -81,8 +88,25 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
|
|
|
81
88
|
capabilities: {
|
|
82
89
|
async health() {
|
|
83
90
|
const { settings } = current();
|
|
84
|
-
const { rows } = await database.execute(sql`select
|
|
85
|
-
|
|
91
|
+
const { rows } = await database.execute(sql`select organisation_id, tenant_id from xero_connections where mode = ${settings.mode}`);
|
|
92
|
+
const guard = current().guardFor(database);
|
|
93
|
+
const calls = await Promise.all(rows.map(async (r) => ({ tenantId: r.tenant_id, ...await guard.status(r.tenant_id) })));
|
|
94
|
+
const stopped = calls.some((c) => c.circuit !== 'closed' || c.stoppedResources.length || c.failedToday >= c.dailyFailureBudget);
|
|
95
|
+
// How fresh each copy of Xero is (lookup.js): a complete copy that is
|
|
96
|
+
// not fresh stops drafts by rule (issue #131).
|
|
97
|
+
const copies = await Promise.all(rows.map(async (r) => {
|
|
98
|
+
const { complete, fresh, lastPassAt } = await mirrorState(database, r.organisation_id);
|
|
99
|
+
return { tenantId: r.tenant_id, complete, fresh, lastPassAt };
|
|
100
|
+
}));
|
|
101
|
+
const stale = copies.some((c) => c.complete && !c.fresh);
|
|
102
|
+
return { ok: !stopped && !stale, mode: settings.mode, writesAllowed: writesAllowed(settings), connected: rows.length, calls, copies };
|
|
103
|
+
},
|
|
104
|
+
// What only a person can clear, for the Integrations page
|
|
105
|
+
// (server/core/alerts.js): a refused connection stops calls.
|
|
106
|
+
async alerts({ organisationId }) {
|
|
107
|
+
const connection = await readConnection(database, organisationId);
|
|
108
|
+
if (!connection || connection.mode !== current().settings.mode) return [];
|
|
109
|
+
return guardAlerts('Xero', await current().guardFor(database).status(connection.tenantId));
|
|
86
110
|
},
|
|
87
111
|
// One issued invoice to a Xero draft (drafts.js). The Flow invoice id
|
|
88
112
|
// and version are the idempotency key.
|
|
@@ -96,6 +120,9 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
|
|
|
96
120
|
const connection = await readConnection(database, organisationId);
|
|
97
121
|
return { connected: Boolean(connection && connection.mode === settings.mode), writesAllowed: writesAllowed(settings) };
|
|
98
122
|
},
|
|
123
|
+
// What the customer's Xero copy of a Flow invoice version says, from
|
|
124
|
+
// Flow's copy of Xero (drafts.js; no calls). The invoice PDF uses it.
|
|
125
|
+
documentFacts: ({ organisationId, invoiceId, version = null }) => documentFacts(database, current(), { organisationId, invoiceId, version }),
|
|
99
126
|
// Sales invoice payments changed since `since` (ledger.js). The answer's
|
|
100
127
|
// cursor is the next `since`.
|
|
101
128
|
pullPayments: ({ organisationId, since }) => readPayments(database, current(), organisationId, { since: new Date(since) }),
|
|
@@ -106,6 +133,8 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
|
|
|
106
133
|
listCustomers: ({ organisationId }) => listCustomers(database, organisationId),
|
|
107
134
|
// Sales invoices from the mirror, with the mirror's state (lookup.js).
|
|
108
135
|
findInvoices: ({ organisationId, ...query }) => findInvoices(database, organisationId, query),
|
|
136
|
+
// Xero items, from Flow's copy, for the price list (lookup.js).
|
|
137
|
+
listPriceItems: ({ organisationId }) => listPriceItems(database, organisationId, { taxTypes: current().settings.taxTypes }),
|
|
109
138
|
},
|
|
110
139
|
routes: ({ db, personAuth, log, messagePage }) => createXeroRouter({ db, personAuth, runtime: current(), log, messagePage }),
|
|
111
140
|
};
|
|
@@ -22,6 +22,32 @@ export async function listCustomers(db, organisationId) {
|
|
|
22
22
|
return rows.map((r) => ({ id: r.xero_contact_id, name: r.name }));
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
// Xero's items, from Flow's copy (catalogue.js, read on each sync), for the
|
|
26
|
+
// price list's links and price check (server/core/price-list.js). A tax type
|
|
27
|
+
// that charges GST reads as GST; any other set type reads as GST free.
|
|
28
|
+
const GST_TAX_TYPES = new Set(['OUTPUT', 'OUTPUT2']);
|
|
29
|
+
export async function listPriceItems(db, organisationId, { taxTypes = {} } = {}) {
|
|
30
|
+
const { rows } = await db.execute(sql`
|
|
31
|
+
select xero_item_id, code, name, description, is_sold, sales_unit_amount_cents, sales_tax_type, synced_at
|
|
32
|
+
from xero_items where organisation_id = ${organisationId} order by code`);
|
|
33
|
+
const taxOf = (type) => {
|
|
34
|
+
if (!type) return null;
|
|
35
|
+
return type === taxTypes.GST || GST_TAX_TYPES.has(type) ? 'GST' : 'GST_FREE';
|
|
36
|
+
};
|
|
37
|
+
return {
|
|
38
|
+
kind: 'accounting',
|
|
39
|
+
syncedAt: rows.reduce((at, r) => (!at || r.synced_at > at ? r.synced_at : at), null),
|
|
40
|
+
items: rows.map((r) => ({
|
|
41
|
+
id: r.xero_item_id,
|
|
42
|
+
code: r.code,
|
|
43
|
+
name: r.name || r.description || r.code,
|
|
44
|
+
unitAmountCents: r.sales_unit_amount_cents,
|
|
45
|
+
taxCode: taxOf(r.sales_tax_type),
|
|
46
|
+
sold: r.is_sold,
|
|
47
|
+
})),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
25
51
|
// Whether the mirror holds the whole history and was read lately. Until then a
|
|
26
52
|
// search that finds nothing proves nothing.
|
|
27
53
|
export async function mirrorState(db, organisationId, { now = new Date() } = {}) {
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
-- The call guard's state and telemetry (server/integrations/call-guard.js,
|
|
2
|
+
-- issue 111): one row per Xero organisation (tenant) with the circuit, a halt
|
|
3
|
+
-- and the resources Xero refused, and one row per call sent. A call row keeps
|
|
4
|
+
-- the resource and path, never the host, the query, a token or a response.
|
|
5
|
+
CREATE TABLE IF NOT EXISTS "xero_call_guard" (
|
|
6
|
+
"key" text PRIMARY KEY NOT NULL,
|
|
7
|
+
"circuit_open_until" timestamp with time zone DEFAULT 'epoch' NOT NULL,
|
|
8
|
+
"circuit_opens" integer DEFAULT 0 NOT NULL,
|
|
9
|
+
"halted_reason" text,
|
|
10
|
+
"halted_at" timestamp with time zone,
|
|
11
|
+
"stopped_resources" jsonb DEFAULT '{}'::jsonb NOT NULL,
|
|
12
|
+
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
|
|
13
|
+
);
|
|
14
|
+
--> statement-breakpoint
|
|
15
|
+
CREATE TABLE IF NOT EXISTS "xero_calls" (
|
|
16
|
+
"id" bigserial PRIMARY KEY NOT NULL,
|
|
17
|
+
"key" text NOT NULL,
|
|
18
|
+
"at" timestamp with time zone NOT NULL,
|
|
19
|
+
"resource" text,
|
|
20
|
+
"request_path" text,
|
|
21
|
+
"status_code" integer,
|
|
22
|
+
"failure_kind" text,
|
|
23
|
+
"duration_ms" integer
|
|
24
|
+
);
|
|
25
|
+
--> statement-breakpoint
|
|
26
|
+
CREATE INDEX IF NOT EXISTS "xero_calls_window" ON "xero_calls" USING btree ("key","at");
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { sql } from 'drizzle-orm';
|
|
3
|
+
import { isCallsStopped } from '../call-guard.js';
|
|
3
4
|
import { parseXeroDate } from './catalogue.js';
|
|
4
5
|
import { withXeroLane } from './governor.js';
|
|
5
6
|
|
|
@@ -257,12 +258,20 @@ export async function mirrorResource(db, runtime, organisationId, resource, { ma
|
|
|
257
258
|
}
|
|
258
259
|
|
|
259
260
|
// Reads each resource in turn, one organisation at a time with any other Xero
|
|
260
|
-
// work for it (runtime.serial).
|
|
261
|
+
// work for it (runtime.serial). A resource Xero refused (the call guard
|
|
262
|
+
// stopped it: the connection lacks its scope) is skipped as `refused`, so
|
|
263
|
+
// the others still read.
|
|
261
264
|
export async function mirrorXero(db, runtime, organisationId, { resources = MIRROR_RESOURCES, maxPages } = {}) {
|
|
262
265
|
return runtime.serial(organisationId, async () => {
|
|
263
266
|
const results = [];
|
|
264
267
|
for (const resource of resources) {
|
|
265
|
-
|
|
268
|
+
let result;
|
|
269
|
+
try {
|
|
270
|
+
result = await mirrorResource(db, runtime, organisationId, resource, { maxPages });
|
|
271
|
+
} catch (error) {
|
|
272
|
+
if (!isCallsStopped(error) || error.reason !== 'resource_stopped') throw error;
|
|
273
|
+
result = { resource, pages: 0, read: 0, inserted: 0, changed: 0, unchanged: 0, stopped: 'refused' };
|
|
274
|
+
}
|
|
266
275
|
results.push(result);
|
|
267
276
|
if (result.stopped === 'budget') break;
|
|
268
277
|
}
|
|
@@ -53,7 +53,10 @@ async function tokenRequest(settings, fetchImpl, form, what) {
|
|
|
53
53
|
if (body?.error === 'invalid_grant') {
|
|
54
54
|
throw xeroError('xero_reconnect_needed', 'Xero no longer accepts this connection. An owner must connect Xero again.', { status: 409 });
|
|
55
55
|
}
|
|
56
|
-
throw xeroError('xero_token_failed', `Xero refused ${what} (${response.status}${body?.error ? `, ${body.error}` : ''}).`, {
|
|
56
|
+
throw xeroError('xero_token_failed', `Xero refused ${what} (${response.status}${body?.error ? `, ${body.error}` : ''}).`, {
|
|
57
|
+
status: 502,
|
|
58
|
+
oauthError: typeof body?.error === 'string' ? body.error.slice(0, 40) : null,
|
|
59
|
+
});
|
|
57
60
|
}
|
|
58
61
|
if (!body?.access_token || !body?.refresh_token) {
|
|
59
62
|
throw xeroError('xero_token_failed', `Xero's answer to ${what} had no tokens.`, { status: 502 });
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { sql } from 'drizzle-orm';
|
|
2
|
+
import { openReview, resolveReviewFor } from '../../core/review.js';
|
|
3
|
+
|
|
4
|
+
// Each invoice Flow sent, against Flow's copy of Xero (issue #132). People
|
|
5
|
+
// change invoices in both places after Flow sends them: a Xero draft is edited
|
|
6
|
+
// before approval, an invoice is voided in Xero, or voided in Flow. The issued
|
|
7
|
+
// version Flow sent is the merge base: the document both sides agreed on.
|
|
8
|
+
// This module compares the Xero copy with it and with the Flow invoice now.
|
|
9
|
+
// It never calls Xero and never writes to Xero or to the Flow invoice: a
|
|
10
|
+
// difference opens one review item (alert only), and a person fixes it where
|
|
11
|
+
// they choose. The item closes when a later check finds no difference.
|
|
12
|
+
|
|
13
|
+
export const OUT_OF_SYNC = 'invoice.accounting_differs';
|
|
14
|
+
export const ACTOR = 'system:xero-reconcile';
|
|
15
|
+
|
|
16
|
+
// Xero's status in the words the Flow screens show.
|
|
17
|
+
const LABELS = {
|
|
18
|
+
not_sent: 'Not in Xero',
|
|
19
|
+
sending: 'Sending to Xero',
|
|
20
|
+
not_copied: 'Sent, not read back yet',
|
|
21
|
+
draft: 'Draft in Xero',
|
|
22
|
+
approved: 'Approved in Xero',
|
|
23
|
+
part_paid: 'Part paid in Xero',
|
|
24
|
+
paid: 'Paid in Xero',
|
|
25
|
+
voided: 'Voided in Xero',
|
|
26
|
+
deleted: 'Deleted in Xero',
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const GONE = new Set(['VOIDED', 'DELETED']);
|
|
30
|
+
|
|
31
|
+
function xeroState(row) {
|
|
32
|
+
switch (row.status) {
|
|
33
|
+
case 'DRAFT':
|
|
34
|
+
case 'SUBMITTED':
|
|
35
|
+
return 'draft';
|
|
36
|
+
case 'AUTHORISED':
|
|
37
|
+
return Number(row.amount_paid_cents ?? 0) > 0 ? 'part_paid' : 'approved';
|
|
38
|
+
case 'PAID':
|
|
39
|
+
return 'paid';
|
|
40
|
+
case 'VOIDED':
|
|
41
|
+
return 'voided';
|
|
42
|
+
case 'DELETED':
|
|
43
|
+
return 'deleted';
|
|
44
|
+
default:
|
|
45
|
+
return 'draft';
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const num = (value) => (value === null || value === undefined ? null : Number(value));
|
|
50
|
+
const day = (value) => (value instanceof Date ? value.toISOString().slice(0, 10) : value ? String(value).slice(0, 10) : null);
|
|
51
|
+
const codes = (lines) => [...(lines ?? []).map((l) => l.itemCode ?? '')].sort().join(',');
|
|
52
|
+
|
|
53
|
+
// What differs between the Xero copy and the version Flow sent. Free text
|
|
54
|
+
// (reference, descriptions) is not compared: the copy keeps it only when the
|
|
55
|
+
// lab turns that on (XERO_KEEP_REFERENCE).
|
|
56
|
+
export function sentDifferences(sent, document, copy) {
|
|
57
|
+
const found = [];
|
|
58
|
+
const add = (field, flow, xero) => found.push({ field, flow, xero });
|
|
59
|
+
if (sent?.Contact?.ContactID && copy.xero_contact_id !== sent.Contact.ContactID) add('contact', sent.Contact.ContactID, copy.xero_contact_id);
|
|
60
|
+
if (num(copy.subtotal_cents) !== document.subtotalCents) add('subtotal', document.subtotalCents, num(copy.subtotal_cents));
|
|
61
|
+
if (num(copy.tax_cents) !== document.taxCents) add('tax', document.taxCents, num(copy.tax_cents));
|
|
62
|
+
if (num(copy.total_cents) !== document.totalCents) add('total', document.totalCents, num(copy.total_cents));
|
|
63
|
+
const lines = copy.lines ?? [];
|
|
64
|
+
if (lines.length !== document.lines.length) add('line count', document.lines.length, lines.length);
|
|
65
|
+
else if (codes(lines) !== codes(document.lines)) add('items', codes(document.lines), codes(lines));
|
|
66
|
+
if (sent?.Date && day(copy.issued_on) !== sent.Date) add('date', sent.Date, day(copy.issued_on));
|
|
67
|
+
return found;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// The finding for one invoice, or null when Flow and Xero agree.
|
|
71
|
+
// differs Xero's copy no longer matches the version Flow sent
|
|
72
|
+
// voided_in_xero Xero voided or deleted it; the Flow invoice is still issued
|
|
73
|
+
// voided_in_flow Flow voided it; Xero's invoice is still live
|
|
74
|
+
function findingFor(status) {
|
|
75
|
+
if (!status.xero) return null;
|
|
76
|
+
const live = !GONE.has(status.xero.status);
|
|
77
|
+
if (status.flowStatus === 'void' && live) return 'voided_in_flow';
|
|
78
|
+
if (status.flowStatus === 'issued' && status.current && !live) return 'voided_in_xero';
|
|
79
|
+
if (live && status.differences.length) return 'differs';
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const MESSAGES = {
|
|
84
|
+
differs: 'Xero no longer matches the invoice Flow sent.',
|
|
85
|
+
voided_in_xero: 'Xero voided or deleted this invoice. It is still issued in Flow.',
|
|
86
|
+
voided_in_flow: 'This invoice is void in Flow. Xero still holds it as a live invoice.',
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// Each invoice's Xero status, from Flow's copy (no Xero calls). Answers a map
|
|
90
|
+
// from Flow invoice id to { state, label, xero, differences, finding, message }.
|
|
91
|
+
export async function accountingStatus(db, runtime, organisationId, invoiceIds = null) {
|
|
92
|
+
const only = invoiceIds ? sql`and i.id = any(${sql.param([...new Set(invoiceIds.map(String))])}::uuid[])` : sql``;
|
|
93
|
+
// Without ids: only invoices Flow sent, for the check after each read.
|
|
94
|
+
const sentOnly = invoiceIds ? sql`` : sql`and p.id is not null`;
|
|
95
|
+
const { rows } = await db.execute(sql`
|
|
96
|
+
select i.id, i.status, i.issued_version, i.total_cents,
|
|
97
|
+
p.version as sent_version, p.state as push_state, p.request_body, p.xero_invoice_id, p.xero_invoice_number,
|
|
98
|
+
v.document,
|
|
99
|
+
x.status as x_status, x.invoice_number as x_number, x.xero_contact_id, x.subtotal_cents, x.tax_cents,
|
|
100
|
+
x.total_cents as x_total_cents, x.amount_paid_cents, x.amount_due_cents, x.lines, x.issued_on
|
|
101
|
+
from invoices i
|
|
102
|
+
left join lateral (
|
|
103
|
+
select * from xero_invoice_pushes
|
|
104
|
+
where organisation_id = i.organisation_id and invoice_id = i.id
|
|
105
|
+
order by version desc limit 1
|
|
106
|
+
) p on true
|
|
107
|
+
left join invoice_versions v on v.invoice_id = i.id and v.version = p.version
|
|
108
|
+
left join xero_invoices x on x.organisation_id = i.organisation_id and x.xero_invoice_id = p.xero_invoice_id
|
|
109
|
+
where i.organisation_id = ${organisationId} ${only} ${sentOnly}`);
|
|
110
|
+
const web = runtime.settings.endpoints?.web ?? 'https://go.xero.com';
|
|
111
|
+
const out = new Map();
|
|
112
|
+
for (const r of rows) {
|
|
113
|
+
let state;
|
|
114
|
+
if (!r.push_state) state = 'not_sent';
|
|
115
|
+
else if (['dispatching', 'ambiguous'].includes(r.push_state)) state = 'sending';
|
|
116
|
+
else if (!r.x_status) state = 'not_copied';
|
|
117
|
+
else state = xeroState({ status: r.x_status, amount_paid_cents: r.amount_paid_cents });
|
|
118
|
+
let sent = null;
|
|
119
|
+
try {
|
|
120
|
+
sent = r.request_body ? JSON.parse(r.request_body) : null;
|
|
121
|
+
} catch {
|
|
122
|
+
sent = null;
|
|
123
|
+
}
|
|
124
|
+
const xero = r.x_status ? {
|
|
125
|
+
invoiceId: r.xero_invoice_id,
|
|
126
|
+
number: r.x_number ?? r.xero_invoice_number ?? null,
|
|
127
|
+
status: r.x_status,
|
|
128
|
+
totalCents: num(r.x_total_cents),
|
|
129
|
+
amountDueCents: num(r.amount_due_cents),
|
|
130
|
+
url: `${web}/AccountsReceivable/View.aspx?InvoiceID=${encodeURIComponent(r.xero_invoice_id)}`,
|
|
131
|
+
} : null;
|
|
132
|
+
const status = {
|
|
133
|
+
invoiceId: r.id,
|
|
134
|
+
flowStatus: r.status,
|
|
135
|
+
sentVersion: r.sent_version ?? null,
|
|
136
|
+
current: r.sent_version !== null && r.sent_version === r.issued_version,
|
|
137
|
+
state,
|
|
138
|
+
label: LABELS[state],
|
|
139
|
+
xero,
|
|
140
|
+
differences: xero && r.document && !GONE.has(r.x_status) ? sentDifferences(sent, r.document, { ...r, total_cents: r.x_total_cents }) : [],
|
|
141
|
+
flowTotalCents: num(r.total_cents),
|
|
142
|
+
};
|
|
143
|
+
status.finding = findingFor(status);
|
|
144
|
+
status.message = status.finding ? MESSAGES[status.finding] : null;
|
|
145
|
+
out.set(r.id, status);
|
|
146
|
+
}
|
|
147
|
+
return out;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Opens or refreshes one review item for each finding, and closes the item of
|
|
151
|
+
// an invoice whose finding went away. Answers counts.
|
|
152
|
+
export async function checkSentInvoices(db, runtime, organisationId, { invoiceIds = null } = {}) {
|
|
153
|
+
const statuses = await accountingStatus(db, runtime, organisationId, invoiceIds);
|
|
154
|
+
const result = { checked: 0, open: 0, closed: 0 };
|
|
155
|
+
for (const status of statuses.values()) {
|
|
156
|
+
if (status.state === 'not_sent') continue;
|
|
157
|
+
result.checked += 1;
|
|
158
|
+
if (status.finding) {
|
|
159
|
+
await openReview(db, organisationId, {
|
|
160
|
+
kind: OUT_OF_SYNC,
|
|
161
|
+
entity: 'invoice',
|
|
162
|
+
flowId: status.invoiceId,
|
|
163
|
+
systems: ['flow', 'xero'],
|
|
164
|
+
evidence: { finding: status.finding, message: status.message, flowStatus: status.flowStatus, sentVersion: status.sentVersion, xero: status.xero, differences: status.differences },
|
|
165
|
+
moneyAtRiskCents: Math.abs(status.xero?.totalCents ?? status.flowTotalCents ?? 0),
|
|
166
|
+
});
|
|
167
|
+
result.open += 1;
|
|
168
|
+
} else if (await resolveReviewFor(db, organisationId, { kind: OUT_OF_SYNC, entity: 'invoice', flowId: status.invoiceId, resolution: 'no_difference', actor: ACTOR })) {
|
|
169
|
+
result.closed += 1;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return result;
|
|
173
|
+
}
|