@arcforgelabs/flow 2026.10.30 → 2026.10.32-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +2 -0
  2. package/npm-shrinkwrap.json +188 -2
  3. package/package.json +2 -1
  4. package/server/api/alerts.js +49 -0
  5. package/server/api/invoicing.js +110 -3
  6. package/server/app.js +29 -2
  7. package/server/core/alerts.js +157 -0
  8. package/server/core/history.js +65 -0
  9. package/server/core/invoice-pdf.js +266 -0
  10. package/server/core/invoicing-rules.js +12 -0
  11. package/server/core/job-board.js +25 -3
  12. package/server/core/line-mapping.js +36 -10
  13. package/server/core/pipeline.js +75 -3
  14. package/server/core/price-list.js +348 -0
  15. package/server/db/commands.js +42 -0
  16. package/server/db/doctor.js +168 -0
  17. package/server/db/migrations/0014_price_list.sql +34 -0
  18. package/server/db/migrations/0015_alert_dismissals.sql +17 -0
  19. package/server/db/migrations/meta/0014_snapshot.json +3773 -0
  20. package/server/db/migrations/meta/0015_snapshot.json +3854 -0
  21. package/server/db/migrations/meta/_journal.json +14 -0
  22. package/server/db/schema.js +44 -0
  23. package/server/flow-server.js +20 -1
  24. package/server/index.js +2 -1
  25. package/server/integrations/call-guard.js +362 -0
  26. package/server/integrations/port.js +20 -1
  27. package/server/integrations/seazona/feed.js +23 -1
  28. package/server/integrations/seazona/governor.js +50 -122
  29. package/server/integrations/seazona/history/backfill.js +208 -0
  30. package/server/integrations/seazona/history/csv.js +65 -0
  31. package/server/integrations/seazona/history/index.js +584 -0
  32. package/server/integrations/seazona/history/read.js +196 -0
  33. package/server/integrations/seazona/index.js +37 -1
  34. package/server/integrations/seazona/lanes.js +186 -50
  35. package/server/integrations/seazona/migrations/0003_call_guard.sql +2 -0
  36. package/server/integrations/seazona/migrations/0004_history_import.sql +68 -0
  37. package/server/integrations/seazona/migrations/meta/0003_snapshot.json +895 -0
  38. package/server/integrations/seazona/migrations/meta/0004_snapshot.json +1323 -0
  39. package/server/integrations/seazona/migrations/meta/_journal.json +14 -0
  40. package/server/integrations/seazona/routes.js +145 -0
  41. package/server/integrations/seazona/schema.js +94 -0
  42. package/server/integrations/seazona/settings.js +14 -3
  43. package/server/integrations/seazona/sync-once.js +1 -1
  44. package/server/integrations/seazona/tables.js +4 -0
  45. package/server/integrations/xero/client.js +27 -7
  46. package/server/integrations/xero/drafts.js +46 -8
  47. package/server/integrations/xero/import.js +125 -17
  48. package/server/integrations/xero/index.js +36 -7
  49. package/server/integrations/xero/lookup.js +26 -0
  50. package/server/integrations/xero/migrations/0006_xero_call_guard.sql +26 -0
  51. package/server/integrations/xero/migrations/meta/_journal.json +7 -0
  52. package/server/integrations/xero/mirror.js +11 -2
  53. package/server/integrations/xero/oauth.js +4 -1
  54. package/server/integrations/xero/reconcile.js +173 -0
  55. package/server/integrations/xero/routes.js +35 -0
  56. package/server/integrations/xero/runtime.js +45 -4
  57. package/server/release-channel.js +3 -3
  58. package/server/ui/alerts.js +77 -0
  59. package/server/ui/app-page.js +1 -1
  60. package/server/ui/app.js +48 -14
  61. package/server/ui/clerk-appearance.js +42 -9
  62. package/server/ui/flow.css +31 -2
  63. package/server/ui/invoicing-settings.js +1 -1
  64. package/server/ui/invoicing.js +121 -8
  65. package/server/ui/line-mappings.js +3 -2
  66. package/server/ui/price-list.js +247 -0
  67. package/server/ui/routes.js +1 -1
@@ -1,14 +1,20 @@
1
1
  import { randomUUID } from 'node:crypto';
2
2
  import { sql } from 'drizzle-orm';
3
- import { enqueueJob, runNextJob } from '../../core/queue.js';
3
+ import { deadHoldSince, deadSince, enqueueJob, runNextJob } from '../../core/queue.js';
4
+ import { isCallsStopped } from '../call-guard.js';
4
5
  import { lastSync, syncCatalogue } from './catalogue.js';
6
+ import { mirrorState } from './lookup.js';
5
7
  import { MIRROR_RESOURCES, mirrorXero } from './mirror.js';
8
+ import { checkSentInvoices } from './reconcile.js';
6
9
 
7
- // The history import. When an owner connects Xero, Flow copies the
8
- // organisation's invoices, credit notes and payments into the mirror
9
- // (mirror.js) as a background job, so nobody runs a script by hand. The job
10
- // reads in slices; a slice that stops at the day's budget or its page limit
11
- // queues the next slice. It only reads from Xero.
10
+ // The history import, then the reads of changes. When an owner connects Xero,
11
+ // Flow copies the organisation's invoices, credit notes and payments into the
12
+ // mirror (mirror.js) as a background job, so nobody runs a script by hand. The
13
+ // job reads in slices; a slice that stops at the day's budget or its page
14
+ // limit queues the next slice. When every resource is read to its end, the
15
+ // same job is queued again for REFRESH_MS later and reads only what changed
16
+ // since its cursor (issue #131), so the copy stays fresh (lookup.js). It only
17
+ // reads from Xero.
12
18
 
13
19
  export const IMPORT_KIND = 'xero.import';
14
20
  const IMPORT_KEY = 'history';
@@ -18,6 +24,10 @@ const LEASE_MS = 30 * 60 * 1000;
18
24
  // Xero's day limit resets on a rolling day; an hour is a cheap re-check.
19
25
  const BUDGET_WAIT_MS = 60 * 60 * 1000;
20
26
  const IDLE_MS = 30 * 1000;
27
+ // How often a finished copy reads Xero's changes again. Each read costs about
28
+ // one call per resource when little changed: 96 a day for each of invoices and
29
+ // credit notes, inside the reserve the mirror leaves (XERO_MIRROR_RESERVE_FRACTION).
30
+ export const REFRESH_MS = 15 * 60 * 1000;
21
31
  // Contacts and items are read again once a day, so the matcher can link each
22
32
  // customer (server/core/pipeline.js) without a person pressing Sync.
23
33
  const CATALOGUE_EVERY_MS = 24 * 60 * 60 * 1000;
@@ -44,12 +54,33 @@ export async function organisationsToImport(db, mode) {
44
54
  return rows.map((r) => r.organisation_id);
45
55
  }
46
56
 
57
+ // The next read for one organisation: queued now unless one is open already.
58
+ export async function queueChangesRead(db, organisationId, { runAfter = null } = {}) {
59
+ return queueHistoryImport(db, organisationId, { reason: 'changes', runAfter });
60
+ }
61
+
47
62
  // One slice of the import for one job. Answers what it read and whether more
48
63
  // remains (`more`: 'budget' or 'max_pages').
64
+ // While the call guard stops Xero calls the slice does not fail, so it does
65
+ // not spend the job's attempts: it answers `more: 'stopped'` with the wait,
66
+ // or no wait for a stop only a person clears (connecting queues the import).
49
67
  export async function runImportSlice(db, runtime, job, { maxPages = SLICE_PAGES } = {}) {
50
- const results = await mirrorXero(db, runtime, job.organisationId, { maxPages });
51
- const stopped = results.find((r) => r.stopped !== 'done')?.stopped ?? null;
52
- return { results, more: stopped };
68
+ let results;
69
+ try {
70
+ results = await mirrorXero(db, runtime, job.organisationId, { maxPages });
71
+ } catch (error) {
72
+ if (!isCallsStopped(error)) throw error;
73
+ return { results: [], more: error.retryAfterMs === null ? null : 'stopped', retryAfterMs: error.retryAfterMs, stoppedBy: error.code };
74
+ }
75
+ // A refused resource waits for a person; it never queues another slice.
76
+ const stopped = results.find((r) => !['done', 'refused'].includes(r.stopped))?.stopped ?? null;
77
+ // Each invoice Flow sent, against what the copy now holds (issue #132):
78
+ // an edit, void or delete in Xero opens a review item. No Xero calls.
79
+ const invoices = results.find((r) => r.resource === 'invoices');
80
+ const sent = invoices && (invoices.inserted || invoices.changed || job.payload?.reason !== 'changes')
81
+ ? await checkSentInvoices(db, runtime, job.organisationId)
82
+ : null;
83
+ return { results, more: stopped, ...(sent ? { sent } : {}) };
53
84
  }
54
85
 
55
86
  // Claims and runs one due import job, then queues the next slice when one is
@@ -61,33 +92,106 @@ export async function runImportOnce(db, runtime, { workerId = `xero-import-${ran
61
92
  handlers: { [IMPORT_KIND]: (job) => runImportSlice(db, runtime, job, { maxPages }) },
62
93
  });
63
94
  if (ran?.outcome === 'succeeded' && ran.job.result?.more) {
64
- const wait = ran.job.result.more === 'budget' ? BUDGET_WAIT_MS : 0;
95
+ const { more, retryAfterMs } = ran.job.result;
96
+ const wait = more === 'budget' ? BUDGET_WAIT_MS : more === 'stopped' ? Math.max(IDLE_MS, Number(retryAfterMs) || BUDGET_WAIT_MS) : 0;
65
97
  await queueHistoryImport(db, ran.job.organisationId, { reason: ran.job.result.more, runAfter: new Date(Number(now()) + wait) });
98
+ } else if (ran?.outcome === 'succeeded' && !ran.job.result?.stoppedBy) {
99
+ // Read to the end: read the changes again later. A stop only a person
100
+ // clears queues nothing; connecting or saving the settings queues it.
101
+ await queueChangesRead(db, ran.job.organisationId, { runAfter: new Date(Number(now()) + REFRESH_MS) });
66
102
  }
67
103
  return ran;
68
104
  }
69
105
 
106
+ // Failed catalogue reads in a row since the last good one, and the newest
107
+ // failure's code.
108
+ async function failuresInARow(db, organisationId) {
109
+ const { rows } = await db.execute(sql`
110
+ select count(*)::int as failures,
111
+ (array_agg(error_code order by started_at desc))[1] as last_code
112
+ from xero_sync_runs
113
+ where organisation_id = ${organisationId} and status = 'failed'
114
+ and started_at > coalesce((select max(started_at) from xero_sync_runs
115
+ where organisation_id = ${organisationId} and status = 'succeeded'), 'epoch')`);
116
+ return { failures: rows[0]?.failures ?? 0, lastCode: rows[0]?.last_code ?? null };
117
+ }
118
+
119
+ // The wait after `failures` failed reads in a row: an hour, doubling, at most
120
+ // a day (issue #111: never every hour for ever).
121
+ export function catalogueRetryMs(failures) {
122
+ return Math.min(CATALOGUE_EVERY_MS, BUDGET_WAIT_MS * 2 ** Math.max(0, failures - 1));
123
+ }
124
+
70
125
  // Reads the contacts and items of each connected organisation whose last good
71
- // read is older than a day. A failure is logged and tried again next hour.
126
+ // read is older than a day. A failed read waits longer each time (above). A
127
+ // failure is reported once with log.error; the same failure again is a
128
+ // warning. While the call guard stops Xero calls, no read is tried.
72
129
  export async function refreshStaleCatalogues(db, runtime, { now = Date.now, log = console } = {}) {
73
- const { rows } = await db.execute(sql`select organisation_id from xero_connections where mode = ${runtime.settings.mode}`);
74
- for (const { organisation_id: organisationId } of rows) {
130
+ const { rows } = await db.execute(sql`select organisation_id, tenant_id from xero_connections where mode = ${runtime.settings.mode}`);
131
+ for (const { organisation_id: organisationId, tenant_id: tenantId } of rows) {
75
132
  const last = await lastSync(db, organisationId);
76
133
  const at = last?.finishedAt ?? last?.startedAt;
77
- if (last && at && Number(now()) - new Date(at) < (last.status === 'succeeded' ? CATALOGUE_EVERY_MS : BUDGET_WAIT_MS)) continue;
134
+ const { failures, lastCode } = last?.status === 'failed' ? await failuresInARow(db, organisationId) : { failures: 0, lastCode: null };
135
+ if (last && at && Number(now()) - new Date(at) < (last.status === 'succeeded' ? CATALOGUE_EVERY_MS : catalogueRetryMs(failures))) continue;
136
+ try {
137
+ await runtime.guardFor(db).check(tenantId);
138
+ } catch (error) {
139
+ if (isCallsStopped(error)) continue;
140
+ throw error;
141
+ }
78
142
  try {
79
143
  const run = await syncCatalogue(db, runtime, organisationId);
80
144
  log.info?.(`[xero] contacts and items read for organisation ${organisationId}: ${run.contactCount} contacts, ${run.itemCount} items`);
81
145
  } catch (error) {
82
- log.error?.(`[xero] reading contacts and items for organisation ${organisationId} failed: ${error?.code ?? 'error'}`);
146
+ const code = error?.code ?? 'error';
147
+ const line = `[xero] reading contacts and items for organisation ${organisationId} failed: ${code}; next try in ${Math.round(catalogueRetryMs(failures + 1) / 60_000)} minutes`;
148
+ // The guard reported a stop already; a repeat of the same failure is a warning.
149
+ if (isCallsStopped(error) || (failures > 0 && code === lastCode)) log.warn?.(line);
150
+ else log.error?.(line);
151
+ }
152
+ }
153
+ }
154
+
155
+ // Keeps a read queued for each connected organisation. A read that gave up
156
+ // (dead) waits for a person, a restart or the next day (issue #101), and
157
+ // while the call guard stops Xero calls no read is queued.
158
+ export async function keepReadsQueued(db, runtime, { startedAtMs = Date.now(), now = Date.now } = {}) {
159
+ const { rows } = await db.execute(sql`select organisation_id, tenant_id from xero_connections where mode = ${runtime.settings.mode}`);
160
+ for (const { organisation_id: organisationId, tenant_id: tenantId } of rows) {
161
+ if (await deadSince(db, organisationId, { kind: IMPORT_KIND, dedupeKey: IMPORT_KEY, since: deadHoldSince(startedAtMs, Number(now())) })) continue;
162
+ try {
163
+ await runtime.guardFor(db).check(tenantId);
164
+ } catch (error) {
165
+ if (isCallsStopped(error)) continue;
166
+ throw error;
167
+ }
168
+ await queueChangesRead(db, organisationId);
169
+ }
170
+ }
171
+
172
+ // Reports a copy that went stale once with log.error, and its recovery with
173
+ // log.info. `stale` is the set of organisations already reported.
174
+ export async function reportStaleCopies(db, runtime, { stale = new Set(), now = Date.now, log = console } = {}) {
175
+ const { rows } = await db.execute(sql`select organisation_id from xero_connections where mode = ${runtime.settings.mode}`);
176
+ for (const { organisation_id: organisationId } of rows) {
177
+ const state = await mirrorState(db, organisationId, { now: new Date(Number(now())) });
178
+ if (!state.complete) continue;
179
+ if (!state.fresh && !stale.has(organisationId)) {
180
+ stale.add(organisationId);
181
+ log.error?.(`[xero] the copy of Xero for organisation ${organisationId} is out of date (last full read ${state.lastPassAt?.toISOString() ?? 'never'}); drafts by rule wait until it is read again`);
182
+ } else if (state.fresh && stale.delete(organisationId)) {
183
+ log.info?.(`[xero] the copy of Xero for organisation ${organisationId} is up to date again`);
83
184
  }
84
185
  }
186
+ return stale;
85
187
  }
86
188
 
87
189
  // The background worker: runs due slices one at a time, then waits. Started
88
190
  // by the adapter outside replay mode; stop() waits for a slice in flight.
89
191
  export function createImportWorker({ db, runtime, log = console, timers = { setTimeout, clearTimeout }, firstDelayMs = 10_000 }) {
90
192
  const workerId = `xero-import-${randomUUID()}`;
193
+ const startedAtMs = Date.now();
194
+ const stale = new Set();
91
195
  let timer = null;
92
196
  let running = null;
93
197
  let stopped = true;
@@ -105,12 +209,14 @@ export function createImportWorker({ db, runtime, log = console, timers = { setT
105
209
  running = (async () => {
106
210
  try {
107
211
  await refreshStaleCatalogues(db, runtime, { log });
212
+ await keepReadsQueued(db, runtime, { startedAtMs });
213
+ await reportStaleCopies(db, runtime, { stale, log });
108
214
  const ran = await runImportOnce(db, runtime, { workerId });
109
215
  if (ran) {
110
216
  busy = true;
111
217
  const read = (ran.job.result?.results ?? []).map((r) => `${r.resource} ${r.read}`).join(', ');
112
- if (ran.outcome === 'dead') log.error?.('[xero] the history import gave up after its last attempt');
113
- log.info?.(`[xero] import ${ran.outcome} for organisation ${ran.job.organisationId}${read ? `: ${read}` : ''}${ran.job.result?.more ? `; more (${ran.job.result.more})` : ''}`);
218
+ if (ran.outcome === 'dead') log.error?.('[xero] reading Xero gave up after its last attempt');
219
+ log.info?.(`[xero] read ${ran.outcome} for organisation ${ran.job.organisationId}${read ? `: ${read}` : ''}${ran.job.result?.more ? `; more (${ran.job.result.more})` : ''}`);
114
220
  }
115
221
  } catch (error) {
116
222
  log.error?.(`[xero] import worker: ${error?.code ?? 'error'}: ${error?.message ?? error}`);
@@ -127,6 +233,8 @@ export function createImportWorker({ db, runtime, log = console, timers = { setT
127
233
  for (const organisationId of await organisationsToImport(db, runtime.settings.mode)) {
128
234
  await queueHistoryImport(db, organisationId, { reason: 'start' });
129
235
  }
236
+ // A finished copy reads its changes again from the first tick on
237
+ // (keepReadsQueued), so an upgrade or a restart needs no step.
130
238
  schedule(firstDelayMs);
131
239
  },
132
240
  async stop() {
@@ -1,19 +1,20 @@
1
1
  import { fileURLToPath } from 'node:url';
2
2
  import { sql } from 'drizzle-orm';
3
- import { checkDrafts, pushInvoice, pushInvoices } from './drafts.js';
3
+ import { guardAlerts } from '../call-guard.js';
4
+ import { checkDrafts, documentFacts, pushInvoice, pushInvoices } from './drafts.js';
4
5
  import { readConnection } from './connections.js';
5
6
  import { createImportWorker } from './import.js';
6
7
  import { readPayments, xeroLiveCheck } from './ledger.js';
7
- import { findInvoices, listCustomers } from './lookup.js';
8
+ import { findInvoices, listCustomers, listPriceItems, mirrorState } from './lookup.js';
8
9
  import { createXeroRouter } from './routes.js';
9
- import { createXeroRuntime, requireXeroSettings } from './runtime.js';
10
+ import { createXeroRuntime, requireXeroSettings, XERO_CALL_GUARD } from './runtime.js';
10
11
  import { writesAllowed } from './settings.js';
11
12
 
12
13
  // The Xero adapter (docs/XERO.md). FLOW_INTEGRATIONS=xero turns it on and
13
14
  // XERO_MODE says which Xero it acts on. Its tables are created only while it
14
15
  // is on (docs/INTEGRATIONS.md).
15
16
 
16
- export const XERO_TABLES = Object.freeze(['xero_connections', 'xero_contacts', 'xero_items', 'xero_sync_runs', 'xero_invoice_pushes', 'xero_payment_cursors', 'xero_invoices', 'xero_credit_notes', 'xero_payments', 'xero_mirror_cursors']);
17
+ export const XERO_TABLES = Object.freeze(['xero_connections', 'xero_contacts', 'xero_items', 'xero_sync_runs', 'xero_invoice_pushes', 'xero_payment_cursors', 'xero_invoices', 'xero_credit_notes', 'xero_payments', 'xero_mirror_cursors', 'xero_call_guard', 'xero_calls']);
17
18
 
18
19
  // What an owner enters in Settings, Integrations (server/integrations/setup.js).
19
20
  // The refresh token's key comes from Flow's secrets key.
@@ -61,10 +62,16 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
61
62
  tables: [...XERO_TABLES],
62
63
  },
63
64
  setup: XERO_SETUP,
64
- async start({ db, log, env }) {
65
+ // Every Xero call goes through this call guard (docs/INTEGRATIONS.md,
66
+ // "Outbound calls").
67
+ callGuard: XERO_CALL_GUARD,
68
+ // `reconfigured` is true when a person saved the settings or turned Xero
69
+ // on: that clears a halt, stopped resources and an open circuit.
70
+ async start({ db, log, env, reconfigured = false }) {
65
71
  database = db;
66
72
  if (!given) runtime = createXeroRuntime({ settings: requireXeroSettings(env ?? process.env) });
67
73
  const { settings } = current();
74
+ if (reconfigured) await current().guardFor(db).resume();
68
75
  log.info?.(`[xero] mode ${settings.mode}; writes ${writesAllowed(settings) ? 'on' : 'off (XERO_ALLOW_LIVE_WRITES)'}; redirect URI ${settings.redirectUri ?? 'not set (FLOW_PUBLIC_ORIGIN)'}`);
69
76
  // The history import (import.js) runs in the background outside replay
70
77
  // mode. Replay tests run its slices directly.
@@ -81,8 +88,25 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
81
88
  capabilities: {
82
89
  async health() {
83
90
  const { settings } = current();
84
- const { rows } = await database.execute(sql`select count(*)::int as connected from xero_connections where mode = ${settings.mode}`);
85
- return { ok: true, mode: settings.mode, writesAllowed: writesAllowed(settings), connected: rows[0].connected };
91
+ const { rows } = await database.execute(sql`select organisation_id, tenant_id from xero_connections where mode = ${settings.mode}`);
92
+ const guard = current().guardFor(database);
93
+ const calls = await Promise.all(rows.map(async (r) => ({ tenantId: r.tenant_id, ...await guard.status(r.tenant_id) })));
94
+ const stopped = calls.some((c) => c.circuit !== 'closed' || c.stoppedResources.length || c.failedToday >= c.dailyFailureBudget);
95
+ // How fresh each copy of Xero is (lookup.js): a complete copy that is
96
+ // not fresh stops drafts by rule (issue #131).
97
+ const copies = await Promise.all(rows.map(async (r) => {
98
+ const { complete, fresh, lastPassAt } = await mirrorState(database, r.organisation_id);
99
+ return { tenantId: r.tenant_id, complete, fresh, lastPassAt };
100
+ }));
101
+ const stale = copies.some((c) => c.complete && !c.fresh);
102
+ return { ok: !stopped && !stale, mode: settings.mode, writesAllowed: writesAllowed(settings), connected: rows.length, calls, copies };
103
+ },
104
+ // What only a person can clear, for the Integrations page
105
+ // (server/core/alerts.js): a refused connection stops calls.
106
+ async alerts({ organisationId }) {
107
+ const connection = await readConnection(database, organisationId);
108
+ if (!connection || connection.mode !== current().settings.mode) return [];
109
+ return guardAlerts('Xero', await current().guardFor(database).status(connection.tenantId));
86
110
  },
87
111
  // One issued invoice to a Xero draft (drafts.js). The Flow invoice id
88
112
  // and version are the idempotency key.
@@ -96,6 +120,9 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
96
120
  const connection = await readConnection(database, organisationId);
97
121
  return { connected: Boolean(connection && connection.mode === settings.mode), writesAllowed: writesAllowed(settings) };
98
122
  },
123
+ // What the customer's Xero copy of a Flow invoice version says, from
124
+ // Flow's copy of Xero (drafts.js; no calls). The invoice PDF uses it.
125
+ documentFacts: ({ organisationId, invoiceId, version = null }) => documentFacts(database, current(), { organisationId, invoiceId, version }),
99
126
  // Sales invoice payments changed since `since` (ledger.js). The answer's
100
127
  // cursor is the next `since`.
101
128
  pullPayments: ({ organisationId, since }) => readPayments(database, current(), organisationId, { since: new Date(since) }),
@@ -106,6 +133,8 @@ export function createXeroAdapter({ runtime: given = null } = {}) {
106
133
  listCustomers: ({ organisationId }) => listCustomers(database, organisationId),
107
134
  // Sales invoices from the mirror, with the mirror's state (lookup.js).
108
135
  findInvoices: ({ organisationId, ...query }) => findInvoices(database, organisationId, query),
136
+ // Xero items, from Flow's copy, for the price list (lookup.js).
137
+ listPriceItems: ({ organisationId }) => listPriceItems(database, organisationId, { taxTypes: current().settings.taxTypes }),
109
138
  },
110
139
  routes: ({ db, personAuth, log, messagePage }) => createXeroRouter({ db, personAuth, runtime: current(), log, messagePage }),
111
140
  };
@@ -22,6 +22,32 @@ export async function listCustomers(db, organisationId) {
22
22
  return rows.map((r) => ({ id: r.xero_contact_id, name: r.name }));
23
23
  }
24
24
 
25
+ // Xero's items, from Flow's copy (catalogue.js, read on each sync), for the
26
+ // price list's links and price check (server/core/price-list.js). A tax type
27
+ // that charges GST reads as GST; any other set type reads as GST free.
28
+ const GST_TAX_TYPES = new Set(['OUTPUT', 'OUTPUT2']);
29
+ export async function listPriceItems(db, organisationId, { taxTypes = {} } = {}) {
30
+ const { rows } = await db.execute(sql`
31
+ select xero_item_id, code, name, description, is_sold, sales_unit_amount_cents, sales_tax_type, synced_at
32
+ from xero_items where organisation_id = ${organisationId} order by code`);
33
+ const taxOf = (type) => {
34
+ if (!type) return null;
35
+ return type === taxTypes.GST || GST_TAX_TYPES.has(type) ? 'GST' : 'GST_FREE';
36
+ };
37
+ return {
38
+ kind: 'accounting',
39
+ syncedAt: rows.reduce((at, r) => (!at || r.synced_at > at ? r.synced_at : at), null),
40
+ items: rows.map((r) => ({
41
+ id: r.xero_item_id,
42
+ code: r.code,
43
+ name: r.name || r.description || r.code,
44
+ unitAmountCents: r.sales_unit_amount_cents,
45
+ taxCode: taxOf(r.sales_tax_type),
46
+ sold: r.is_sold,
47
+ })),
48
+ };
49
+ }
50
+
25
51
  // Whether the mirror holds the whole history and was read lately. Until then a
26
52
  // search that finds nothing proves nothing.
27
53
  export async function mirrorState(db, organisationId, { now = new Date() } = {}) {
@@ -0,0 +1,26 @@
1
+ -- The call guard's state and telemetry (server/integrations/call-guard.js,
2
+ -- issue 111): one row per Xero organisation (tenant) with the circuit, a halt
3
+ -- and the resources Xero refused, and one row per call sent. A call row keeps
4
+ -- the resource and path, never the host, the query, a token or a response.
5
+ CREATE TABLE IF NOT EXISTS "xero_call_guard" (
6
+ "key" text PRIMARY KEY NOT NULL,
7
+ "circuit_open_until" timestamp with time zone DEFAULT 'epoch' NOT NULL,
8
+ "circuit_opens" integer DEFAULT 0 NOT NULL,
9
+ "halted_reason" text,
10
+ "halted_at" timestamp with time zone,
11
+ "stopped_resources" jsonb DEFAULT '{}'::jsonb NOT NULL,
12
+ "updated_at" timestamp with time zone DEFAULT now() NOT NULL
13
+ );
14
+ --> statement-breakpoint
15
+ CREATE TABLE IF NOT EXISTS "xero_calls" (
16
+ "id" bigserial PRIMARY KEY NOT NULL,
17
+ "key" text NOT NULL,
18
+ "at" timestamp with time zone NOT NULL,
19
+ "resource" text,
20
+ "request_path" text,
21
+ "status_code" integer,
22
+ "failure_kind" text,
23
+ "duration_ms" integer
24
+ );
25
+ --> statement-breakpoint
26
+ CREATE INDEX IF NOT EXISTS "xero_calls_window" ON "xero_calls" USING btree ("key","at");
@@ -43,6 +43,13 @@
43
43
  "when": 1792406000000,
44
44
  "tag": "0005_xero_send_modes",
45
45
  "breakpoints": true
46
+ },
47
+ {
48
+ "idx": 6,
49
+ "version": "7",
50
+ "when": 1792490000000,
51
+ "tag": "0006_xero_call_guard",
52
+ "breakpoints": true
46
53
  }
47
54
  ]
48
55
  }
@@ -1,5 +1,6 @@
1
1
  import { createHash } from 'node:crypto';
2
2
  import { sql } from 'drizzle-orm';
3
+ import { isCallsStopped } from '../call-guard.js';
3
4
  import { parseXeroDate } from './catalogue.js';
4
5
  import { withXeroLane } from './governor.js';
5
6
 
@@ -257,12 +258,20 @@ export async function mirrorResource(db, runtime, organisationId, resource, { ma
257
258
  }
258
259
 
259
260
  // Reads each resource in turn, one organisation at a time with any other Xero
260
- // work for it (runtime.serial).
261
+ // work for it (runtime.serial). A resource Xero refused (the call guard
262
+ // stopped it: the connection lacks its scope) is skipped as `refused`, so
263
+ // the others still read.
261
264
  export async function mirrorXero(db, runtime, organisationId, { resources = MIRROR_RESOURCES, maxPages } = {}) {
262
265
  return runtime.serial(organisationId, async () => {
263
266
  const results = [];
264
267
  for (const resource of resources) {
265
- const result = await mirrorResource(db, runtime, organisationId, resource, { maxPages });
268
+ let result;
269
+ try {
270
+ result = await mirrorResource(db, runtime, organisationId, resource, { maxPages });
271
+ } catch (error) {
272
+ if (!isCallsStopped(error) || error.reason !== 'resource_stopped') throw error;
273
+ result = { resource, pages: 0, read: 0, inserted: 0, changed: 0, unchanged: 0, stopped: 'refused' };
274
+ }
266
275
  results.push(result);
267
276
  if (result.stopped === 'budget') break;
268
277
  }
@@ -53,7 +53,10 @@ async function tokenRequest(settings, fetchImpl, form, what) {
53
53
  if (body?.error === 'invalid_grant') {
54
54
  throw xeroError('xero_reconnect_needed', 'Xero no longer accepts this connection. An owner must connect Xero again.', { status: 409 });
55
55
  }
56
- throw xeroError('xero_token_failed', `Xero refused ${what} (${response.status}${body?.error ? `, ${body.error}` : ''}).`, { status: 502 });
56
+ throw xeroError('xero_token_failed', `Xero refused ${what} (${response.status}${body?.error ? `, ${body.error}` : ''}).`, {
57
+ status: 502,
58
+ oauthError: typeof body?.error === 'string' ? body.error.slice(0, 40) : null,
59
+ });
57
60
  }
58
61
  if (!body?.access_token || !body?.refresh_token) {
59
62
  throw xeroError('xero_token_failed', `Xero's answer to ${what} had no tokens.`, { status: 502 });
@@ -0,0 +1,173 @@
1
+ import { sql } from 'drizzle-orm';
2
+ import { openReview, resolveReviewFor } from '../../core/review.js';
3
+
4
+ // Each invoice Flow sent, against Flow's copy of Xero (issue #132). People
5
+ // change invoices in both places after Flow sends them: a Xero draft is edited
6
+ // before approval, an invoice is voided in Xero, or voided in Flow. The issued
7
+ // version Flow sent is the merge base: the document both sides agreed on.
8
+ // This module compares the Xero copy with it and with the Flow invoice now.
9
+ // It never calls Xero and never writes to Xero or to the Flow invoice: a
10
+ // difference opens one review item (alert only), and a person fixes it where
11
+ // they choose. The item closes when a later check finds no difference.
12
+
13
+ export const OUT_OF_SYNC = 'invoice.accounting_differs';
14
+ export const ACTOR = 'system:xero-reconcile';
15
+
16
+ // Xero's status in the words the Flow screens show.
17
+ const LABELS = {
18
+ not_sent: 'Not in Xero',
19
+ sending: 'Sending to Xero',
20
+ not_copied: 'Sent, not read back yet',
21
+ draft: 'Draft in Xero',
22
+ approved: 'Approved in Xero',
23
+ part_paid: 'Part paid in Xero',
24
+ paid: 'Paid in Xero',
25
+ voided: 'Voided in Xero',
26
+ deleted: 'Deleted in Xero',
27
+ };
28
+
29
+ const GONE = new Set(['VOIDED', 'DELETED']);
30
+
31
+ function xeroState(row) {
32
+ switch (row.status) {
33
+ case 'DRAFT':
34
+ case 'SUBMITTED':
35
+ return 'draft';
36
+ case 'AUTHORISED':
37
+ return Number(row.amount_paid_cents ?? 0) > 0 ? 'part_paid' : 'approved';
38
+ case 'PAID':
39
+ return 'paid';
40
+ case 'VOIDED':
41
+ return 'voided';
42
+ case 'DELETED':
43
+ return 'deleted';
44
+ default:
45
+ return 'draft';
46
+ }
47
+ }
48
+
49
+ const num = (value) => (value === null || value === undefined ? null : Number(value));
50
+ const day = (value) => (value instanceof Date ? value.toISOString().slice(0, 10) : value ? String(value).slice(0, 10) : null);
51
+ const codes = (lines) => [...(lines ?? []).map((l) => l.itemCode ?? '')].sort().join(',');
52
+
53
+ // What differs between the Xero copy and the version Flow sent. Free text
54
+ // (reference, descriptions) is not compared: the copy keeps it only when the
55
+ // lab turns that on (XERO_KEEP_REFERENCE).
56
+ export function sentDifferences(sent, document, copy) {
57
+ const found = [];
58
+ const add = (field, flow, xero) => found.push({ field, flow, xero });
59
+ if (sent?.Contact?.ContactID && copy.xero_contact_id !== sent.Contact.ContactID) add('contact', sent.Contact.ContactID, copy.xero_contact_id);
60
+ if (num(copy.subtotal_cents) !== document.subtotalCents) add('subtotal', document.subtotalCents, num(copy.subtotal_cents));
61
+ if (num(copy.tax_cents) !== document.taxCents) add('tax', document.taxCents, num(copy.tax_cents));
62
+ if (num(copy.total_cents) !== document.totalCents) add('total', document.totalCents, num(copy.total_cents));
63
+ const lines = copy.lines ?? [];
64
+ if (lines.length !== document.lines.length) add('line count', document.lines.length, lines.length);
65
+ else if (codes(lines) !== codes(document.lines)) add('items', codes(document.lines), codes(lines));
66
+ if (sent?.Date && day(copy.issued_on) !== sent.Date) add('date', sent.Date, day(copy.issued_on));
67
+ return found;
68
+ }
69
+
70
+ // The finding for one invoice, or null when Flow and Xero agree.
71
+ // differs Xero's copy no longer matches the version Flow sent
72
+ // voided_in_xero Xero voided or deleted it; the Flow invoice is still issued
73
+ // voided_in_flow Flow voided it; Xero's invoice is still live
74
+ function findingFor(status) {
75
+ if (!status.xero) return null;
76
+ const live = !GONE.has(status.xero.status);
77
+ if (status.flowStatus === 'void' && live) return 'voided_in_flow';
78
+ if (status.flowStatus === 'issued' && status.current && !live) return 'voided_in_xero';
79
+ if (live && status.differences.length) return 'differs';
80
+ return null;
81
+ }
82
+
83
+ const MESSAGES = {
84
+ differs: 'Xero no longer matches the invoice Flow sent.',
85
+ voided_in_xero: 'Xero voided or deleted this invoice. It is still issued in Flow.',
86
+ voided_in_flow: 'This invoice is void in Flow. Xero still holds it as a live invoice.',
87
+ };
88
+
89
+ // Each invoice's Xero status, from Flow's copy (no Xero calls). Answers a map
90
+ // from Flow invoice id to { state, label, xero, differences, finding, message }.
91
+ export async function accountingStatus(db, runtime, organisationId, invoiceIds = null) {
92
+ const only = invoiceIds ? sql`and i.id = any(${sql.param([...new Set(invoiceIds.map(String))])}::uuid[])` : sql``;
93
+ // Without ids: only invoices Flow sent, for the check after each read.
94
+ const sentOnly = invoiceIds ? sql`` : sql`and p.id is not null`;
95
+ const { rows } = await db.execute(sql`
96
+ select i.id, i.status, i.issued_version, i.total_cents,
97
+ p.version as sent_version, p.state as push_state, p.request_body, p.xero_invoice_id, p.xero_invoice_number,
98
+ v.document,
99
+ x.status as x_status, x.invoice_number as x_number, x.xero_contact_id, x.subtotal_cents, x.tax_cents,
100
+ x.total_cents as x_total_cents, x.amount_paid_cents, x.amount_due_cents, x.lines, x.issued_on
101
+ from invoices i
102
+ left join lateral (
103
+ select * from xero_invoice_pushes
104
+ where organisation_id = i.organisation_id and invoice_id = i.id
105
+ order by version desc limit 1
106
+ ) p on true
107
+ left join invoice_versions v on v.invoice_id = i.id and v.version = p.version
108
+ left join xero_invoices x on x.organisation_id = i.organisation_id and x.xero_invoice_id = p.xero_invoice_id
109
+ where i.organisation_id = ${organisationId} ${only} ${sentOnly}`);
110
+ const web = runtime.settings.endpoints?.web ?? 'https://go.xero.com';
111
+ const out = new Map();
112
+ for (const r of rows) {
113
+ let state;
114
+ if (!r.push_state) state = 'not_sent';
115
+ else if (['dispatching', 'ambiguous'].includes(r.push_state)) state = 'sending';
116
+ else if (!r.x_status) state = 'not_copied';
117
+ else state = xeroState({ status: r.x_status, amount_paid_cents: r.amount_paid_cents });
118
+ let sent = null;
119
+ try {
120
+ sent = r.request_body ? JSON.parse(r.request_body) : null;
121
+ } catch {
122
+ sent = null;
123
+ }
124
+ const xero = r.x_status ? {
125
+ invoiceId: r.xero_invoice_id,
126
+ number: r.x_number ?? r.xero_invoice_number ?? null,
127
+ status: r.x_status,
128
+ totalCents: num(r.x_total_cents),
129
+ amountDueCents: num(r.amount_due_cents),
130
+ url: `${web}/AccountsReceivable/View.aspx?InvoiceID=${encodeURIComponent(r.xero_invoice_id)}`,
131
+ } : null;
132
+ const status = {
133
+ invoiceId: r.id,
134
+ flowStatus: r.status,
135
+ sentVersion: r.sent_version ?? null,
136
+ current: r.sent_version !== null && r.sent_version === r.issued_version,
137
+ state,
138
+ label: LABELS[state],
139
+ xero,
140
+ differences: xero && r.document && !GONE.has(r.x_status) ? sentDifferences(sent, r.document, { ...r, total_cents: r.x_total_cents }) : [],
141
+ flowTotalCents: num(r.total_cents),
142
+ };
143
+ status.finding = findingFor(status);
144
+ status.message = status.finding ? MESSAGES[status.finding] : null;
145
+ out.set(r.id, status);
146
+ }
147
+ return out;
148
+ }
149
+
150
+ // Opens or refreshes one review item for each finding, and closes the item of
151
+ // an invoice whose finding went away. Answers counts.
152
+ export async function checkSentInvoices(db, runtime, organisationId, { invoiceIds = null } = {}) {
153
+ const statuses = await accountingStatus(db, runtime, organisationId, invoiceIds);
154
+ const result = { checked: 0, open: 0, closed: 0 };
155
+ for (const status of statuses.values()) {
156
+ if (status.state === 'not_sent') continue;
157
+ result.checked += 1;
158
+ if (status.finding) {
159
+ await openReview(db, organisationId, {
160
+ kind: OUT_OF_SYNC,
161
+ entity: 'invoice',
162
+ flowId: status.invoiceId,
163
+ systems: ['flow', 'xero'],
164
+ evidence: { finding: status.finding, message: status.message, flowStatus: status.flowStatus, sentVersion: status.sentVersion, xero: status.xero, differences: status.differences },
165
+ moneyAtRiskCents: Math.abs(status.xero?.totalCents ?? status.flowTotalCents ?? 0),
166
+ });
167
+ result.open += 1;
168
+ } else if (await resolveReviewFor(db, organisationId, { kind: OUT_OF_SYNC, entity: 'invoice', flowId: status.invoiceId, resolution: 'no_difference', actor: ACTOR })) {
169
+ result.closed += 1;
170
+ }
171
+ }
172
+ return result;
173
+ }