@arcforgelabs/flow 2026.10.32-beta.1 → 2026.10.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +3 -1
  2. package/npm-shrinkwrap.json +2 -2
  3. package/package.json +2 -1
  4. package/server/api/invoicing.js +69 -26
  5. package/server/app.js +4 -0
  6. package/server/core/automations.js +17 -3
  7. package/server/core/invoices.js +2 -2
  8. package/server/core/invoicing-rules.js +64 -5
  9. package/server/core/job-board.js +15 -4
  10. package/server/core/ledger.js +245 -8
  11. package/server/core/pipeline.js +30 -22
  12. package/server/core/read-before-act.js +96 -0
  13. package/server/core/reconcile.js +350 -0
  14. package/server/core/references.js +99 -0
  15. package/server/core/sending.js +52 -8
  16. package/server/db/migrations/0016_invoice_reference.sql +1 -0
  17. package/server/db/migrations/0017_sync_basis.sql +1 -0
  18. package/server/db/migrations/meta/0016_snapshot.json +3861 -0
  19. package/server/db/migrations/meta/0017_snapshot.json +3867 -0
  20. package/server/db/migrations/meta/_journal.json +14 -0
  21. package/server/db/schema.js +5 -0
  22. package/server/index.js +5 -0
  23. package/server/integrations/port.js +4 -0
  24. package/server/integrations/seazona/act.js +48 -0
  25. package/server/integrations/seazona/feed.js +77 -50
  26. package/server/integrations/seazona/governor.js +28 -2
  27. package/server/integrations/seazona/index.js +69 -3
  28. package/server/integrations/seazona/lanes.js +82 -9
  29. package/server/integrations/seazona/ledger.js +14 -9
  30. package/server/integrations/seazona/migrations/0005_record_reread.sql +1 -0
  31. package/server/integrations/seazona/migrations/0006_sweep_findings.sql +11 -0
  32. package/server/integrations/seazona/migrations/meta/0005_snapshot.json +1329 -0
  33. package/server/integrations/seazona/migrations/meta/0006_snapshot.json +1403 -0
  34. package/server/integrations/seazona/migrations/meta/_journal.json +14 -0
  35. package/server/integrations/seazona/patients.js +30 -0
  36. package/server/integrations/seazona/reconcile.js +40 -0
  37. package/server/integrations/seazona/scheduler.js +32 -4
  38. package/server/integrations/seazona/schema.js +21 -0
  39. package/server/integrations/seazona/settings.js +12 -0
  40. package/server/integrations/seazona/store.js +24 -2
  41. package/server/integrations/seazona/sweep.js +574 -0
  42. package/server/integrations/seazona/tables.js +1 -0
  43. package/server/integrations/xero/client.js +12 -0
  44. package/server/integrations/xero/import.js +9 -3
  45. package/server/integrations/xero/index.js +7 -1
  46. package/server/integrations/xero/migrations/0007_xero_webhooks.sql +30 -0
  47. package/server/integrations/xero/migrations/meta/_journal.json +7 -0
  48. package/server/integrations/xero/mirror.js +16 -8
  49. package/server/integrations/xero/routes.js +20 -0
  50. package/server/integrations/xero/settings.js +3 -0
  51. package/server/integrations/xero/webhooks.js +346 -0
  52. package/server/ui/app.js +15 -7
  53. package/server/ui/flow.css +56 -7
  54. package/server/ui/invoice-notes.js +26 -0
  55. package/server/ui/invoicing-settings.js +3 -1
  56. package/server/ui/invoicing.js +231 -68
  57. package/server/ui/job-board.js +2 -1
  58. package/server/ui/jobs.js +19 -1
  59. package/server/ui/routes.js +1 -1
@@ -4,8 +4,9 @@ import {
4
4
  billingObligations, obligationEvents, obligationPayments, obligationWritebacks, priceItems,
5
5
  } from '../db/schema.js';
6
6
  import { FlowError, notFound } from './errors.js';
7
- import { createDraftInvoice } from './invoices.js';
7
+ import { createDraftInvoice, getInvoice, updateDraftInvoice } from './invoices.js';
8
8
  import { invoiceTotals, lineTotals } from './money.js';
9
+ import { openReview, resolveReviewFor } from './review.js';
9
10
 
10
11
  // The billing obligation ledger. One billable job in a work system is one
11
12
  // obligation. Its view (In work, Ready, Pending, Held, Drafted, Invoiced,
@@ -23,6 +24,29 @@ export const EXACT_MATCHES = ['flow_invoice', 'source_invoice_ref', 'accounting_
23
24
 
24
25
  const LINKED_STATES = ['drafted', 'authorised', 'part_paid', 'paid', 'voided', 'deleted'];
25
26
 
27
+ // Which system owns each fact about a billed job (docs/DATA_SPINE.md, "Who
28
+ // owns each field"). The work system owns the work; Flow owns its invoice
29
+ // and the send; the accounting system owns approval and payment. Only the
30
+ // work fields are compared when a job changes after its invoice was made.
31
+ export const FIELD_OWNERS = {
32
+ lines: 'work',
33
+ contactId: 'work',
34
+ sourceStatus: 'work',
35
+ reference: 'flow',
36
+ dueDate: 'flow',
37
+ send: 'flow',
38
+ approval: 'accounting',
39
+ payment: 'accounting',
40
+ };
41
+ const COMPARED = Object.keys(FIELD_OWNERS).filter((field) => FIELD_OWNERS[field] === 'work' && field !== 'sourceStatus');
42
+
43
+ export const CHANGED_REASON = 'the job changed after its invoice was made';
44
+ export const STATUS_BACK_REASON = 'the job status moved back after its invoice was made';
45
+ // Review queue kinds (server/core/review.js). An item only alerts.
46
+ export const REVIEW_CHANGED = 'job.changed';
47
+ export const REVIEW_STATUS_BACK = 'job.status_back';
48
+ export const REVIEW_DELETED = 'job.deleted';
49
+
26
50
  async function loadObligation(tx, organisationId, obligationId, { lock = false } = {}) {
27
51
  let query = tx.select().from(billingObligations)
28
52
  .where(and(eq(billingObligations.organisationId, organisationId), eq(billingObligations.id, obligationId)));
@@ -149,26 +173,235 @@ export async function observeJob(db, organisationId, job) {
149
173
  const changed = ['sourceInvoiceRef', 'caseRef', 'contactId', 'practitioner', 'shippedOn', 'lineFingerprint', 'totalCents']
150
174
  .filter((key) => String(existing[key] ?? '') !== String(facts[key] ?? ''));
151
175
  if (!changed.length) {
152
- return update(tx, existing, {
176
+ const row = await update(tx, existing, {
153
177
  sourceSeenAt: seenAt, mappingComplete: facts.mappingComplete, lines, jobDate: facts.jobDate, sourceStatus: facts.sourceStatus,
154
178
  });
179
+ await statusBillableAgain(tx, organisationId, existing, row, actor);
180
+ return row;
155
181
  }
156
182
  const fields = { ...facts, sourceSeenAt: seenAt };
183
+ let compare = null;
157
184
  if (existing.invoiceId && changed.some((key) => ['lineFingerprint', 'totalCents', 'contactId'].includes(key))) {
158
- fields.holdState = 'review';
159
- fields.holdReason = 'the job changed after its invoice was made';
185
+ compare = await compareWithDraft(tx, organisationId, existing, facts);
186
+ if (compare.outcome !== 'none') {
187
+ fields.holdState = 'review';
188
+ fields.holdReason = CHANGED_REASON;
189
+ } else if (existing.holdState === 'review' && existing.holdReason === CHANGED_REASON) {
190
+ // The job is back to what the invoice was built from.
191
+ Object.assign(fields, { holdState: 'none', holdReason: null });
192
+ }
160
193
  }
161
194
  const row = await update(tx, existing, fields);
162
195
  await addEvent(tx, row, 'source_changed', actor, {
163
196
  changed,
164
197
  before: { lineFingerprint: existing.lineFingerprint, totalCents: existing.totalCents },
165
198
  after: { lineFingerprint: row.lineFingerprint, totalCents: row.totalCents },
166
- ...(fields.holdState ? { holdReason: fields.holdReason } : {}),
199
+ ...(compare ? { compare: { outcome: compare.outcome, fields: compare.fields.map(({ field, changedIn }) => ({ field, changedIn })) } } : {}),
200
+ ...(fields.holdState === 'review' ? { holdReason: fields.holdReason } : {}),
167
201
  });
202
+ if (compare?.outcome === 'none') {
203
+ await resolveReviewFor(tx, organisationId, { kind: REVIEW_CHANGED, entity: 'obligation', flowId: row.id, resolution: 'job_matches_invoice', actor });
204
+ } else if (compare) {
205
+ await openReview(tx, organisationId, changedReview(row, compare));
206
+ }
207
+ await statusBillableAgain(tx, organisationId, existing, row, actor);
168
208
  return row;
169
209
  });
170
210
  }
171
211
 
212
+ // A job billable again after its status moved back (observeWithdrawn): the
213
+ // alert is over when no invoice was made in the meantime.
214
+ async function statusBillableAgain(tx, organisationId, before, row, actor) {
215
+ if (row.invoiceId || String(before.sourceStatus ?? '') === String(row.sourceStatus ?? '')) return;
216
+ await resolveReviewFor(tx, organisationId, { kind: REVIEW_STATUS_BACK, entity: 'obligation', flowId: row.id, resolution: 'status_billable_again', actor });
217
+ }
218
+
219
+ // The job facts an invoice is built from, kept on the obligation when Flow
220
+ // drafts the invoice: the merge base of the three-way compare.
221
+ function basisOf(o) {
222
+ return {
223
+ lines: o.lines.map(({ itemCode = null, priceItemId = null, accountCode = null, description, quantity, unitAmountCents, taxCode }) => (
224
+ { itemCode, priceItemId, accountCode, description, quantity, unitAmountCents, taxCode }
225
+ )),
226
+ contactId: o.contactId ?? null,
227
+ subtotalCents: o.subtotalCents,
228
+ taxCents: o.taxCents,
229
+ totalCents: o.totalCents,
230
+ sourceStatus: o.sourceStatus ?? null,
231
+ sourceSeenAt: o.sourceSeenAt ? new Date(o.sourceSeenAt).toISOString() : null,
232
+ };
233
+ }
234
+
235
+ // One key for a set of lines, the same for the job's lines and the invoice's.
236
+ function linesKey(lines) {
237
+ const keys = (lines ?? [])
238
+ .map((l) => [l.priceItemId ?? l.accountCode ?? String(l.description ?? '').trim().toLowerCase(), l.quantity, l.unitAmountCents, l.taxCode])
239
+ .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
240
+ return createHash('sha256').update(JSON.stringify(keys)).digest('hex');
241
+ }
242
+
243
+ const sideValue = (side, field) => (field === 'lines' ? linesKey(side.lines) : (side[field] ?? null));
244
+
245
+ // The three-way compare, field by field, of the facts the invoice was built
246
+ // from (base), the work system now (source) and Flow's invoice now (flow).
247
+ // Only work fields are compared (FIELD_OWNERS). The outcome:
248
+ // none the work system is as it was, or as the invoice is: nothing to do
249
+ // offer only the work system changed and the draft is not sent: a person
250
+ // can take the new lines into Flow's own draft (useSourceLines)
251
+ // conflict both changed, or the invoice left Flow: an alert only
252
+ export function compareThreeWays(base, source, flow, { sent = false } = {}) {
253
+ const fields = COMPARED.map((field) => {
254
+ const [b, s, f] = [base, source, flow].map((side) => sideValue(side, field));
255
+ const changedIn = s === b ? (f === b ? 'none' : 'flow') : (f === b ? 'work' : (f === s ? 'same' : 'both'));
256
+ return { field, owner: FIELD_OWNERS[field], changedIn };
257
+ });
258
+ const work = fields.filter((f) => ['work', 'both'].includes(f.changedIn));
259
+ let outcome = 'none';
260
+ if (work.length) outcome = sent || work.some((f) => f.changedIn === 'both') ? 'conflict' : 'offer';
261
+ return { outcome, sent, fields };
262
+ }
263
+
264
+ // The invoice has left Flow's hands: issued, queued or sent, or the
265
+ // accounting system holds it.
266
+ function invoiceSent(invoice, obligation) {
267
+ return invoice.status !== 'draft' || ['queued', 'sent'].includes(invoice.sendState)
268
+ || LINKED_STATES.includes(obligation.accountingState);
269
+ }
270
+
271
+ async function compareWithDraft(tx, organisationId, obligation, facts) {
272
+ // An invoice drafted before the basis was kept: the last report stands in.
273
+ const base = obligation.draftBasis ?? basisOf(obligation);
274
+ let invoice;
275
+ try {
276
+ invoice = await getInvoice(tx, organisationId, obligation.invoiceId);
277
+ } catch (error) {
278
+ if (error?.code !== 'not_found') throw error;
279
+ return { outcome: 'conflict', sent: false, fields: [], base, missing: true };
280
+ }
281
+ const flow = { lines: invoice.lines, contactId: invoice.contactId };
282
+ const compare = compareThreeWays(base, facts, flow, { sent: invoiceSent(invoice, obligation) });
283
+ return { ...compare, base, flow, invoiceStatus: invoice.status };
284
+ }
285
+
286
+ function changedReview(obligation, compare) {
287
+ return {
288
+ kind: REVIEW_CHANGED,
289
+ entity: 'obligation',
290
+ flowId: obligation.id,
291
+ systems: [obligation.sourceSystem, 'flow'],
292
+ moneyAtRiskCents: Math.max(0, obligation.totalCents ?? 0),
293
+ evidence: {
294
+ outcome: compare.outcome,
295
+ sent: compare.sent,
296
+ invoiceId: obligation.invoiceId,
297
+ fields: compare.fields,
298
+ basis: compare.base,
299
+ job: basisOf(obligation),
300
+ invoice: compare.flow ? { contactId: compare.flow.contactId, lines: compare.flow.lines.map(({ priceItemId, accountCode, description, quantity, unitAmountCents, taxCode }) => ({ priceItemId, accountCode, description, quantity, unitAmountCents, taxCode })) } : null,
301
+ canUseNewLines: compare.outcome === 'offer',
302
+ },
303
+ };
304
+ }
305
+
306
+ // "Use new lines": a person takes the work system's new lines (and its
307
+ // customer, when that changed) into Flow's own unsent draft. Flow writes only
308
+ // its own draft, never another system. Refused when the draft changed too,
309
+ // or has left Flow: those stay alerts for a person.
310
+ export async function useSourceLines(db, organisationId, obligationId, { actor }) {
311
+ requireActor(actor);
312
+ return db.transaction(async (tx) => {
313
+ const obligation = await loadObligation(tx, organisationId, obligationId, { lock: true });
314
+ if (!obligation.invoiceId) throw new FlowError('invalid', 'the job has no invoice');
315
+ const compare = await compareWithDraft(tx, organisationId, obligation, obligation);
316
+ if (compare.outcome === 'conflict') {
317
+ throw new FlowError('conflict', compare.sent ? 'the invoice has left Flow; change it where it is now' : 'the draft changed in Flow too; a person must compare them', { fields: compare.fields });
318
+ }
319
+ const changes = {};
320
+ for (const { field, changedIn } of compare.fields) {
321
+ if (changedIn !== 'work') continue;
322
+ if (field === 'lines') {
323
+ changes.lines = obligation.lines.map(({ priceItemId, accountCode, description, quantity, unitAmountCents, taxCode }) => (
324
+ { priceItemId, accountCode, description, quantity, unitAmountCents, taxCode }
325
+ ));
326
+ } else changes[field] = obligation[field];
327
+ }
328
+ const invoice = Object.keys(changes).length
329
+ ? await updateDraftInvoice(tx, organisationId, obligation.invoiceId, changes)
330
+ : await getInvoice(tx, organisationId, obligation.invoiceId);
331
+ const fields = { draftBasis: basisOf(obligation) };
332
+ if (obligation.holdState === 'review' && obligation.holdReason === CHANGED_REASON) Object.assign(fields, { holdState: 'none', holdReason: null });
333
+ const row = await update(tx, obligation, fields);
334
+ await addEvent(tx, row, 'source_lines_used', actor, { invoiceId: invoice.id, changed: Object.keys(changes), totalCents: invoice.totalCents });
335
+ await resolveReviewFor(tx, organisationId, { kind: REVIEW_CHANGED, entity: 'obligation', flowId: row.id, resolution: 'used_new_lines', actor });
336
+ return { obligation: row, invoice };
337
+ });
338
+ }
339
+
340
+ // Records that a job the ledger knows is no longer billable in the work
341
+ // system: deleted there, or its status moved back (for example from Shipped
342
+ // to In production). Flow writes nothing to another system:
343
+ // deleted review hold and an alert
344
+ // status back, invoice made review hold and an alert
345
+ // status back, only ready the ready mark is cleared and an alert opens;
346
+ // the alert closes when the job is billable again
347
+ // The status is stored, so a repeat report of the same status does nothing.
348
+ export async function observeWithdrawn(db, organisationId, entry) {
349
+ const { sourceSystem, sourceJobRef, sourceStatus = null, deleted = false, reason = null, actor = `system:${sourceSystem}` } = entry;
350
+ if (!sourceSystem || !sourceJobRef) throw new FlowError('invalid', 'sourceSystem and sourceJobRef are required');
351
+ const seenAt = entry.observedAt ? new Date(entry.observedAt) : new Date();
352
+ return db.transaction(async (tx) => {
353
+ const [obligation] = await tx.select().from(billingObligations)
354
+ .where(and(
355
+ eq(billingObligations.organisationId, organisationId),
356
+ eq(billingObligations.sourceSystem, sourceSystem),
357
+ eq(billingObligations.sourceJobRef, sourceJobRef),
358
+ ))
359
+ .for('update');
360
+ if (!obligation) return { obligation: null, action: 'none' };
361
+ if (deleted) {
362
+ if (obligation.holdState === 'review') return { obligation, action: 'none' };
363
+ const holdReason = reason ?? 'the job was deleted in the work system';
364
+ const row = await update(tx, obligation, { holdState: 'review', holdReason });
365
+ await addEvent(tx, row, 'review', actor, { from: obligation.holdState, reason: holdReason });
366
+ await openReview(tx, organisationId, statusReview(REVIEW_DELETED, row, { reason: holdReason }));
367
+ return { obligation: row, action: 'review' };
368
+ }
369
+ if (seenAt < obligation.sourceSeenAt || String(obligation.sourceStatus ?? '') === String(sourceStatus ?? '')) {
370
+ return { obligation, action: 'none' };
371
+ }
372
+ const fields = { sourceStatus, sourceSeenAt: seenAt };
373
+ let action = 'status';
374
+ if (obligation.invoiceId) {
375
+ if (obligation.holdState !== 'review') Object.assign(fields, { holdState: 'review', holdReason: STATUS_BACK_REASON });
376
+ action = 'review';
377
+ } else if (obligation.readyAt) {
378
+ // Not setReady's "in work": that marks a person's choice, and the ready
379
+ // routine must mark the job again when its status is billable again.
380
+ Object.assign(fields, { readyAt: null, readyBy: null });
381
+ action = 'ready_cleared';
382
+ }
383
+ const row = await update(tx, obligation, fields);
384
+ await addEvent(tx, row, 'status_moved_back', actor, {
385
+ from: obligation.sourceStatus, to: sourceStatus, action, ...(fields.holdState ? { holdReason: fields.holdReason } : {}),
386
+ });
387
+ if (action !== 'status') {
388
+ await openReview(tx, organisationId, statusReview(REVIEW_STATUS_BACK, row, { from: obligation.sourceStatus, to: sourceStatus, action }));
389
+ }
390
+ return { obligation: row, action };
391
+ });
392
+ }
393
+
394
+ function statusReview(kind, obligation, evidence) {
395
+ return {
396
+ kind,
397
+ entity: 'obligation',
398
+ flowId: obligation.id,
399
+ systems: [obligation.sourceSystem, 'flow'],
400
+ moneyAtRiskCents: Math.max(0, obligation.totalCents ?? 0),
401
+ evidence: { ...evidence, sourceJobRef: obligation.sourceJobRef, invoiceId: obligation.invoiceId ?? null },
402
+ };
403
+ }
404
+
172
405
  // Records what the accounting system says about the job's invoice. Evidence
173
406
  // that is not tied to the job by an exact match never counts as found: it
174
407
  // makes the obligation ambiguous, for a person to resolve.
@@ -399,7 +632,7 @@ function draftBlockers(o, { now, sourceMaxAgeMs, accounting = true }) {
399
632
  // the write, finds no invoice for the job. Any doubt refuses the draft, and
400
633
  // the refusal is recorded.
401
634
  export async function draftInvoiceForObligation(db, organisationId, obligationId, {
402
- liveCheck, actor, now = new Date(), sourceMaxAgeMs = SOURCE_MAX_AGE_MS, dueDate = null,
635
+ liveCheck, actor, now = new Date(), sourceMaxAgeMs = SOURCE_MAX_AGE_MS, dueDate = null, reference = null,
403
636
  }) {
404
637
  requireActor(actor);
405
638
  // A null liveCheck says no accounting system is connected (ledgerView).
@@ -423,7 +656,8 @@ export async function draftInvoiceForObligation(db, organisationId, obligationId
423
656
  }
424
657
  const invoice = await createDraftInvoice(tx, organisationId, {
425
658
  contactId: obligation.contactId,
426
- reference: obligation.caseRef ?? obligation.sourceJobRef,
659
+ // The Reference rule's text (references.js), never a work system id.
660
+ reference,
427
661
  dueDate,
428
662
  lines: obligation.lines.map(({ priceItemId, accountCode, description, quantity, unitAmountCents, taxCode }) => (
429
663
  { priceItemId, accountCode, description, quantity, unitAmountCents, taxCode }
@@ -431,7 +665,10 @@ export async function draftInvoiceForObligation(db, organisationId, obligationId
431
665
  });
432
666
  // Drafting the invoice says the job is ready, whoever asked.
433
667
  const row = await update(tx, obligation, {
434
- invoiceId: invoice.id, accountingCheckedAt: now, ...(obligation.readyAt ? {} : { readyAt: now, readyBy: actor }),
668
+ invoiceId: invoice.id,
669
+ draftBasis: basisOf(obligation),
670
+ accountingCheckedAt: now,
671
+ ...(obligation.readyAt ? {} : { readyAt: now, readyBy: actor }),
435
672
  });
436
673
  await addEvent(tx, row, 'drafted', actor, { invoiceId: invoice.id, totalCents: invoice.totalCents, live });
437
674
  return { obligation: row, invoice };
@@ -6,7 +6,7 @@ import {
6
6
  import { FlowError } from './errors.js';
7
7
  import { listAutomations, runAutomations } from './automations.js';
8
8
  import { readInvoicingSettings } from './invoicing-rules.js';
9
- import { observeJob, recordAccountingEvidence, setHold } from './ledger.js';
9
+ import { observeJob, observeWithdrawn, recordAccountingEvidence, setHold } from './ledger.js';
10
10
  import { loadMappingContext, mapLines } from './line-mapping.js';
11
11
  import { checkPrices, readCatalogues } from './price-list.js';
12
12
  import { linkRecord, priorLinkFor } from './links.js';
@@ -130,25 +130,38 @@ async function writeCursor(db, organisationId, name, cursor) {
130
130
  .onConflictDoUpdate({ target: [pipelineCursors.organisationId, pipelineCursors.name], set: { cursor, updatedAt: sql`now()` } });
131
131
  }
132
132
 
133
- // Reads the work system's billable jobs since the last run into the ledger.
134
- // `restart` reads every job again, for example after new customer links.
135
- export async function ingestWork(db, organisationId, { adapter, restart = false, pages = WORK_PAGES }) {
136
- const name = `${adapter.id}.work`;
137
- let cursor = restart ? null : await readCursor(db, organisationId, name);
133
+ // Turns a work item into the job the ledger observes: its customer from the
134
+ // identity map, and its lines mapped to price items when "Map lines" is on.
135
+ // `mapped` says whether lines are mapped.
136
+ export async function workJobMaker(db, organisationId, adapter) {
138
137
  const links = await activeLinks(db, organisationId, { system: adapter.id, entity: 'contact' });
139
138
  const contactOf = new Map(links.map((l) => [l.externalId, l.flowId]));
140
- const result = { observed: 0, withoutCustomer: 0, skipped: 0, sentToReview: 0, unmappedLines: 0, more: false };
141
139
  // Line mapping (line-mapping.js): each line finds its price item first.
142
140
  const mapping = (await listAutomations(db, organisationId)).find((a) => a.key === 'map_lines');
143
141
  const context = mapping?.enabled ? await loadMappingContext(db, organisationId, mapping.config, { system: adapter.id }) : null;
142
+ const toJob = (item) => {
143
+ const job = context ? { ...item.ledgerJob, lines: mapLines(item.ledgerJob.lines ?? [], context) } : item.ledgerJob;
144
+ return { ...job, contactId: contactOf.get(String(item.clientId ?? '')) ?? null };
145
+ };
146
+ toJob.mapped = Boolean(context);
147
+ return toJob;
148
+ }
149
+
150
+ // Reads the work system's billable jobs since the last run into the ledger.
151
+ // `restart` reads every job again, for example after new customer links.
152
+ export async function ingestWork(db, organisationId, { adapter, restart = false, pages = WORK_PAGES }) {
153
+ const name = `${adapter.id}.work`;
154
+ let cursor = restart ? null : await readCursor(db, organisationId, name);
155
+ const toJob = await workJobMaker(db, organisationId, adapter);
156
+ const result = { observed: 0, withoutCustomer: 0, skipped: 0, sentToReview: 0, readyCleared: 0, unmappedLines: 0, more: false };
144
157
  for (let page = 0; page < pages; page += 1) {
145
158
  const { items, withdrawn, cursor: next, more } = await adapter.capabilities.pullWorkItems(cursor, { limit: 200 });
146
159
  for (const item of items) {
147
- const contactId = contactOf.get(String(item.clientId ?? '')) ?? null;
148
- const job = context ? { ...item.ledgerJob, lines: mapLines(item.ledgerJob.lines ?? [], context) } : item.ledgerJob;
149
- if (context) result.unmappedLines += job.lines.filter((l) => !l.itemCode && !l.accountCode).length;
160
+ const job = toJob(item);
161
+ const { contactId } = job;
162
+ if (toJob.mapped) result.unmappedLines += job.lines.filter((l) => !l.itemCode && !l.accountCode).length;
150
163
  try {
151
- await observeJob(db, organisationId, { ...job, contactId });
164
+ await observeJob(db, organisationId, job);
152
165
  result.observed += 1;
153
166
  if (!contactId) result.withoutCustomer += 1;
154
167
  } catch (error) {
@@ -157,17 +170,12 @@ export async function ingestWork(db, organisationId, { adapter, restart = false,
157
170
  result.skipped += 1;
158
171
  }
159
172
  }
173
+ // A job no longer billable: deleted, or its status moved back (ledger.js).
160
174
  for (const entry of withdrawn) {
161
- if (!entry.ledgerHold) continue;
162
- const { sourceSystem, sourceJobRef, holdState, reason } = entry.ledgerHold;
163
- const [obligation] = await db.select().from(billingObligations).where(and(
164
- eq(billingObligations.organisationId, organisationId),
165
- eq(billingObligations.sourceSystem, sourceSystem),
166
- eq(billingObligations.sourceJobRef, sourceJobRef),
167
- ));
168
- if (!obligation || obligation.holdState === holdState) continue;
169
- await setHold(db, organisationId, obligation.id, { holdState, reason, actor: `system:${adapter.id}` });
170
- result.sentToReview += 1;
175
+ if (!entry.ledgerWithdrawn) continue;
176
+ const { action } = await observeWithdrawn(db, organisationId, { ...entry.ledgerWithdrawn, actor: `system:${adapter.id}` });
177
+ if (action === 'review') result.sentToReview += 1;
178
+ if (action === 'ready_cleared') result.readyCleared += 1;
171
179
  }
172
180
  cursor = next;
173
181
  await writeCursor(db, organisationId, name, cursor);
@@ -511,7 +519,7 @@ export async function runPipelineOnce(db, integrations, { now = new Date() } = {
511
519
  const settings = await readInvoicingSettings(db, organisationId);
512
520
  summary.matching = await matchJobs(db, organisationId, { accounting, now, staleDays: settings.staleDays });
513
521
  }
514
- if (work || accounting) summary.automations = await runAutomations(db, organisationId, { accounting: accounting ?? null, now });
522
+ if (work || accounting) summary.automations = await runAutomations(db, organisationId, { accounting: accounting ?? null, work: work ?? null, now });
515
523
  return summary;
516
524
  }
517
525
 
@@ -0,0 +1,96 @@
1
+ import { and, eq } from 'drizzle-orm';
2
+ import { billingObligations } from '../db/schema.js';
3
+ import { FlowError } from './errors.js';
4
+ import { observeJob, observeWithdrawn } from './ledger.js';
5
+ import { matchJobs, workJobMaker } from './pipeline.js';
6
+
7
+ // Read before act (issue #141). Flow drafts and sends from its own copy of
8
+ // the work system. Just before a draft, and before Send all ready, a job
9
+ // whose work system report is older than READ_BEFORE_ACT_MS is read again
10
+ // from the work system (its readWorkItem capability): one call per job. What
11
+ // the read finds goes into the ledger as any report does (observeJob), so a
12
+ // change after the invoice was made sends the job to review there.
13
+ //
14
+ // A job the work system deleted, or that is no longer billable, goes through
15
+ // the ledger's observeWithdrawn like a feed report: review when it was
16
+ // deleted or already has its invoice. A job that is not billable now, or a
17
+ // read that cannot run, waits: Flow never drafts or sends from an old copy.
18
+ //
19
+ // The lanes re-report each job they read at most once an hour
20
+ // (seazona/feed.js), so most reports are under two hours old and cost no
21
+ // call here.
22
+
23
+ export const READ_BEFORE_ACT_MS = 2 * 60 * 60 * 1000;
24
+ // The most reads one pipeline run or one Send all ready press makes; the
25
+ // other old jobs wait for the next one. Seazona's calls are spaced 2 s apart.
26
+ export const READS_PER_PASS = 20;
27
+ const NO_LONGER_BILLABLE = 'the work system no longer reports the job as billable';
28
+
29
+ // Reads each job that needs it. `work` is the work integration, or null.
30
+ // Answers Map(obligationId -> { outcome, obligation, reason? }), where
31
+ // outcome is fresh (no read needed), read, review (sent to review) or
32
+ // waiting (the read could not run).
33
+ export async function readBeforeAct(db, organisationId, obligations, { work, now = new Date(), actor }) {
34
+ const outcomes = new Map();
35
+ let toJob = null;
36
+ for (const obligation of obligations) {
37
+ if (!needsRead(obligation, { work, now })) {
38
+ outcomes.set(obligation.id, { outcome: 'fresh', obligation });
39
+ continue;
40
+ }
41
+ const answer = await work.capabilities.readWorkItem({ sourceJobRef: obligation.sourceJobRef });
42
+ if (answer?.state === 'read') {
43
+ toJob ??= await workJobMaker(db, organisationId, work);
44
+ try {
45
+ await observeJob(db, organisationId, toJob(answer.item));
46
+ } catch (error) {
47
+ if (!(error instanceof FlowError) || error.code !== 'invalid') throw error;
48
+ outcomes.set(obligation.id, { outcome: 'waiting', obligation, reason: 'the work system job has no lines' });
49
+ continue;
50
+ }
51
+ const fresh = await reload(db, organisationId, obligation.id);
52
+ outcomes.set(obligation.id, { outcome: fresh.holdState === 'review' ? 'review' : 'read', obligation: fresh });
53
+ } else if (answer?.state === 'withdrawn') {
54
+ // The ledger decides as for the feed (observeWithdrawn): deleted, or
55
+ // moved back after its invoice, goes to review; moved back while only
56
+ // ready clears the ready mark.
57
+ if (answer.ledgerWithdrawn) await observeWithdrawn(db, organisationId, { ...answer.ledgerWithdrawn, actor });
58
+ const row = await reload(db, organisationId, obligation.id);
59
+ outcomes.set(obligation.id, row.holdState === 'review'
60
+ ? { outcome: 'review', obligation: row, reason: row.holdReason }
61
+ : { outcome: 'waiting', obligation: row, reason: NO_LONGER_BILLABLE });
62
+ } else {
63
+ outcomes.set(obligation.id, { outcome: 'waiting', obligation, reason: answer?.reason || 'the work system read failed' });
64
+ }
65
+ }
66
+ return outcomes;
67
+ }
68
+
69
+ // True when the job's report is old and its work system can read it now.
70
+ export function needsRead(obligation, { work, now = new Date() }) {
71
+ return typeof work?.capabilities?.readWorkItem === 'function'
72
+ && obligation.sourceSystem === work.id
73
+ && now - new Date(obligation.sourceSeenAt) > READ_BEFORE_ACT_MS;
74
+ }
75
+
76
+ // The accounting side of read before act (issue #45, step 5): once per
77
+ // automatic draft pass or Send all ready press, before the first draft or
78
+ // send, the accounting system reads its latest changes into its copy (its
79
+ // readChanges capability). When the copy changed, the matcher runs first, so
80
+ // a job someone invoiced in the accounting system meanwhile is linked to that
81
+ // invoice and never drafted or sent twice. Answers { outcome, reason? }:
82
+ // fresh or read (act), or waiting (act on nothing now).
83
+ export async function readAccountingFirst(db, organisationId, { accounting, now = new Date() }) {
84
+ if (typeof accounting?.capabilities?.readChanges !== 'function') return { outcome: 'fresh' };
85
+ const answer = await accounting.capabilities.readChanges({ organisationId });
86
+ if (answer?.state === 'fresh') return { outcome: 'fresh' };
87
+ if (answer?.state !== 'read') return { outcome: 'waiting', reason: answer?.reason || 'the accounting system read failed' };
88
+ if (answer.changed) await matchJobs(db, organisationId, { accounting, now });
89
+ return { outcome: 'read', changed: answer.changed ?? 0 };
90
+ }
91
+
92
+ async function reload(db, organisationId, id) {
93
+ const [row] = await db.select().from(billingObligations)
94
+ .where(and(eq(billingObligations.organisationId, organisationId), eq(billingObligations.id, id)));
95
+ return row;
96
+ }