@arcforgelabs/flow 2026.10.21 → 2026.10.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +2 -1
  2. package/VISION.md +40 -9
  3. package/npm-shrinkwrap.json +2 -414
  4. package/package.json +6 -1
  5. package/server/app.js +7 -0
  6. package/server/core/links.js +166 -0
  7. package/server/core/outbox.js +129 -0
  8. package/server/core/queue.js +142 -0
  9. package/server/core/review.js +92 -0
  10. package/server/core/secrets.js +74 -0
  11. package/server/db/migrations/0005_data_spine.sql +99 -0
  12. package/server/db/migrations/0006_prior_links.sql +18 -0
  13. package/server/db/migrations/0007_integration_settings.sql +23 -0
  14. package/server/db/migrations/meta/0005_snapshot.json +2145 -0
  15. package/server/db/migrations/meta/0006_snapshot.json +2306 -0
  16. package/server/db/migrations/meta/0007_snapshot.json +2473 -0
  17. package/server/db/migrations/meta/_journal.json +21 -0
  18. package/server/db/schema.js +187 -1
  19. package/server/import/at-start.js +69 -1
  20. package/server/index.js +28 -8
  21. package/server/integrations/index.js +9 -3
  22. package/server/integrations/port.js +101 -9
  23. package/server/integrations/seazona/index.js +25 -3
  24. package/server/integrations/setup.js +234 -0
  25. package/server/integrations/xero/client.js +13 -0
  26. package/server/integrations/xero/import.js +116 -0
  27. package/server/integrations/xero/index.js +45 -5
  28. package/server/integrations/xero/migrations/0004_xero_mirror.sql +96 -0
  29. package/server/integrations/xero/migrations/meta/_journal.json +7 -0
  30. package/server/integrations/xero/mirror.js +295 -0
  31. package/server/integrations/xero/routes.js +26 -1
  32. package/server/integrations/xero/settings.js +8 -0
  33. package/server/ui/app-page.js +1 -1
  34. package/server/ui/app.js +142 -0
  35. package/server/ui/routes.js +9 -2
@@ -36,6 +36,27 @@
36
36
  "when": 1792329649575,
37
37
  "tag": "0004_ledger_events_append_only",
38
38
  "breakpoints": true
39
+ },
40
+ {
41
+ "idx": 5,
42
+ "version": "7",
43
+ "when": 1792388648997,
44
+ "tag": "0005_data_spine",
45
+ "breakpoints": true
46
+ },
47
+ {
48
+ "idx": 6,
49
+ "version": "7",
50
+ "when": 1792390491235,
51
+ "tag": "0006_prior_links",
52
+ "breakpoints": true
53
+ },
54
+ {
55
+ "idx": 7,
56
+ "version": "7",
57
+ "when": 1792391146736,
58
+ "tag": "0007_integration_settings",
59
+ "breakpoints": true
39
60
  }
40
61
  ]
41
62
  }
@@ -1,19 +1,23 @@
1
1
  import { sql } from 'drizzle-orm';
2
2
  import {
3
+ bigint,
3
4
  boolean,
4
5
  check,
5
6
  date,
6
7
  foreignKey,
8
+ index,
7
9
  integer,
8
10
  jsonb,
9
11
  pgTable,
10
12
  text,
11
13
  timestamp,
12
14
  unique,
15
+ uniqueIndex,
13
16
  uuid,
14
17
  } from 'drizzle-orm/pg-core';
15
18
 
16
- // Every table except `organisations` carries `organisation_id`. Child rows
19
+ // Every table except `organisations` and the installation-wide tables (people,
20
+ // integration settings) carries `organisation_id`. Child rows
17
21
  // point at their parent through (organisation_id, id), so the database itself
18
22
  // refuses a row that joins two organisations. Money is integer cents.
19
23
 
@@ -304,3 +308,185 @@ export const obligationWritebacks = pgTable('obligation_writebacks', {
304
308
  }),
305
309
  check('obligation_writebacks_state_valid', inList('state', WRITEBACK_STATES)),
306
310
  ]);
311
+
312
+ // The data spine (docs/DATA_SPINE.md, issue 56): the tables every integration
313
+ // shares. Each row changes state; the spine never deletes one.
314
+
315
+ // Identity map (issue 57). Which external record is each Flow record. At most
316
+ // one active link per external record and per Flow record, in each system. A
317
+ // link that is not exact stays `proposed` until a person confirms it.
318
+ export const LINK_ENTITIES = ['contact', 'price_item', 'obligation', 'invoice', 'payment', 'credit_note'];
319
+ export const LINK_MATCHES = ['created_by_flow', 'exact_id', 'imported', 'name', 'person'];
320
+ export const LINK_STATES = ['active', 'proposed', 'superseded'];
321
+
322
+ export const externalLinks = pgTable('external_links', {
323
+ id: id(),
324
+ organisationId: organisationId().references(() => organisations.id),
325
+ entity: text('entity').notNull(),
326
+ flowId: uuid('flow_id').notNull(),
327
+ system: text('system').notNull(),
328
+ externalId: text('external_id').notNull(),
329
+ matchedBy: text('matched_by').notNull(),
330
+ state: text('state').notNull(),
331
+ confirmedBy: uuid('confirmed_by').references(() => people.id, { onDelete: 'set null' }),
332
+ confirmedAt: timestamp('confirmed_at', { withTimezone: true }),
333
+ createdAt: createdAt(),
334
+ updatedAt: updatedAt(),
335
+ }, (t) => [
336
+ unique('external_links_org_id').on(t.organisationId, t.id),
337
+ uniqueIndex('external_links_active_external')
338
+ .on(t.organisationId, t.system, t.entity, t.externalId).where(sql`state = 'active'`),
339
+ uniqueIndex('external_links_active_flow')
340
+ .on(t.organisationId, t.system, t.entity, t.flowId).where(sql`state = 'active'`),
341
+ index('external_links_flow').on(t.organisationId, t.entity, t.flowId),
342
+ check('external_links_entity_valid', inList('entity', LINK_ENTITIES)),
343
+ check('external_links_matched_by_valid', inList('matched_by', LINK_MATCHES)),
344
+ check('external_links_state_valid', inList('state', LINK_STATES)),
345
+ check('external_links_name_needs_person', sql`${t.matchedBy} <> 'name' or ${t.state} <> 'active' or ${t.confirmedAt} is not null`),
346
+ ]);
347
+
348
+ // Links between a work system's job and an accounting record, carried over
349
+ // from an earlier installation (server/import/at-start.js). They wait here
350
+ // until the job becomes an obligation; then they are evidence for its link.
351
+ // `person` means a person made the link there; `imported` means the earlier
352
+ // app matched it by itself.
353
+ export const PRIOR_LINK_MATCHES = ['person', 'imported'];
354
+
355
+ export const priorLinks = pgTable('prior_links', {
356
+ id: id(),
357
+ organisationId: organisationId().references(() => organisations.id),
358
+ sourceSystem: text('source_system').notNull(),
359
+ sourceJobRef: text('source_job_ref').notNull(),
360
+ system: text('system').notNull(),
361
+ externalId: text('external_id').notNull(),
362
+ externalNumber: text('external_number'),
363
+ matchedBy: text('matched_by').notNull(),
364
+ matchedAt: timestamp('matched_at', { withTimezone: true }),
365
+ origin: text('origin').notNull(),
366
+ createdAt: createdAt(),
367
+ }, (t) => [
368
+ uniqueIndex('prior_links_job').on(t.organisationId, t.sourceSystem, t.sourceJobRef, t.system),
369
+ index('prior_links_external').on(t.organisationId, t.system, t.externalId),
370
+ check('prior_links_matched_by_valid', inList('matched_by', PRIOR_LINK_MATCHES)),
371
+ ]);
372
+
373
+ // Background job queue (issue 6). Workers claim with FOR UPDATE SKIP LOCKED and
374
+ // hold a lease; a lease that runs out makes the job claimable again. One queued
375
+ // or running job per (organisation, kind, dedupe key).
376
+ export const BACKGROUND_JOB_STATES = ['queued', 'running', 'succeeded', 'failed', 'dead'];
377
+
378
+ export const backgroundJobs = pgTable('background_jobs', {
379
+ id: id(),
380
+ organisationId: organisationId().references(() => organisations.id),
381
+ kind: text('kind').notNull(),
382
+ dedupeKey: text('dedupe_key').notNull().default(''),
383
+ payload: jsonb('payload').notNull().default(sql`'{}'::jsonb`),
384
+ state: text('state').notNull().default('queued'),
385
+ runAfter: timestamp('run_after', { withTimezone: true }).notNull().defaultNow(),
386
+ attempts: integer('attempts').notNull().default(0),
387
+ maxAttempts: integer('max_attempts').notNull().default(8),
388
+ lockedBy: text('locked_by'),
389
+ lockedUntil: timestamp('locked_until', { withTimezone: true }),
390
+ result: jsonb('result'),
391
+ lastError: text('last_error'),
392
+ createdAt: createdAt(),
393
+ updatedAt: updatedAt(),
394
+ }, (t) => [
395
+ uniqueIndex('background_jobs_one_open')
396
+ .on(t.organisationId, t.kind, t.dedupeKey).where(sql`state in ('queued', 'running')`),
397
+ index('background_jobs_due').on(t.state, t.runAfter),
398
+ check('background_jobs_state_valid', inList('state', BACKGROUND_JOB_STATES)),
399
+ check('background_jobs_attempts_valid', sql`${t.attempts} >= 0 and ${t.maxAttempts} > 0`),
400
+ ]);
401
+
402
+ // Outbox. Every write to another system is a row here before the call
403
+ // leaves, written in the same transaction as the Flow change it reports. The
404
+ // idempotency key is unique, so a retry never makes a second record.
405
+ export const OUTBOX_STATES = ['pending', 'sending', 'ambiguous', 'sent', 'failed', 'cancelled'];
406
+
407
+ export const outbox = pgTable('outbox', {
408
+ id: id(),
409
+ organisationId: organisationId().references(() => organisations.id),
410
+ system: text('system').notNull(),
411
+ action: text('action').notNull(),
412
+ entity: text('entity').notNull(),
413
+ flowId: uuid('flow_id').notNull(),
414
+ idempotencyKey: text('idempotency_key').notNull(),
415
+ request: jsonb('request').notNull(),
416
+ state: text('state').notNull().default('pending'),
417
+ externalId: text('external_id'),
418
+ attempts: integer('attempts').notNull().default(0),
419
+ lastError: text('last_error'),
420
+ createdBy: text('created_by').notNull(),
421
+ sentAt: timestamp('sent_at', { withTimezone: true }),
422
+ createdAt: createdAt(),
423
+ updatedAt: updatedAt(),
424
+ }, (t) => [
425
+ unique('outbox_org_id').on(t.organisationId, t.id),
426
+ unique('outbox_idempotency_key').on(t.idempotencyKey),
427
+ index('outbox_pending').on(t.system, t.state, t.createdAt),
428
+ check('outbox_state_valid', inList('state', OUTBOX_STATES)),
429
+ check('outbox_sent_has_id', sql`${t.state} <> 'sent' or ${t.externalId} is not null`),
430
+ ]);
431
+
432
+ // Review queue (issue 59). One list of everything a person must decide. One
433
+ // open item per (organisation, kind, entity, Flow id); a repeat finding
434
+ // updates its evidence.
435
+ export const REVIEW_STATES = ['open', 'resolved', 'dismissed'];
436
+
437
+ export const reviewItems = pgTable('review_items', {
438
+ id: id(),
439
+ organisationId: organisationId().references(() => organisations.id),
440
+ kind: text('kind').notNull(),
441
+ entity: text('entity').notNull(),
442
+ flowId: uuid('flow_id').notNull(),
443
+ systems: text('systems').array().notNull().default(sql`'{}'::text[]`),
444
+ evidence: jsonb('evidence').notNull().default(sql`'{}'::jsonb`),
445
+ moneyAtRiskCents: bigint('money_at_risk_cents', { mode: 'number' }).notNull().default(0),
446
+ state: text('state').notNull().default('open'),
447
+ seenCount: integer('seen_count').notNull().default(1),
448
+ resolution: text('resolution'),
449
+ note: text('note'),
450
+ resolvedBy: text('resolved_by'),
451
+ resolvedAt: timestamp('resolved_at', { withTimezone: true }),
452
+ createdAt: createdAt(),
453
+ updatedAt: updatedAt(),
454
+ }, (t) => [
455
+ unique('review_items_org_id').on(t.organisationId, t.id),
456
+ uniqueIndex('review_items_one_open')
457
+ .on(t.organisationId, t.kind, t.entity, t.flowId).where(sql`state = 'open'`),
458
+ index('review_items_queue').on(t.organisationId, t.state, t.moneyAtRiskCents),
459
+ check('review_items_state_valid', inList('state', REVIEW_STATES)),
460
+ check('review_items_money_not_negative', sql`${t.moneyAtRiskCents} >= 0`),
461
+ check('review_items_closed_has_resolution', sql`${t.state} = 'open' or (${t.resolution} is not null and ${t.resolvedBy} is not null and ${t.resolvedAt} is not null)`),
462
+ ]);
463
+
464
+ // Integration settings an owner enters in Settings (server/integrations/setup.js).
465
+ // One row per adapter for the installation. `config` holds plain settings by
466
+ // their environment name; `secrets` holds credentials sealed with Flow's
467
+ // secrets key (server/core/secrets.js), never in clear. A host's own
468
+ // environment setting wins over a value here.
469
+ export const integrationSettings = pgTable('integration_settings', {
470
+ adapterId: text('adapter_id').primaryKey(),
471
+ enabled: boolean('enabled').notNull().default(false),
472
+ config: jsonb('config').notNull().default(sql`'{}'::jsonb`),
473
+ secrets: jsonb('secrets').notNull().default(sql`'{}'::jsonb`),
474
+ updatedBy: uuid('updated_by').references(() => people.id, { onDelete: 'set null' }),
475
+ createdAt: createdAt(),
476
+ updatedAt: updatedAt(),
477
+ });
478
+
479
+ // Who changed which integration setting, and when. Names only, never a value.
480
+ export const INTEGRATION_SETTING_EVENTS = ['turned_on', 'turned_off', 'setting_changed', 'secret_set', 'secret_removed'];
481
+
482
+ export const integrationSettingEvents = pgTable('integration_setting_events', {
483
+ id: id(),
484
+ adapterId: text('adapter_id').notNull(),
485
+ kind: text('kind').notNull(),
486
+ settings: text('settings').array().notNull().default(sql`'{}'::text[]`),
487
+ actor: uuid('actor').references(() => people.id, { onDelete: 'set null' }),
488
+ createdAt: createdAt(),
489
+ }, (t) => [
490
+ index('integration_setting_events_adapter').on(t.adapterId, t.createdAt),
491
+ check('integration_setting_events_kind_valid', inList('kind', INTEGRATION_SETTING_EVENTS)),
492
+ ]);
@@ -1,5 +1,5 @@
1
1
  import { sql } from 'drizzle-orm';
2
- import { organisations } from '../db/schema.js';
2
+ import { organisations, priorLinks } from '../db/schema.js';
3
3
  import { applyImport, planImport, readEarlierFlow, summarisePlan } from './earlier-flow.js';
4
4
 
5
5
  // The import from the earlier codebase, run once by the server itself. On a
@@ -50,3 +50,71 @@ export async function importEarlierAtStart({ db, pool, settings, log = console }
50
50
  log.info?.(`[import] imported: ${JSON.stringify(result.counts)}`);
51
51
  return result.counts;
52
52
  }
53
+
54
+ // The earlier app's table of jobs bound to accounting invoices. Each row ties a
55
+ // Seazona order to the Xero invoice that already bills it.
56
+ const EARLIER_LINKS = 'seazona_order_xero_match_overrides';
57
+ const LINK_ORIGIN = `earlier:${EARLIER_LINKS}`;
58
+ const LINK_BATCH = 500;
59
+
60
+ function matchedAt(text) {
61
+ const at = text ? new Date(text) : null;
62
+ return at && !Number.isNaN(at.getTime()) ? at : null;
63
+ }
64
+
65
+ // Carries the earlier app's order -> invoice links into `prior_links`, at
66
+ // every start while the import settings are set. It reads in a read-only
67
+ // transaction and only adds links it does not hold, so a repeat costs one read
68
+ // and changes nothing. It copies ids and how each link was made; never the
69
+ // earlier notes or who made it, which can name people. Answers counts, or null
70
+ // when there is nothing to read.
71
+ export async function importEarlierLinksAtStart({ db, pool, settings, log = console }) {
72
+ if (!settings) return null;
73
+ // The lab the import made, by name; or the only organisation, if renamed since.
74
+ const { rows: labs } = await db.execute(sql`select id, name from ${organisations}`);
75
+ const organisation = labs.find((lab) => lab.name === settings.organisationName) ?? (labs.length === 1 ? labs[0] : null);
76
+ if (!organisation) return null;
77
+
78
+ let rows;
79
+ const client = await pool.connect();
80
+ try {
81
+ await client.query('BEGIN TRANSACTION READ ONLY');
82
+ await client.query(`SET LOCAL search_path TO "${settings.schema}"`);
83
+ const { rows: [found] } = await client.query('select to_regclass($1) is not null as present', [EARLIER_LINKS]);
84
+ if (found.present) {
85
+ ({ rows } = await client.query(`select seazona_order_id, xero_invoice_id, xero_invoice_number, match_source, matched_at
86
+ from ${EARLIER_LINKS} where coalesce(xero_invoice_id, '') <> '' and coalesce(seazona_order_id, '') <> ''`));
87
+ }
88
+ await client.query('COMMIT');
89
+ } catch (error) {
90
+ await client.query('ROLLBACK').catch(() => {});
91
+ throw new Error(`could not read links from schema "${settings.schema}": ${error.message}`);
92
+ } finally {
93
+ client.release();
94
+ }
95
+ if (!rows) {
96
+ log.info?.(`[import] links: schema "${settings.schema}" has no ${EARLIER_LINKS}`);
97
+ return null;
98
+ }
99
+
100
+ const values = rows.map((row) => ({
101
+ organisationId: organisation.id,
102
+ sourceSystem: 'seazona',
103
+ sourceJobRef: String(row.seazona_order_id).trim(),
104
+ system: 'xero',
105
+ externalId: String(row.xero_invoice_id).trim(),
106
+ externalNumber: row.xero_invoice_number?.trim() || null,
107
+ matchedBy: row.match_source === 'operator_link' ? 'person' : 'imported',
108
+ matchedAt: matchedAt(row.matched_at),
109
+ origin: LINK_ORIGIN,
110
+ }));
111
+ let added = 0;
112
+ for (let start = 0; start < values.length; start += LINK_BATCH) {
113
+ const written = await db.insert(priorLinks).values(values.slice(start, start + LINK_BATCH))
114
+ .onConflictDoNothing().returning({ id: priorLinks.id });
115
+ added += written.length;
116
+ }
117
+ const counts = { read: values.length, added };
118
+ log.info?.(`[import] links from schema "${settings.schema}": ${JSON.stringify(counts)}`);
119
+ return counts;
120
+ }
package/server/index.js CHANGED
@@ -5,8 +5,11 @@ import { seedFirstOwner } from './auth/people.js';
5
5
  import { createPersonAuth } from './auth/session.js';
6
6
  import { resolveSignInSettings } from './auth/settings.js';
7
7
  import { migrateDatabase, openDatabase } from './db/index.js';
8
- import { importEarlierAtStart, resolveImportAtStart } from './import/at-start.js';
9
- import { createIntegrations } from './integrations/index.js';
8
+ import { importEarlierAtStart, importEarlierLinksAtStart, resolveImportAtStart } from './import/at-start.js';
9
+ import { resolveSecretsKey } from './core/secrets.js';
10
+ import { adapters as shippedAdapters } from './integrations/adapters.js';
11
+ import { createIntegrations, resolveEnabledIntegrations } from './integrations/index.js';
12
+ import { readStoredSettings, settingsEnvironment, storedEnabled } from './integrations/setup.js';
10
13
  import { resolveDefaultFamily } from './theme/themes.js';
11
14
  import { createUpdateChecker, UPDATE_CHECK_INTERVAL_MS } from './update-check.js';
12
15
 
@@ -27,9 +30,10 @@ try {
27
30
  console.error(`[ui] ${error.message}`);
28
31
  process.exit(1);
29
32
  }
30
- let integrations;
33
+ let secrets;
31
34
  try {
32
- integrations = createIntegrations();
35
+ resolveEnabledIntegrations();
36
+ secrets = resolveSecretsKey();
33
37
  } catch (error) {
34
38
  console.error(`[integrations] ${error.message}`);
35
39
  process.exit(1);
@@ -55,9 +59,25 @@ try {
55
59
  console.error(`[import] ${error.message}`);
56
60
  process.exit(1);
57
61
  }
58
- await integrations.start(database.db);
59
- const integrationsOn = integrations.list().filter((a) => a.enabled).map((a) => a.id);
60
- console.log(`[integrations] on: ${integrationsOn.join(', ') || 'none'} (FLOW_INTEGRATIONS)`);
62
+ // The earlier links are evidence to add, not records Flow needs to open, so a
63
+ // failure is logged and the start goes on.
64
+ await importEarlierLinksAtStart({ db: database.db, pool: database.pool, settings: importAtStart })
65
+ .catch((error) => console.error(`[import] ${error.message}`));
66
+ // What an owner set in Settings, Integrations, with the host's own
67
+ // environment over it (server/integrations/setup.js).
68
+ let integrations;
69
+ try {
70
+ const stored = await readStoredSettings(database.db);
71
+ const { env: settingsEnv, unreadable } = settingsEnvironment({ env: process.env, stored, adapters: shippedAdapters, secrets });
72
+ if (unreadable.length) console.warn(`[integrations] stored credentials Flow cannot read with this secrets key: ${unreadable.join(', ')}. An owner enters them again in Settings.`);
73
+ integrations = createIntegrations({ turnedOnInFlow: storedEnabled(stored, process.env), settingsEnv });
74
+ await integrations.start(database.db);
75
+ } catch (error) {
76
+ console.error(`[integrations] ${error.message}`);
77
+ process.exit(1);
78
+ }
79
+ const integrationsOn = integrations.list().filter((a) => a.enabled).map((a) => `${a.id}${a.state === 'ready' ? '' : ` (${a.state})`}`);
80
+ console.log(`[integrations] on: ${integrationsOn.join(', ') || 'none'}; secrets key ${secrets ? `from ${secrets.source}` : 'not set (FLOW_SECRETS_KEY)'}`);
61
81
 
62
82
  let auth = null;
63
83
  if (settings.mode === 'clerk') {
@@ -100,7 +120,7 @@ const trustProxy = process.env.REPL_ID || process.env.REPLIT_DEPLOYMENT ? 1 : fa
100
120
 
101
121
  const updates = createUpdateChecker();
102
122
 
103
- createApp({ database, auth, trustProxy, updates, integrations, defaultFamily }).listen(port, () => {
123
+ createApp({ database, auth, trustProxy, updates, integrations, secrets, defaultFamily }).listen(port, () => {
104
124
  console.log(`[flow] listening on ${port}; design system ${defaultFamily} (FLOW_THEME)`);
105
125
  });
106
126
 
@@ -16,11 +16,17 @@ export function resolveEnabledIntegrations(env = process.env, adapters = shipped
16
16
  }
17
17
 
18
18
  // The registry for one Flow process: every shipped adapter registered, the
19
- // chosen ones enabled. Call registry.start(db) after the core migrations.
20
- export function createIntegrations({ env = process.env, adapters = shipped, log = console } = {}) {
19
+ // chosen ones enabled. `turnedOnInFlow` are the ids an owner turned on in
20
+ // Settings (server/integrations/setup.js): one of those that cannot start is
21
+ // reported, not fatal. `settingsEnv` is what adapters read as their settings.
22
+ // Call registry.start(db) after the core migrations.
23
+ export function createIntegrations({ env = process.env, adapters = shipped, log = console, turnedOnInFlow = [], settingsEnv = env } = {}) {
21
24
  const enabled = resolveEnabledIntegrations(env, adapters);
22
- const registry = createRegistry({ log });
25
+ const registry = createRegistry({ log, env: settingsEnv });
23
26
  for (const adapter of adapters) registry.register(adapter);
24
27
  for (const id of enabled) registry.enable(id);
28
+ for (const id of turnedOnInFlow) {
29
+ if (!enabled.includes(id) && registry.get(id)) registry.enable(id, { optional: true });
30
+ }
25
31
  return registry;
26
32
  }
@@ -53,6 +53,7 @@ export function turnedOff(adapter, reason = 'disabled') {
53
53
  disabled: `${adapter.name} is turned off on this installation.`,
54
54
  restart_needed: `${adapter.name} was turned on after Flow started. Restart Flow to finish turning it on.`,
55
55
  tables_missing: `${adapter.name} has no tables on this installation. Restart Flow to create them.`,
56
+ failed: `${adapter.name} could not start. An owner can check its settings in Settings, Integrations.`,
56
57
  }[reason];
57
58
  return { ok: false, code: 'integration_off', integration: adapter.id, reason, error };
58
59
  }
@@ -61,7 +62,7 @@ export function defineAdapter(adapter) {
61
62
  if (!adapter || typeof adapter !== 'object') {
62
63
  throw new TypeError('An adapter must be an object');
63
64
  }
64
- const { id, name, bundled = false, capabilities, schema, start, stop, routes } = adapter;
65
+ const { id, name, bundled = false, capabilities, schema, start, stop, routes, setup } = adapter;
65
66
  if (typeof id !== 'string' || !ID_PATTERN.test(id)) {
66
67
  throw new TypeError(`Adapter id must match ${ID_PATTERN}`);
67
68
  }
@@ -90,6 +91,14 @@ export function defineAdapter(adapter) {
90
91
  if (routes !== undefined && typeof routes !== 'function') {
91
92
  throw new TypeError(`Adapter ${id} routes must be a function that returns a router`);
92
93
  }
94
+ if (setup !== undefined && (!setup || !Array.isArray(setup.fields))) {
95
+ throw new TypeError(`Adapter ${id} setup must list its fields`);
96
+ }
97
+ for (const field of setup?.fields ?? []) {
98
+ if (!/^[A-Z][A-Z0-9_]{1,63}$/.test(field?.name ?? '') || !field.label) {
99
+ throw new TypeError(`Adapter ${id} setup field ${field?.name} needs an upper-case setting name and a label`);
100
+ }
101
+ }
93
102
  return Object.freeze({
94
103
  id,
95
104
  name,
@@ -105,12 +114,24 @@ export function defineAdapter(adapter) {
105
114
  // Optional HTTP routes: routes({ db, personAuth, log, messagePage }) returns a router
106
115
  // mounted at /api/<id> (registry.routes).
107
116
  routes: routes ?? null,
117
+ // Optional: the settings an owner may enter in Settings, Integrations
118
+ // (server/integrations/setup.js). `fields` name environment settings;
119
+ // `derivedKeys` name key settings Flow derives from its secrets key.
120
+ setup: setup ? Object.freeze({ fields: Object.freeze(setup.fields.map((f) => Object.freeze({ ...f }))), derivedKeys: Object.freeze([...(setup.derivedKeys ?? [])]), guide: setup.guide ?? null }) : null,
108
121
  });
109
122
  }
110
123
 
111
- export function createRegistry({ log = console } = {}) {
124
+ export function createRegistry({ log = console, env = process.env } = {}) {
112
125
  const adapters = new Map();
113
126
  const enabled = new Set();
127
+ // Adapters turned on in Settings, not by the host: one that cannot start
128
+ // is reported, and Flow starts without it.
129
+ const optional = new Set();
130
+ const failures = new Map(); // id -> message
131
+ const generation = new Map(); // id -> count of (re)starts, for its router
132
+ const routers = new Map(); // id -> { generation, router }
133
+ let settingsEnv = env;
134
+ let database = null;
114
135
  // Adapters whose tables this process created or found at start. An adapter
115
136
  // without a schema needs none, so it is ready as soon as it is enabled.
116
137
  const tablesReady = new Set();
@@ -119,6 +140,7 @@ export function createRegistry({ log = console } = {}) {
119
140
 
120
141
  const state = (adapter) => {
121
142
  if (!enabled.has(adapter.id)) return 'disabled';
143
+ if (failures.has(adapter.id)) return 'failed';
122
144
  if (!adapter.schema || tablesReady.has(adapter.id)) return 'ready';
123
145
  return started ? 'restart_needed' : 'not_started';
124
146
  };
@@ -133,9 +155,11 @@ export function createRegistry({ log = console } = {}) {
133
155
  adapters.set(defined.id, defined);
134
156
  return defined;
135
157
  },
136
- enable(id) {
158
+ enable(id, { optional: isOptional = false } = {}) {
137
159
  if (!adapters.has(id)) throw new Error(`No adapter ${id}`);
138
160
  enabled.add(id);
161
+ if (isOptional) optional.add(id);
162
+ else optional.delete(id);
139
163
  },
140
164
  // Turning an adapter off never drops its tables. Removing them is a
141
165
  // separate step (removeAdapterTables in server/integrations/schema.js).
@@ -149,6 +173,7 @@ export function createRegistry({ log = console } = {}) {
149
173
  async start(db) {
150
174
  if (started) throw new Error('The integration registry has already started');
151
175
  started = true;
176
+ database = db;
152
177
  const migrated = [];
153
178
  for (const id of enabled) {
154
179
  const adapter = adapters.get(id);
@@ -160,11 +185,66 @@ export function createRegistry({ log = console } = {}) {
160
185
  for (const id of enabled) {
161
186
  const adapter = adapters.get(id);
162
187
  if (!adapter.start) continue;
163
- await adapter.start({ db, log });
188
+ try {
189
+ await adapter.start({ db, log, env: settingsEnv });
190
+ } catch (error) {
191
+ if (!optional.has(id)) throw error;
192
+ failures.set(id, error.message);
193
+ log.error?.(`[integrations] ${id} could not start: ${error.message}`);
194
+ continue;
195
+ }
164
196
  running.push(adapter);
165
197
  }
166
198
  return migrated;
167
199
  },
200
+ // Applies new settings to one adapter while Flow runs: stops it, then,
201
+ // when it stays on, creates its tables and starts it with `env`. Its
202
+ // routes are built again on the next request. A start that fails leaves
203
+ // it `failed` with the reason, and the rest of Flow running.
204
+ async reconfigure(id, { enabled: on, env: next }) {
205
+ const adapter = adapters.get(id);
206
+ if (!adapter) throw new Error(`No adapter ${id}`);
207
+ if (!started) throw new Error('The integration registry has not started');
208
+ if (next) settingsEnv = next;
209
+ const index = running.indexOf(adapter);
210
+ if (index >= 0) {
211
+ running.splice(index, 1);
212
+ await adapter.stop?.();
213
+ }
214
+ failures.delete(id);
215
+ generation.set(id, (generation.get(id) ?? 0) + 1);
216
+ if (!on) {
217
+ enabled.delete(id);
218
+ return state(adapter);
219
+ }
220
+ enabled.add(id);
221
+ optional.add(id);
222
+ try {
223
+ if (adapter.schema && !tablesReady.has(id)) {
224
+ await migrateAdapterSchema(database, adapter);
225
+ tablesReady.add(id);
226
+ }
227
+ if (adapter.start) {
228
+ await adapter.start({ db: database, log, env: settingsEnv });
229
+ running.push(adapter);
230
+ }
231
+ } catch (error) {
232
+ failures.set(id, error.message);
233
+ log.error?.(`[integrations] ${id} could not start: ${error.message}`);
234
+ }
235
+ return state(adapter);
236
+ },
237
+ get(id) {
238
+ return adapters.get(id) ?? null;
239
+ },
240
+ // Why an enabled adapter is not running, or null. The message names
241
+ // settings, never their values.
242
+ failure(id) {
243
+ return failures.get(id) ?? null;
244
+ },
245
+ env() {
246
+ return settingsEnv;
247
+ },
168
248
  // Calls stop() on every adapter whose start() ran, newest first.
169
249
  async stop() {
170
250
  while (running.length) {
@@ -180,6 +260,7 @@ export function createRegistry({ log = console } = {}) {
180
260
  enabled: enabled.has(a.id),
181
261
  state: state(a),
182
262
  tables: a.schema ? [...a.schema.tables] : [],
263
+ ...(failures.has(a.id) ? { error: failures.get(a.id) } : {}),
183
264
  capabilities: Object.keys(a.capabilities),
184
265
  }));
185
266
  },
@@ -202,10 +283,21 @@ export function createRegistry({ log = console } = {}) {
202
283
  routes(context) {
203
284
  return [...adapters.values()].filter((a) => a.routes).map((adapter) => {
204
285
  const path = `/api/${adapter.id}`;
205
- if (!ready(adapter)) {
206
- return { path, handler: (_req, res) => res.status(503).json(turnedOff(adapter, state(adapter) === 'disabled' ? 'disabled' : 'restart_needed')) };
207
- }
208
- return { path, handler: adapter.routes(context) };
286
+ const handler = (req, res, next) => {
287
+ const now = state(adapter);
288
+ if (now !== 'ready') {
289
+ const reason = now === 'disabled' || now === 'failed' ? now : 'restart_needed';
290
+ return res.status(503).json(turnedOff(adapter, reason));
291
+ }
292
+ const current = generation.get(adapter.id) ?? 0;
293
+ let built = routers.get(adapter.id);
294
+ if (!built || built.generation !== current) {
295
+ built = { generation: current, router: adapter.routes(context) };
296
+ routers.set(adapter.id, built);
297
+ }
298
+ return built.router(req, res, next);
299
+ };
300
+ return { path, handler };
209
301
  });
210
302
  },
211
303
  // The one way to read or write an adapter's tables: runs fn(adapter) when
@@ -216,7 +308,7 @@ export function createRegistry({ log = console } = {}) {
216
308
  const adapter = adapters.get(id);
217
309
  if (!adapter) throw new Error(`No adapter ${id}`);
218
310
  const now = state(adapter);
219
- if (now === 'disabled') return turnedOff(adapter, 'disabled');
311
+ if (now === 'disabled' || now === 'failed') return turnedOff(adapter, now);
220
312
  if (now !== 'ready') return turnedOff(adapter, 'restart_needed');
221
313
  try {
222
314
  return await fn(adapter);
@@ -22,7 +22,27 @@ export function requireSeazonaSettings(env = process.env) {
22
22
 
23
23
  // `env`, `fetchImpl`, `now` and `timers` are for tests. Otherwise start()
24
24
  // reads the environment, so a turned-off adapter never reads its settings.
25
- export function createSeazonaAdapter({ env = process.env, fetchImpl = null, now = Date.now, timers } = {}) {
25
+ // What an owner enters in Settings, Integrations (server/integrations/setup.js).
26
+ export const SEAZONA_SETUP = {
27
+ guide: {
28
+ title: 'Get a Seazona API client',
29
+ link: null,
30
+ steps: [
31
+ 'In Seazona, create an API client for Flow. If you cannot find the option, ask Seazona support for API access.',
32
+ 'Copy the API address, the client id and the client secret.',
33
+ 'Paste them below, choose the mode, and turn Seazona on.',
34
+ 'Live mode starts the background sync. Flow only reads from Seazona.',
35
+ ],
36
+ },
37
+ fields: [
38
+ { name: 'SEAZONA_MODE', label: 'Mode', kind: 'choice', choices: [{ value: 'live', label: 'Live: your Seazona account' }, { value: 'sandbox', label: 'Sandbox: a Seazona test account' }] },
39
+ { name: 'SEAZONA_API_BASE_URL', label: 'API address', kind: 'url', help: 'The API root of your account, https://…' },
40
+ { name: 'SEAZONA_API_CLIENT_ID', label: 'Client id' },
41
+ { name: 'SEAZONA_API_CLIENT_SECRET', label: 'Client secret', secret: true, help: 'Stored encrypted. Flow never shows it again.' },
42
+ ],
43
+ };
44
+
45
+ export function createSeazonaAdapter({ env = null, fetchImpl = null, now = Date.now, timers } = {}) {
26
46
  let runtime = null; // { settings, db, client, scheduler }
27
47
 
28
48
  function current() {
@@ -36,9 +56,10 @@ export function createSeazonaAdapter({ env = process.env, fetchImpl = null, now
36
56
  // VISION.md: Seazona is bundled and on in the dental preset.
37
57
  bundled: true,
38
58
  schema: seazonaSchema,
59
+ setup: SEAZONA_SETUP,
39
60
 
40
- async start({ db, log = console }) {
41
- const settings = requireSeazonaSettings(env);
61
+ async start({ db, log = console, env: settingsEnv }) {
62
+ const settings = requireSeazonaSettings(env ?? settingsEnv ?? process.env);
42
63
  const client = createSeazonaClient({ settings, db, fetchImpl: fetchImpl ?? undefined, workflow: 'seazona-sync', now });
43
64
  const run = (options = {}) => runSeazonaSync(db, client, { settings, now, ...options });
44
65
  // The background sync starts in live mode only. A sandbox or replay
@@ -51,6 +72,7 @@ export function createSeazonaAdapter({ env = process.env, fetchImpl = null, now
51
72
 
52
73
  async stop() {
53
74
  await runtime?.scheduler?.stop();
75
+ runtime = null;
54
76
  },
55
77
 
56
78
  capabilities: {