@arcforgelabs/flow 2026.10.21 → 2026.10.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/VISION.md +40 -9
- package/npm-shrinkwrap.json +2 -414
- package/package.json +6 -1
- package/server/app.js +7 -0
- package/server/core/links.js +166 -0
- package/server/core/outbox.js +129 -0
- package/server/core/queue.js +142 -0
- package/server/core/review.js +92 -0
- package/server/core/secrets.js +74 -0
- package/server/db/migrations/0005_data_spine.sql +99 -0
- package/server/db/migrations/0006_prior_links.sql +18 -0
- package/server/db/migrations/0007_integration_settings.sql +23 -0
- package/server/db/migrations/meta/0005_snapshot.json +2145 -0
- package/server/db/migrations/meta/0006_snapshot.json +2306 -0
- package/server/db/migrations/meta/0007_snapshot.json +2473 -0
- package/server/db/migrations/meta/_journal.json +21 -0
- package/server/db/schema.js +187 -1
- package/server/import/at-start.js +69 -1
- package/server/index.js +28 -8
- package/server/integrations/index.js +9 -3
- package/server/integrations/port.js +101 -9
- package/server/integrations/seazona/index.js +25 -3
- package/server/integrations/setup.js +234 -0
- package/server/integrations/xero/client.js +13 -0
- package/server/integrations/xero/import.js +116 -0
- package/server/integrations/xero/index.js +45 -5
- package/server/integrations/xero/migrations/0004_xero_mirror.sql +96 -0
- package/server/integrations/xero/migrations/meta/_journal.json +7 -0
- package/server/integrations/xero/mirror.js +295 -0
- package/server/integrations/xero/routes.js +26 -1
- package/server/integrations/xero/settings.js +8 -0
- package/server/ui/app-page.js +1 -1
- package/server/ui/app.js +142 -0
- package/server/ui/routes.js +9 -2
|
@@ -36,6 +36,27 @@
|
|
|
36
36
|
"when": 1792329649575,
|
|
37
37
|
"tag": "0004_ledger_events_append_only",
|
|
38
38
|
"breakpoints": true
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"idx": 5,
|
|
42
|
+
"version": "7",
|
|
43
|
+
"when": 1792388648997,
|
|
44
|
+
"tag": "0005_data_spine",
|
|
45
|
+
"breakpoints": true
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"idx": 6,
|
|
49
|
+
"version": "7",
|
|
50
|
+
"when": 1792390491235,
|
|
51
|
+
"tag": "0006_prior_links",
|
|
52
|
+
"breakpoints": true
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"idx": 7,
|
|
56
|
+
"version": "7",
|
|
57
|
+
"when": 1792391146736,
|
|
58
|
+
"tag": "0007_integration_settings",
|
|
59
|
+
"breakpoints": true
|
|
39
60
|
}
|
|
40
61
|
]
|
|
41
62
|
}
|
package/server/db/schema.js
CHANGED
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
import { sql } from 'drizzle-orm';
|
|
2
2
|
import {
|
|
3
|
+
bigint,
|
|
3
4
|
boolean,
|
|
4
5
|
check,
|
|
5
6
|
date,
|
|
6
7
|
foreignKey,
|
|
8
|
+
index,
|
|
7
9
|
integer,
|
|
8
10
|
jsonb,
|
|
9
11
|
pgTable,
|
|
10
12
|
text,
|
|
11
13
|
timestamp,
|
|
12
14
|
unique,
|
|
15
|
+
uniqueIndex,
|
|
13
16
|
uuid,
|
|
14
17
|
} from 'drizzle-orm/pg-core';
|
|
15
18
|
|
|
16
|
-
// Every table except `organisations`
|
|
19
|
+
// Every table except `organisations` and the installation-wide tables (people,
|
|
20
|
+
// integration settings) carries `organisation_id`. Child rows
|
|
17
21
|
// point at their parent through (organisation_id, id), so the database itself
|
|
18
22
|
// refuses a row that joins two organisations. Money is integer cents.
|
|
19
23
|
|
|
@@ -304,3 +308,185 @@ export const obligationWritebacks = pgTable('obligation_writebacks', {
|
|
|
304
308
|
}),
|
|
305
309
|
check('obligation_writebacks_state_valid', inList('state', WRITEBACK_STATES)),
|
|
306
310
|
]);
|
|
311
|
+
|
|
312
|
+
// The data spine (docs/DATA_SPINE.md, issue 56): the tables every integration
|
|
313
|
+
// shares. Each row changes state; the spine never deletes one.
|
|
314
|
+
|
|
315
|
+
// Identity map (issue 57). Which external record is each Flow record. At most
|
|
316
|
+
// one active link per external record and per Flow record, in each system. A
|
|
317
|
+
// link that is not exact stays `proposed` until a person confirms it.
|
|
318
|
+
export const LINK_ENTITIES = ['contact', 'price_item', 'obligation', 'invoice', 'payment', 'credit_note'];
|
|
319
|
+
export const LINK_MATCHES = ['created_by_flow', 'exact_id', 'imported', 'name', 'person'];
|
|
320
|
+
export const LINK_STATES = ['active', 'proposed', 'superseded'];
|
|
321
|
+
|
|
322
|
+
export const externalLinks = pgTable('external_links', {
|
|
323
|
+
id: id(),
|
|
324
|
+
organisationId: organisationId().references(() => organisations.id),
|
|
325
|
+
entity: text('entity').notNull(),
|
|
326
|
+
flowId: uuid('flow_id').notNull(),
|
|
327
|
+
system: text('system').notNull(),
|
|
328
|
+
externalId: text('external_id').notNull(),
|
|
329
|
+
matchedBy: text('matched_by').notNull(),
|
|
330
|
+
state: text('state').notNull(),
|
|
331
|
+
confirmedBy: uuid('confirmed_by').references(() => people.id, { onDelete: 'set null' }),
|
|
332
|
+
confirmedAt: timestamp('confirmed_at', { withTimezone: true }),
|
|
333
|
+
createdAt: createdAt(),
|
|
334
|
+
updatedAt: updatedAt(),
|
|
335
|
+
}, (t) => [
|
|
336
|
+
unique('external_links_org_id').on(t.organisationId, t.id),
|
|
337
|
+
uniqueIndex('external_links_active_external')
|
|
338
|
+
.on(t.organisationId, t.system, t.entity, t.externalId).where(sql`state = 'active'`),
|
|
339
|
+
uniqueIndex('external_links_active_flow')
|
|
340
|
+
.on(t.organisationId, t.system, t.entity, t.flowId).where(sql`state = 'active'`),
|
|
341
|
+
index('external_links_flow').on(t.organisationId, t.entity, t.flowId),
|
|
342
|
+
check('external_links_entity_valid', inList('entity', LINK_ENTITIES)),
|
|
343
|
+
check('external_links_matched_by_valid', inList('matched_by', LINK_MATCHES)),
|
|
344
|
+
check('external_links_state_valid', inList('state', LINK_STATES)),
|
|
345
|
+
check('external_links_name_needs_person', sql`${t.matchedBy} <> 'name' or ${t.state} <> 'active' or ${t.confirmedAt} is not null`),
|
|
346
|
+
]);
|
|
347
|
+
|
|
348
|
+
// Links between a work system's job and an accounting record, carried over
|
|
349
|
+
// from an earlier installation (server/import/at-start.js). They wait here
|
|
350
|
+
// until the job becomes an obligation; then they are evidence for its link.
|
|
351
|
+
// `person` means a person made the link there; `imported` means the earlier
|
|
352
|
+
// app matched it by itself.
|
|
353
|
+
export const PRIOR_LINK_MATCHES = ['person', 'imported'];
|
|
354
|
+
|
|
355
|
+
export const priorLinks = pgTable('prior_links', {
|
|
356
|
+
id: id(),
|
|
357
|
+
organisationId: organisationId().references(() => organisations.id),
|
|
358
|
+
sourceSystem: text('source_system').notNull(),
|
|
359
|
+
sourceJobRef: text('source_job_ref').notNull(),
|
|
360
|
+
system: text('system').notNull(),
|
|
361
|
+
externalId: text('external_id').notNull(),
|
|
362
|
+
externalNumber: text('external_number'),
|
|
363
|
+
matchedBy: text('matched_by').notNull(),
|
|
364
|
+
matchedAt: timestamp('matched_at', { withTimezone: true }),
|
|
365
|
+
origin: text('origin').notNull(),
|
|
366
|
+
createdAt: createdAt(),
|
|
367
|
+
}, (t) => [
|
|
368
|
+
uniqueIndex('prior_links_job').on(t.organisationId, t.sourceSystem, t.sourceJobRef, t.system),
|
|
369
|
+
index('prior_links_external').on(t.organisationId, t.system, t.externalId),
|
|
370
|
+
check('prior_links_matched_by_valid', inList('matched_by', PRIOR_LINK_MATCHES)),
|
|
371
|
+
]);
|
|
372
|
+
|
|
373
|
+
// Background job queue (issue 6). Workers claim with FOR UPDATE SKIP LOCKED and
|
|
374
|
+
// hold a lease; a lease that runs out makes the job claimable again. One queued
|
|
375
|
+
// or running job per (organisation, kind, dedupe key).
|
|
376
|
+
export const BACKGROUND_JOB_STATES = ['queued', 'running', 'succeeded', 'failed', 'dead'];
|
|
377
|
+
|
|
378
|
+
export const backgroundJobs = pgTable('background_jobs', {
|
|
379
|
+
id: id(),
|
|
380
|
+
organisationId: organisationId().references(() => organisations.id),
|
|
381
|
+
kind: text('kind').notNull(),
|
|
382
|
+
dedupeKey: text('dedupe_key').notNull().default(''),
|
|
383
|
+
payload: jsonb('payload').notNull().default(sql`'{}'::jsonb`),
|
|
384
|
+
state: text('state').notNull().default('queued'),
|
|
385
|
+
runAfter: timestamp('run_after', { withTimezone: true }).notNull().defaultNow(),
|
|
386
|
+
attempts: integer('attempts').notNull().default(0),
|
|
387
|
+
maxAttempts: integer('max_attempts').notNull().default(8),
|
|
388
|
+
lockedBy: text('locked_by'),
|
|
389
|
+
lockedUntil: timestamp('locked_until', { withTimezone: true }),
|
|
390
|
+
result: jsonb('result'),
|
|
391
|
+
lastError: text('last_error'),
|
|
392
|
+
createdAt: createdAt(),
|
|
393
|
+
updatedAt: updatedAt(),
|
|
394
|
+
}, (t) => [
|
|
395
|
+
uniqueIndex('background_jobs_one_open')
|
|
396
|
+
.on(t.organisationId, t.kind, t.dedupeKey).where(sql`state in ('queued', 'running')`),
|
|
397
|
+
index('background_jobs_due').on(t.state, t.runAfter),
|
|
398
|
+
check('background_jobs_state_valid', inList('state', BACKGROUND_JOB_STATES)),
|
|
399
|
+
check('background_jobs_attempts_valid', sql`${t.attempts} >= 0 and ${t.maxAttempts} > 0`),
|
|
400
|
+
]);
|
|
401
|
+
|
|
402
|
+
// Outbox. Every write to another system is a row here before the call
|
|
403
|
+
// leaves, written in the same transaction as the Flow change it reports. The
|
|
404
|
+
// idempotency key is unique, so a retry never makes a second record.
|
|
405
|
+
export const OUTBOX_STATES = ['pending', 'sending', 'ambiguous', 'sent', 'failed', 'cancelled'];
|
|
406
|
+
|
|
407
|
+
export const outbox = pgTable('outbox', {
|
|
408
|
+
id: id(),
|
|
409
|
+
organisationId: organisationId().references(() => organisations.id),
|
|
410
|
+
system: text('system').notNull(),
|
|
411
|
+
action: text('action').notNull(),
|
|
412
|
+
entity: text('entity').notNull(),
|
|
413
|
+
flowId: uuid('flow_id').notNull(),
|
|
414
|
+
idempotencyKey: text('idempotency_key').notNull(),
|
|
415
|
+
request: jsonb('request').notNull(),
|
|
416
|
+
state: text('state').notNull().default('pending'),
|
|
417
|
+
externalId: text('external_id'),
|
|
418
|
+
attempts: integer('attempts').notNull().default(0),
|
|
419
|
+
lastError: text('last_error'),
|
|
420
|
+
createdBy: text('created_by').notNull(),
|
|
421
|
+
sentAt: timestamp('sent_at', { withTimezone: true }),
|
|
422
|
+
createdAt: createdAt(),
|
|
423
|
+
updatedAt: updatedAt(),
|
|
424
|
+
}, (t) => [
|
|
425
|
+
unique('outbox_org_id').on(t.organisationId, t.id),
|
|
426
|
+
unique('outbox_idempotency_key').on(t.idempotencyKey),
|
|
427
|
+
index('outbox_pending').on(t.system, t.state, t.createdAt),
|
|
428
|
+
check('outbox_state_valid', inList('state', OUTBOX_STATES)),
|
|
429
|
+
check('outbox_sent_has_id', sql`${t.state} <> 'sent' or ${t.externalId} is not null`),
|
|
430
|
+
]);
|
|
431
|
+
|
|
432
|
+
// Review queue (issue 59). One list of everything a person must decide. One
|
|
433
|
+
// open item per (organisation, kind, entity, Flow id); a repeat finding
|
|
434
|
+
// updates its evidence.
|
|
435
|
+
export const REVIEW_STATES = ['open', 'resolved', 'dismissed'];
|
|
436
|
+
|
|
437
|
+
export const reviewItems = pgTable('review_items', {
|
|
438
|
+
id: id(),
|
|
439
|
+
organisationId: organisationId().references(() => organisations.id),
|
|
440
|
+
kind: text('kind').notNull(),
|
|
441
|
+
entity: text('entity').notNull(),
|
|
442
|
+
flowId: uuid('flow_id').notNull(),
|
|
443
|
+
systems: text('systems').array().notNull().default(sql`'{}'::text[]`),
|
|
444
|
+
evidence: jsonb('evidence').notNull().default(sql`'{}'::jsonb`),
|
|
445
|
+
moneyAtRiskCents: bigint('money_at_risk_cents', { mode: 'number' }).notNull().default(0),
|
|
446
|
+
state: text('state').notNull().default('open'),
|
|
447
|
+
seenCount: integer('seen_count').notNull().default(1),
|
|
448
|
+
resolution: text('resolution'),
|
|
449
|
+
note: text('note'),
|
|
450
|
+
resolvedBy: text('resolved_by'),
|
|
451
|
+
resolvedAt: timestamp('resolved_at', { withTimezone: true }),
|
|
452
|
+
createdAt: createdAt(),
|
|
453
|
+
updatedAt: updatedAt(),
|
|
454
|
+
}, (t) => [
|
|
455
|
+
unique('review_items_org_id').on(t.organisationId, t.id),
|
|
456
|
+
uniqueIndex('review_items_one_open')
|
|
457
|
+
.on(t.organisationId, t.kind, t.entity, t.flowId).where(sql`state = 'open'`),
|
|
458
|
+
index('review_items_queue').on(t.organisationId, t.state, t.moneyAtRiskCents),
|
|
459
|
+
check('review_items_state_valid', inList('state', REVIEW_STATES)),
|
|
460
|
+
check('review_items_money_not_negative', sql`${t.moneyAtRiskCents} >= 0`),
|
|
461
|
+
check('review_items_closed_has_resolution', sql`${t.state} = 'open' or (${t.resolution} is not null and ${t.resolvedBy} is not null and ${t.resolvedAt} is not null)`),
|
|
462
|
+
]);
|
|
463
|
+
|
|
464
|
+
// Integration settings an owner enters in Settings (server/integrations/setup.js).
|
|
465
|
+
// One row per adapter for the installation. `config` holds plain settings by
|
|
466
|
+
// their environment name; `secrets` holds credentials sealed with Flow's
|
|
467
|
+
// secrets key (server/core/secrets.js), never in clear. A host's own
|
|
468
|
+
// environment setting wins over a value here.
|
|
469
|
+
export const integrationSettings = pgTable('integration_settings', {
|
|
470
|
+
adapterId: text('adapter_id').primaryKey(),
|
|
471
|
+
enabled: boolean('enabled').notNull().default(false),
|
|
472
|
+
config: jsonb('config').notNull().default(sql`'{}'::jsonb`),
|
|
473
|
+
secrets: jsonb('secrets').notNull().default(sql`'{}'::jsonb`),
|
|
474
|
+
updatedBy: uuid('updated_by').references(() => people.id, { onDelete: 'set null' }),
|
|
475
|
+
createdAt: createdAt(),
|
|
476
|
+
updatedAt: updatedAt(),
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
// Who changed which integration setting, and when. Names only, never a value.
|
|
480
|
+
export const INTEGRATION_SETTING_EVENTS = ['turned_on', 'turned_off', 'setting_changed', 'secret_set', 'secret_removed'];
|
|
481
|
+
|
|
482
|
+
export const integrationSettingEvents = pgTable('integration_setting_events', {
|
|
483
|
+
id: id(),
|
|
484
|
+
adapterId: text('adapter_id').notNull(),
|
|
485
|
+
kind: text('kind').notNull(),
|
|
486
|
+
settings: text('settings').array().notNull().default(sql`'{}'::text[]`),
|
|
487
|
+
actor: uuid('actor').references(() => people.id, { onDelete: 'set null' }),
|
|
488
|
+
createdAt: createdAt(),
|
|
489
|
+
}, (t) => [
|
|
490
|
+
index('integration_setting_events_adapter').on(t.adapterId, t.createdAt),
|
|
491
|
+
check('integration_setting_events_kind_valid', inList('kind', INTEGRATION_SETTING_EVENTS)),
|
|
492
|
+
]);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { sql } from 'drizzle-orm';
|
|
2
|
-
import { organisations } from '../db/schema.js';
|
|
2
|
+
import { organisations, priorLinks } from '../db/schema.js';
|
|
3
3
|
import { applyImport, planImport, readEarlierFlow, summarisePlan } from './earlier-flow.js';
|
|
4
4
|
|
|
5
5
|
// The import from the earlier codebase, run once by the server itself. On a
|
|
@@ -50,3 +50,71 @@ export async function importEarlierAtStart({ db, pool, settings, log = console }
|
|
|
50
50
|
log.info?.(`[import] imported: ${JSON.stringify(result.counts)}`);
|
|
51
51
|
return result.counts;
|
|
52
52
|
}
|
|
53
|
+
|
|
54
|
+
// The earlier app's table of jobs bound to accounting invoices. Each row ties a
|
|
55
|
+
// Seazona order to the Xero invoice that already bills it.
|
|
56
|
+
const EARLIER_LINKS = 'seazona_order_xero_match_overrides';
|
|
57
|
+
const LINK_ORIGIN = `earlier:${EARLIER_LINKS}`;
|
|
58
|
+
const LINK_BATCH = 500;
|
|
59
|
+
|
|
60
|
+
function matchedAt(text) {
|
|
61
|
+
const at = text ? new Date(text) : null;
|
|
62
|
+
return at && !Number.isNaN(at.getTime()) ? at : null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Carries the earlier app's order -> invoice links into `prior_links`, at
|
|
66
|
+
// every start while the import settings are set. It reads in a read-only
|
|
67
|
+
// transaction and only adds links it does not hold, so a repeat costs one read
|
|
68
|
+
// and changes nothing. It copies ids and how each link was made; never the
|
|
69
|
+
// earlier notes or who made it, which can name people. Answers counts, or null
|
|
70
|
+
// when there is nothing to read.
|
|
71
|
+
export async function importEarlierLinksAtStart({ db, pool, settings, log = console }) {
|
|
72
|
+
if (!settings) return null;
|
|
73
|
+
// The lab the import made, by name; or the only organisation, if renamed since.
|
|
74
|
+
const { rows: labs } = await db.execute(sql`select id, name from ${organisations}`);
|
|
75
|
+
const organisation = labs.find((lab) => lab.name === settings.organisationName) ?? (labs.length === 1 ? labs[0] : null);
|
|
76
|
+
if (!organisation) return null;
|
|
77
|
+
|
|
78
|
+
let rows;
|
|
79
|
+
const client = await pool.connect();
|
|
80
|
+
try {
|
|
81
|
+
await client.query('BEGIN TRANSACTION READ ONLY');
|
|
82
|
+
await client.query(`SET LOCAL search_path TO "${settings.schema}"`);
|
|
83
|
+
const { rows: [found] } = await client.query('select to_regclass($1) is not null as present', [EARLIER_LINKS]);
|
|
84
|
+
if (found.present) {
|
|
85
|
+
({ rows } = await client.query(`select seazona_order_id, xero_invoice_id, xero_invoice_number, match_source, matched_at
|
|
86
|
+
from ${EARLIER_LINKS} where coalesce(xero_invoice_id, '') <> '' and coalesce(seazona_order_id, '') <> ''`));
|
|
87
|
+
}
|
|
88
|
+
await client.query('COMMIT');
|
|
89
|
+
} catch (error) {
|
|
90
|
+
await client.query('ROLLBACK').catch(() => {});
|
|
91
|
+
throw new Error(`could not read links from schema "${settings.schema}": ${error.message}`);
|
|
92
|
+
} finally {
|
|
93
|
+
client.release();
|
|
94
|
+
}
|
|
95
|
+
if (!rows) {
|
|
96
|
+
log.info?.(`[import] links: schema "${settings.schema}" has no ${EARLIER_LINKS}`);
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const values = rows.map((row) => ({
|
|
101
|
+
organisationId: organisation.id,
|
|
102
|
+
sourceSystem: 'seazona',
|
|
103
|
+
sourceJobRef: String(row.seazona_order_id).trim(),
|
|
104
|
+
system: 'xero',
|
|
105
|
+
externalId: String(row.xero_invoice_id).trim(),
|
|
106
|
+
externalNumber: row.xero_invoice_number?.trim() || null,
|
|
107
|
+
matchedBy: row.match_source === 'operator_link' ? 'person' : 'imported',
|
|
108
|
+
matchedAt: matchedAt(row.matched_at),
|
|
109
|
+
origin: LINK_ORIGIN,
|
|
110
|
+
}));
|
|
111
|
+
let added = 0;
|
|
112
|
+
for (let start = 0; start < values.length; start += LINK_BATCH) {
|
|
113
|
+
const written = await db.insert(priorLinks).values(values.slice(start, start + LINK_BATCH))
|
|
114
|
+
.onConflictDoNothing().returning({ id: priorLinks.id });
|
|
115
|
+
added += written.length;
|
|
116
|
+
}
|
|
117
|
+
const counts = { read: values.length, added };
|
|
118
|
+
log.info?.(`[import] links from schema "${settings.schema}": ${JSON.stringify(counts)}`);
|
|
119
|
+
return counts;
|
|
120
|
+
}
|
package/server/index.js
CHANGED
|
@@ -5,8 +5,11 @@ import { seedFirstOwner } from './auth/people.js';
|
|
|
5
5
|
import { createPersonAuth } from './auth/session.js';
|
|
6
6
|
import { resolveSignInSettings } from './auth/settings.js';
|
|
7
7
|
import { migrateDatabase, openDatabase } from './db/index.js';
|
|
8
|
-
import { importEarlierAtStart, resolveImportAtStart } from './import/at-start.js';
|
|
9
|
-
import {
|
|
8
|
+
import { importEarlierAtStart, importEarlierLinksAtStart, resolveImportAtStart } from './import/at-start.js';
|
|
9
|
+
import { resolveSecretsKey } from './core/secrets.js';
|
|
10
|
+
import { adapters as shippedAdapters } from './integrations/adapters.js';
|
|
11
|
+
import { createIntegrations, resolveEnabledIntegrations } from './integrations/index.js';
|
|
12
|
+
import { readStoredSettings, settingsEnvironment, storedEnabled } from './integrations/setup.js';
|
|
10
13
|
import { resolveDefaultFamily } from './theme/themes.js';
|
|
11
14
|
import { createUpdateChecker, UPDATE_CHECK_INTERVAL_MS } from './update-check.js';
|
|
12
15
|
|
|
@@ -27,9 +30,10 @@ try {
|
|
|
27
30
|
console.error(`[ui] ${error.message}`);
|
|
28
31
|
process.exit(1);
|
|
29
32
|
}
|
|
30
|
-
let
|
|
33
|
+
let secrets;
|
|
31
34
|
try {
|
|
32
|
-
|
|
35
|
+
resolveEnabledIntegrations();
|
|
36
|
+
secrets = resolveSecretsKey();
|
|
33
37
|
} catch (error) {
|
|
34
38
|
console.error(`[integrations] ${error.message}`);
|
|
35
39
|
process.exit(1);
|
|
@@ -55,9 +59,25 @@ try {
|
|
|
55
59
|
console.error(`[import] ${error.message}`);
|
|
56
60
|
process.exit(1);
|
|
57
61
|
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
62
|
+
// The earlier links are evidence to add, not records Flow needs to open, so a
|
|
63
|
+
// failure is logged and the start goes on.
|
|
64
|
+
await importEarlierLinksAtStart({ db: database.db, pool: database.pool, settings: importAtStart })
|
|
65
|
+
.catch((error) => console.error(`[import] ${error.message}`));
|
|
66
|
+
// What an owner set in Settings, Integrations, with the host's own
|
|
67
|
+
// environment over it (server/integrations/setup.js).
|
|
68
|
+
let integrations;
|
|
69
|
+
try {
|
|
70
|
+
const stored = await readStoredSettings(database.db);
|
|
71
|
+
const { env: settingsEnv, unreadable } = settingsEnvironment({ env: process.env, stored, adapters: shippedAdapters, secrets });
|
|
72
|
+
if (unreadable.length) console.warn(`[integrations] stored credentials Flow cannot read with this secrets key: ${unreadable.join(', ')}. An owner enters them again in Settings.`);
|
|
73
|
+
integrations = createIntegrations({ turnedOnInFlow: storedEnabled(stored, process.env), settingsEnv });
|
|
74
|
+
await integrations.start(database.db);
|
|
75
|
+
} catch (error) {
|
|
76
|
+
console.error(`[integrations] ${error.message}`);
|
|
77
|
+
process.exit(1);
|
|
78
|
+
}
|
|
79
|
+
const integrationsOn = integrations.list().filter((a) => a.enabled).map((a) => `${a.id}${a.state === 'ready' ? '' : ` (${a.state})`}`);
|
|
80
|
+
console.log(`[integrations] on: ${integrationsOn.join(', ') || 'none'}; secrets key ${secrets ? `from ${secrets.source}` : 'not set (FLOW_SECRETS_KEY)'}`);
|
|
61
81
|
|
|
62
82
|
let auth = null;
|
|
63
83
|
if (settings.mode === 'clerk') {
|
|
@@ -100,7 +120,7 @@ const trustProxy = process.env.REPL_ID || process.env.REPLIT_DEPLOYMENT ? 1 : fa
|
|
|
100
120
|
|
|
101
121
|
const updates = createUpdateChecker();
|
|
102
122
|
|
|
103
|
-
createApp({ database, auth, trustProxy, updates, integrations, defaultFamily }).listen(port, () => {
|
|
123
|
+
createApp({ database, auth, trustProxy, updates, integrations, secrets, defaultFamily }).listen(port, () => {
|
|
104
124
|
console.log(`[flow] listening on ${port}; design system ${defaultFamily} (FLOW_THEME)`);
|
|
105
125
|
});
|
|
106
126
|
|
|
@@ -16,11 +16,17 @@ export function resolveEnabledIntegrations(env = process.env, adapters = shipped
|
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
// The registry for one Flow process: every shipped adapter registered, the
|
|
19
|
-
// chosen ones enabled.
|
|
20
|
-
|
|
19
|
+
// chosen ones enabled. `turnedOnInFlow` are the ids an owner turned on in
|
|
20
|
+
// Settings (server/integrations/setup.js): one of those that cannot start is
|
|
21
|
+
// reported, not fatal. `settingsEnv` is what adapters read as their settings.
|
|
22
|
+
// Call registry.start(db) after the core migrations.
|
|
23
|
+
export function createIntegrations({ env = process.env, adapters = shipped, log = console, turnedOnInFlow = [], settingsEnv = env } = {}) {
|
|
21
24
|
const enabled = resolveEnabledIntegrations(env, adapters);
|
|
22
|
-
const registry = createRegistry({ log });
|
|
25
|
+
const registry = createRegistry({ log, env: settingsEnv });
|
|
23
26
|
for (const adapter of adapters) registry.register(adapter);
|
|
24
27
|
for (const id of enabled) registry.enable(id);
|
|
28
|
+
for (const id of turnedOnInFlow) {
|
|
29
|
+
if (!enabled.includes(id) && registry.get(id)) registry.enable(id, { optional: true });
|
|
30
|
+
}
|
|
25
31
|
return registry;
|
|
26
32
|
}
|
|
@@ -53,6 +53,7 @@ export function turnedOff(adapter, reason = 'disabled') {
|
|
|
53
53
|
disabled: `${adapter.name} is turned off on this installation.`,
|
|
54
54
|
restart_needed: `${adapter.name} was turned on after Flow started. Restart Flow to finish turning it on.`,
|
|
55
55
|
tables_missing: `${adapter.name} has no tables on this installation. Restart Flow to create them.`,
|
|
56
|
+
failed: `${adapter.name} could not start. An owner can check its settings in Settings, Integrations.`,
|
|
56
57
|
}[reason];
|
|
57
58
|
return { ok: false, code: 'integration_off', integration: adapter.id, reason, error };
|
|
58
59
|
}
|
|
@@ -61,7 +62,7 @@ export function defineAdapter(adapter) {
|
|
|
61
62
|
if (!adapter || typeof adapter !== 'object') {
|
|
62
63
|
throw new TypeError('An adapter must be an object');
|
|
63
64
|
}
|
|
64
|
-
const { id, name, bundled = false, capabilities, schema, start, stop, routes } = adapter;
|
|
65
|
+
const { id, name, bundled = false, capabilities, schema, start, stop, routes, setup } = adapter;
|
|
65
66
|
if (typeof id !== 'string' || !ID_PATTERN.test(id)) {
|
|
66
67
|
throw new TypeError(`Adapter id must match ${ID_PATTERN}`);
|
|
67
68
|
}
|
|
@@ -90,6 +91,14 @@ export function defineAdapter(adapter) {
|
|
|
90
91
|
if (routes !== undefined && typeof routes !== 'function') {
|
|
91
92
|
throw new TypeError(`Adapter ${id} routes must be a function that returns a router`);
|
|
92
93
|
}
|
|
94
|
+
if (setup !== undefined && (!setup || !Array.isArray(setup.fields))) {
|
|
95
|
+
throw new TypeError(`Adapter ${id} setup must list its fields`);
|
|
96
|
+
}
|
|
97
|
+
for (const field of setup?.fields ?? []) {
|
|
98
|
+
if (!/^[A-Z][A-Z0-9_]{1,63}$/.test(field?.name ?? '') || !field.label) {
|
|
99
|
+
throw new TypeError(`Adapter ${id} setup field ${field?.name} needs an upper-case setting name and a label`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
93
102
|
return Object.freeze({
|
|
94
103
|
id,
|
|
95
104
|
name,
|
|
@@ -105,12 +114,24 @@ export function defineAdapter(adapter) {
|
|
|
105
114
|
// Optional HTTP routes: routes({ db, personAuth, log, messagePage }) returns a router
|
|
106
115
|
// mounted at /api/<id> (registry.routes).
|
|
107
116
|
routes: routes ?? null,
|
|
117
|
+
// Optional: the settings an owner may enter in Settings, Integrations
|
|
118
|
+
// (server/integrations/setup.js). `fields` name environment settings;
|
|
119
|
+
// `derivedKeys` name key settings Flow derives from its secrets key.
|
|
120
|
+
setup: setup ? Object.freeze({ fields: Object.freeze(setup.fields.map((f) => Object.freeze({ ...f }))), derivedKeys: Object.freeze([...(setup.derivedKeys ?? [])]), guide: setup.guide ?? null }) : null,
|
|
108
121
|
});
|
|
109
122
|
}
|
|
110
123
|
|
|
111
|
-
export function createRegistry({ log = console } = {}) {
|
|
124
|
+
export function createRegistry({ log = console, env = process.env } = {}) {
|
|
112
125
|
const adapters = new Map();
|
|
113
126
|
const enabled = new Set();
|
|
127
|
+
// Adapters turned on in Settings, not by the host: one that cannot start
|
|
128
|
+
// is reported, and Flow starts without it.
|
|
129
|
+
const optional = new Set();
|
|
130
|
+
const failures = new Map(); // id -> message
|
|
131
|
+
const generation = new Map(); // id -> count of (re)starts, for its router
|
|
132
|
+
const routers = new Map(); // id -> { generation, router }
|
|
133
|
+
let settingsEnv = env;
|
|
134
|
+
let database = null;
|
|
114
135
|
// Adapters whose tables this process created or found at start. An adapter
|
|
115
136
|
// without a schema needs none, so it is ready as soon as it is enabled.
|
|
116
137
|
const tablesReady = new Set();
|
|
@@ -119,6 +140,7 @@ export function createRegistry({ log = console } = {}) {
|
|
|
119
140
|
|
|
120
141
|
const state = (adapter) => {
|
|
121
142
|
if (!enabled.has(adapter.id)) return 'disabled';
|
|
143
|
+
if (failures.has(adapter.id)) return 'failed';
|
|
122
144
|
if (!adapter.schema || tablesReady.has(adapter.id)) return 'ready';
|
|
123
145
|
return started ? 'restart_needed' : 'not_started';
|
|
124
146
|
};
|
|
@@ -133,9 +155,11 @@ export function createRegistry({ log = console } = {}) {
|
|
|
133
155
|
adapters.set(defined.id, defined);
|
|
134
156
|
return defined;
|
|
135
157
|
},
|
|
136
|
-
enable(id) {
|
|
158
|
+
enable(id, { optional: isOptional = false } = {}) {
|
|
137
159
|
if (!adapters.has(id)) throw new Error(`No adapter ${id}`);
|
|
138
160
|
enabled.add(id);
|
|
161
|
+
if (isOptional) optional.add(id);
|
|
162
|
+
else optional.delete(id);
|
|
139
163
|
},
|
|
140
164
|
// Turning an adapter off never drops its tables. Removing them is a
|
|
141
165
|
// separate step (removeAdapterTables in server/integrations/schema.js).
|
|
@@ -149,6 +173,7 @@ export function createRegistry({ log = console } = {}) {
|
|
|
149
173
|
async start(db) {
|
|
150
174
|
if (started) throw new Error('The integration registry has already started');
|
|
151
175
|
started = true;
|
|
176
|
+
database = db;
|
|
152
177
|
const migrated = [];
|
|
153
178
|
for (const id of enabled) {
|
|
154
179
|
const adapter = adapters.get(id);
|
|
@@ -160,11 +185,66 @@ export function createRegistry({ log = console } = {}) {
|
|
|
160
185
|
for (const id of enabled) {
|
|
161
186
|
const adapter = adapters.get(id);
|
|
162
187
|
if (!adapter.start) continue;
|
|
163
|
-
|
|
188
|
+
try {
|
|
189
|
+
await adapter.start({ db, log, env: settingsEnv });
|
|
190
|
+
} catch (error) {
|
|
191
|
+
if (!optional.has(id)) throw error;
|
|
192
|
+
failures.set(id, error.message);
|
|
193
|
+
log.error?.(`[integrations] ${id} could not start: ${error.message}`);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
164
196
|
running.push(adapter);
|
|
165
197
|
}
|
|
166
198
|
return migrated;
|
|
167
199
|
},
|
|
200
|
+
// Applies new settings to one adapter while Flow runs: stops it, then,
|
|
201
|
+
// when it stays on, creates its tables and starts it with `env`. Its
|
|
202
|
+
// routes are built again on the next request. A start that fails leaves
|
|
203
|
+
// it `failed` with the reason, and the rest of Flow running.
|
|
204
|
+
async reconfigure(id, { enabled: on, env: next }) {
|
|
205
|
+
const adapter = adapters.get(id);
|
|
206
|
+
if (!adapter) throw new Error(`No adapter ${id}`);
|
|
207
|
+
if (!started) throw new Error('The integration registry has not started');
|
|
208
|
+
if (next) settingsEnv = next;
|
|
209
|
+
const index = running.indexOf(adapter);
|
|
210
|
+
if (index >= 0) {
|
|
211
|
+
running.splice(index, 1);
|
|
212
|
+
await adapter.stop?.();
|
|
213
|
+
}
|
|
214
|
+
failures.delete(id);
|
|
215
|
+
generation.set(id, (generation.get(id) ?? 0) + 1);
|
|
216
|
+
if (!on) {
|
|
217
|
+
enabled.delete(id);
|
|
218
|
+
return state(adapter);
|
|
219
|
+
}
|
|
220
|
+
enabled.add(id);
|
|
221
|
+
optional.add(id);
|
|
222
|
+
try {
|
|
223
|
+
if (adapter.schema && !tablesReady.has(id)) {
|
|
224
|
+
await migrateAdapterSchema(database, adapter);
|
|
225
|
+
tablesReady.add(id);
|
|
226
|
+
}
|
|
227
|
+
if (adapter.start) {
|
|
228
|
+
await adapter.start({ db: database, log, env: settingsEnv });
|
|
229
|
+
running.push(adapter);
|
|
230
|
+
}
|
|
231
|
+
} catch (error) {
|
|
232
|
+
failures.set(id, error.message);
|
|
233
|
+
log.error?.(`[integrations] ${id} could not start: ${error.message}`);
|
|
234
|
+
}
|
|
235
|
+
return state(adapter);
|
|
236
|
+
},
|
|
237
|
+
get(id) {
|
|
238
|
+
return adapters.get(id) ?? null;
|
|
239
|
+
},
|
|
240
|
+
// Why an enabled adapter is not running, or null. The message names
|
|
241
|
+
// settings, never their values.
|
|
242
|
+
failure(id) {
|
|
243
|
+
return failures.get(id) ?? null;
|
|
244
|
+
},
|
|
245
|
+
env() {
|
|
246
|
+
return settingsEnv;
|
|
247
|
+
},
|
|
168
248
|
// Calls stop() on every adapter whose start() ran, newest first.
|
|
169
249
|
async stop() {
|
|
170
250
|
while (running.length) {
|
|
@@ -180,6 +260,7 @@ export function createRegistry({ log = console } = {}) {
|
|
|
180
260
|
enabled: enabled.has(a.id),
|
|
181
261
|
state: state(a),
|
|
182
262
|
tables: a.schema ? [...a.schema.tables] : [],
|
|
263
|
+
...(failures.has(a.id) ? { error: failures.get(a.id) } : {}),
|
|
183
264
|
capabilities: Object.keys(a.capabilities),
|
|
184
265
|
}));
|
|
185
266
|
},
|
|
@@ -202,10 +283,21 @@ export function createRegistry({ log = console } = {}) {
|
|
|
202
283
|
routes(context) {
|
|
203
284
|
return [...adapters.values()].filter((a) => a.routes).map((adapter) => {
|
|
204
285
|
const path = `/api/${adapter.id}`;
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
286
|
+
const handler = (req, res, next) => {
|
|
287
|
+
const now = state(adapter);
|
|
288
|
+
if (now !== 'ready') {
|
|
289
|
+
const reason = now === 'disabled' || now === 'failed' ? now : 'restart_needed';
|
|
290
|
+
return res.status(503).json(turnedOff(adapter, reason));
|
|
291
|
+
}
|
|
292
|
+
const current = generation.get(adapter.id) ?? 0;
|
|
293
|
+
let built = routers.get(adapter.id);
|
|
294
|
+
if (!built || built.generation !== current) {
|
|
295
|
+
built = { generation: current, router: adapter.routes(context) };
|
|
296
|
+
routers.set(adapter.id, built);
|
|
297
|
+
}
|
|
298
|
+
return built.router(req, res, next);
|
|
299
|
+
};
|
|
300
|
+
return { path, handler };
|
|
209
301
|
});
|
|
210
302
|
},
|
|
211
303
|
// The one way to read or write an adapter's tables: runs fn(adapter) when
|
|
@@ -216,7 +308,7 @@ export function createRegistry({ log = console } = {}) {
|
|
|
216
308
|
const adapter = adapters.get(id);
|
|
217
309
|
if (!adapter) throw new Error(`No adapter ${id}`);
|
|
218
310
|
const now = state(adapter);
|
|
219
|
-
if (now === 'disabled') return turnedOff(adapter,
|
|
311
|
+
if (now === 'disabled' || now === 'failed') return turnedOff(adapter, now);
|
|
220
312
|
if (now !== 'ready') return turnedOff(adapter, 'restart_needed');
|
|
221
313
|
try {
|
|
222
314
|
return await fn(adapter);
|
|
@@ -22,7 +22,27 @@ export function requireSeazonaSettings(env = process.env) {
|
|
|
22
22
|
|
|
23
23
|
// `env`, `fetchImpl`, `now` and `timers` are for tests. Otherwise start()
|
|
24
24
|
// reads the environment, so a turned-off adapter never reads its settings.
|
|
25
|
-
|
|
25
|
+
// What an owner enters in Settings, Integrations (server/integrations/setup.js).
|
|
26
|
+
export const SEAZONA_SETUP = {
|
|
27
|
+
guide: {
|
|
28
|
+
title: 'Get a Seazona API client',
|
|
29
|
+
link: null,
|
|
30
|
+
steps: [
|
|
31
|
+
'In Seazona, create an API client for Flow. If you cannot find the option, ask Seazona support for API access.',
|
|
32
|
+
'Copy the API address, the client id and the client secret.',
|
|
33
|
+
'Paste them below, choose the mode, and turn Seazona on.',
|
|
34
|
+
'Live mode starts the background sync. Flow only reads from Seazona.',
|
|
35
|
+
],
|
|
36
|
+
},
|
|
37
|
+
fields: [
|
|
38
|
+
{ name: 'SEAZONA_MODE', label: 'Mode', kind: 'choice', choices: [{ value: 'live', label: 'Live: your Seazona account' }, { value: 'sandbox', label: 'Sandbox: a Seazona test account' }] },
|
|
39
|
+
{ name: 'SEAZONA_API_BASE_URL', label: 'API address', kind: 'url', help: 'The API root of your account, https://…' },
|
|
40
|
+
{ name: 'SEAZONA_API_CLIENT_ID', label: 'Client id' },
|
|
41
|
+
{ name: 'SEAZONA_API_CLIENT_SECRET', label: 'Client secret', secret: true, help: 'Stored encrypted. Flow never shows it again.' },
|
|
42
|
+
],
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export function createSeazonaAdapter({ env = null, fetchImpl = null, now = Date.now, timers } = {}) {
|
|
26
46
|
let runtime = null; // { settings, db, client, scheduler }
|
|
27
47
|
|
|
28
48
|
function current() {
|
|
@@ -36,9 +56,10 @@ export function createSeazonaAdapter({ env = process.env, fetchImpl = null, now
|
|
|
36
56
|
// VISION.md: Seazona is bundled and on in the dental preset.
|
|
37
57
|
bundled: true,
|
|
38
58
|
schema: seazonaSchema,
|
|
59
|
+
setup: SEAZONA_SETUP,
|
|
39
60
|
|
|
40
|
-
async start({ db, log = console }) {
|
|
41
|
-
const settings = requireSeazonaSettings(env);
|
|
61
|
+
async start({ db, log = console, env: settingsEnv }) {
|
|
62
|
+
const settings = requireSeazonaSettings(env ?? settingsEnv ?? process.env);
|
|
42
63
|
const client = createSeazonaClient({ settings, db, fetchImpl: fetchImpl ?? undefined, workflow: 'seazona-sync', now });
|
|
43
64
|
const run = (options = {}) => runSeazonaSync(db, client, { settings, now, ...options });
|
|
44
65
|
// The background sync starts in live mode only. A sandbox or replay
|
|
@@ -51,6 +72,7 @@ export function createSeazonaAdapter({ env = process.env, fetchImpl = null, now
|
|
|
51
72
|
|
|
52
73
|
async stop() {
|
|
53
74
|
await runtime?.scheduler?.stop();
|
|
75
|
+
runtime = null;
|
|
54
76
|
},
|
|
55
77
|
|
|
56
78
|
capabilities: {
|