@serfab/quereus-plugin-sereus 1.14.0 → 1.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,458 +1,484 @@
1
- import debug from 'debug';
2
- import type { Database, VTablePluginInfo, FunctionPluginInfo, CollationPluginInfo } from '@quereus/quereus';
3
- import optimysticPlugin from '@optimystic/quereus-plugin-optimystic/plugin';
4
- import type { Libp2p } from '@libp2p/interface';
5
- import type { IRepo } from '@optimystic/db-core';
6
- import type { IRawStorage, NodeOptions } from '@optimystic/db-p2p';
7
- import type { StrandConnectionOptions, SereusPluginResult, StrandTransactor, Libp2pNodeWithRepo } from './types.js';
8
- import { STRAND_SCHEMA } from './strand-schema.js';
9
- import { resolveStrandClusterSize, STRAND_CLUSTER_POLICY } from './cluster-size.js';
10
- import { wrapStorageWithCache, disposeStorageCache } from './cached-storage.js';
11
-
12
- const log = debug('sereus:plugin:strand');
13
- const timing = debug('sereus:plugin:strand:timing');
14
-
15
- /**
16
- * Minimal interface for the CollectionFactory returned by the optimystic plugin.
17
- * We only need the methods this composition actually uses.
18
- */
19
- interface CollectionFactory {
20
- registerLibp2pNode(networkName: string, node: Libp2p, coordinatedRepo: IRepo): void;
21
- shutdown(): Promise<void>;
22
- }
23
-
24
- /**
25
- * Result of registering the optimystic plugin. The published plugin signature
26
- * does not surface `collectionFactory` / `hydrate`, so this is the local shim
27
- * over what `@optimystic/quereus-plugin-optimystic/plugin` actually returns
28
- * (see its `plugin.ts` — `hydrate` delegates to `optimysticModule.hydrateCatalog`).
29
- */
30
- interface OptimysticPluginResult {
31
- collectionFactory: CollectionFactory;
32
- vtables: VTablePluginInfo[];
33
- functions: FunctionPluginInfo[];
34
- collations?: CollationPluginInfo[];
35
- /**
36
- * Hydrate Quereus's in-memory catalog from persisted optimystic vtab schemas.
37
- * Must run BEFORE `apply schema App;` on a warm restart so the declarative
38
- * diff sees the existing tables and skips re-emitting CREATE TABLE / CREATE
39
- * INDEX for each persisted object. Idempotent; `{ tables: 0, indexes: 0 }`
40
- * on cold start.
41
- */
42
- hydrate: (db: Database) => Promise<{ tables: number; indexes: number }>;
43
- [key: string]: unknown;
44
- }
45
-
46
- /** Registration shape shared by the crypto and optimystic plugin results. */
47
- interface PluginRegistrations {
48
- vtables?: VTablePluginInfo[];
49
- functions?: FunctionPluginInfo[];
50
- collations?: CollationPluginInfo[];
51
- }
52
-
53
- /**
54
- * Inline equivalent of `@quereus/quereus`'s `registerPlugin` for plugins that
55
- * only return functions/vtables/collations. Used directly for the optimystic
56
- * result (so the composition can read `collectionFactory` off the same object),
57
- * and by the browser crypto strategy to keep the browser bundle from pulling a
58
- * duplicate `@quereus/quereus` next to the host's instance.
59
- */
60
- export function applyRegistrations(db: Database, result: PluginRegistrations): void {
61
- for (const vtable of result.vtables ?? []) {
62
- db.registerModule(vtable.name, vtable.module, vtable.auxData);
63
- }
64
- for (const func of result.functions ?? []) {
65
- db.registerFunction(func.schema);
66
- }
67
- for (const collation of result.collations ?? []) {
68
- db.registerCollation(collation.name, collation.func, collation.normalizer);
69
- }
70
- }
71
-
72
- /** Context passed to the platform's node-creation strategy. */
73
- export interface CreateNodeContext {
74
- networkName: string;
75
- bootstrapNodes: string[];
76
- fretProfile: 'edge' | 'core';
77
- /** libp2p listening port (Node TCP). Browser transports ignore it. */
78
- port: number;
79
- /**
80
- * Resolved replication cluster size. Already defaulted — pass it to
81
- * `createLibp2pNode` verbatim; omitting it falls back to Optimystic's own
82
- * default of 10, which gates writes on any smaller party.
83
- */
84
- clusterSize: number;
85
- /**
86
- * The strand cluster policy, resolved here so no platform has to remember it.
87
- * Pass it to `createLibp2pNode` verbatim.
88
- *
89
- * Omitting it does NOT fall back to something equivalent: Optimystic then computes
90
- * its read-repair corroboration floor against `clusterSize` instead of against
91
- * {@link STRAND_CLUSTER_POLICY}'s `assumedClusterSize`, which is a stricter demand
92
- * than a two-machine strand can meet — so repair of a damaged block gives up where
93
- * it would otherwise succeed. That is the same defect class that took the *control*
94
- * network's e2e suite down before `42cd12c`; see `cluster-size.ts` for the reasoning
95
- * written out in full.
96
- */
97
- clusterPolicy: NonNullable<NodeOptions['clusterPolicy']>;
98
- /** Resolved persistent storage to back the node, if any. */
99
- storage?: IRawStorage;
100
- }
101
-
102
- /** Context passed to the platform's storage-resolution strategy. */
103
- export interface ResolveStorageContext {
104
- strandId: string;
105
- resolvedTransactor: StrandTransactor;
106
- /** The storage the caller passed in `options.storage`, if any. */
107
- requestedStorage?: IRawStorage;
108
- }
109
-
110
- /**
111
- * Platform-specific seams the shared composition delegates to. Everything else
112
- * (transactor resolution, plugin config, registration, default vtab, hydrate,
113
- * schema apply, cleanup/shutdown) is identical across Node and browser and lives
114
- * in {@link composeStrand}.
115
- */
116
- export interface StrandPlatform {
117
- /**
118
- * Register the crypto plugin against `db`. Node uses `@quereus/quereus`'s
119
- * `registerPlugin`; the browser inlines `applyRegistrations` to avoid bundling
120
- * a second `@quereus/quereus`.
121
- */
122
- registerCrypto(db: Database): void | Promise<void>;
123
- /**
124
- * Resolve the persistent storage to use. Optional — when omitted the caller's
125
- * `options.storage` is used verbatim (Node). The browser supplies this to
126
- * default to IndexedDB so a reload survives.
127
- */
128
- resolveStorage?(ctx: ResolveStorageContext): Promise<IRawStorage | undefined> | IRawStorage | undefined;
129
- /**
130
- * Create a libp2p node. Called only when no node is injected and the
131
- * transactor needs one. Node creates a TCP node via `@optimystic/db-p2p`;
132
- * the browser creates a WebSockets + circuit-relay node via the `/rn` entry.
133
- * Returns the created node (its `coordinatedRepo` is read by the composition).
134
- */
135
- createNode(ctx: CreateNodeContext): Promise<Libp2p>;
136
- }
137
-
138
- /**
139
- * Connect a Quereus `Database` to a Sereus strand. This is the single shared
140
- * SQL-surface composition: `connectToStrand` (Node), `connectToStrandBrowser`
141
- * (browser), and `cadre-core`'s `StrandDatabase` all flow through here, so the
142
- * hydrate-before-apply fix and any future schema wiring land in one place.
143
- *
144
- * Steps: resolve transactor + storage, register crypto + optimystic plugins,
145
- * acquire (inject or create) the libp2p node, set optimystic as the default
146
- * vtab, hydrate the catalog, then apply the sApp schema.
147
- */
148
- export async function composeStrand(
149
- db: Database,
150
- options: StrandConnectionOptions,
151
- platform: StrandPlatform,
152
- ): Promise<SereusPluginResult> {
153
- const {
154
- strandId,
155
- bootstrapNodes = [],
156
- schema,
157
- port = 0,
158
- enableCache = true,
159
- fretProfile = 'edge',
160
- transactor = 'network',
161
- } = options;
162
-
163
- // Resolve (and validate) up front so a nonsense value fails before any plugin
164
- // registration or node creation has happened.
165
- const clusterSize = resolveStrandClusterSize(options.clusterSize);
166
-
167
- // The storage engine every write and read below goes through — see
168
- // `StrandConnectionOptions.transactor`. Named separately from the option so
169
- // the default is applied exactly once and reported on the result.
170
- const resolvedTransactor: StrandTransactor = transactor;
171
-
172
- const networkName = `strand-${strandId}`;
173
- log('Connecting to strand %s (network: %s, transactor=%s)',
174
- strandId, networkName, resolvedTransactor);
175
-
176
- // Resolve storage. Node passes `options.storage` through; the browser
177
- // defaults to IndexedDB. The resolved instance feeds BOTH the local
178
- // transactor's `rawStorageFactory` and node creation, so it must be decided
179
- // up front (before pluginConfig is built). Wrapped in the write-through
180
- // raw-storage cache (cached-storage.ts) — idempotent, so a caller that
181
- // already wrapped (cadre-core's seams) passes through unchanged, with this
182
- // composition counted as one more holder of that same cache.
183
- const resolvedStorage = platform.resolveStorage
184
- ? await platform.resolveStorage({ strandId, resolvedTransactor, requestedStorage: options.storage })
185
- : options.storage;
186
- const storage = resolvedStorage ? wrapStorageWithCache(resolvedStorage, strandId) : resolvedStorage;
187
-
188
- /**
189
- * Release THIS composition's claim on the storage cache — the matching half of the
190
- * wrap above, which took one. `wrapStorageWithCache` counts holders, so this is
191
- * unconditionally correct whoever created the wrapper: the cache stays live while
192
- * cadre-core (or any other seam over the same store) still holds a claim, and is
193
- * emptied and unregistered from the shared cache pool when the last one goes. Holding
194
- * a claim past the strand's life is not free — the registration claims the backing
195
- * store in the pool, and a claim outliving its strand makes the next connect over that
196
- * same directory throw.
197
- *
198
- * Latched, so a caller that calls `shutdown()` twice releases one claim, not two — a
199
- * second release would consume another scope's claim.
200
- */
201
- let storageCacheReleased = false;
202
- const releaseStorageCache = async (): Promise<void> => {
203
- if (storageCacheReleased || !storage) {
204
- return;
205
- }
206
- storageCacheReleased = true;
207
- await disposeStorageCache(storage);
208
- };
209
-
210
- // 1-2. Register the crypto and optimystic plugins. Guarded on its own, because the
211
- // wrap above has ALREADY taken a holder claim: a registration that throws must release
212
- // it, or the claim — and the backing store's registration in the shared cache pool —
213
- // outlives the failed connect for the process lifetime, and no other scope can retire
214
- // the cache. Separate from the larger try below, which has a collection factory and
215
- // possibly a node to tear down as well; neither exists yet here.
216
- let pluginResult: OptimysticPluginResult;
217
- try {
218
- pluginResult = await registerStrandPlugins(db, platform, {
219
- resolvedTransactor,
220
- networkName,
221
- enableCache,
222
- storage,
223
- });
224
- } catch (err) {
225
- await releaseStorageCache();
226
- throw err;
227
- }
228
-
229
- const { collectionFactory } = pluginResult;
230
-
231
- let createdNode: Libp2p | null = null;
232
- let hydrated: { tables: number; indexes: number } | undefined;
233
-
234
- try {
235
- // 3. Acquire the libp2p node. Skip only when this is the unit-test fake
236
- // transactor with no injected node — every real path needs a node.
237
- if (resolvedTransactor !== 'test' || options.libp2pNode) {
238
- let node: Libp2p;
239
- let coordinatedRepo: IRepo;
240
-
241
- if (options.libp2pNode) {
242
- node = options.libp2pNode;
243
- if (!options.coordinatedRepo) {
244
- throw new Error('coordinatedRepo is required when libp2pNode is provided');
245
- }
246
- coordinatedRepo = options.coordinatedRepo;
247
- log('Using injected libp2p node');
248
- } else {
249
- // NOTE: this path takes the frozen policy whole, so it takes COHORT_READ_DEADLINE_MS
250
- // with no way to override it — `StrandConnectionOptions` has no counterpart to
251
- // cadre-core's `NetworkConfig.cohortQueryTimeoutMs` or `linkRoundTripMs`. Fine
252
- // today: every production strand comes up through cadre-core, which does thread
253
- // both, and this path is the plugin's own connect/e2e route, where the constant —
254
- // two link round trips at cadre's default declared link — is the right answer
255
- // anyway. If a plugin embedder ever needs a different deadline,
256
- // `strandClusterPolicy(clusterSize, { cohortQueryTimeoutMs: <ms> })` is the call to
257
- // reach for here.
258
- const created = await platform.createNode({ networkName, bootstrapNodes, fretProfile, port, clusterSize, clusterPolicy: STRAND_CLUSTER_POLICY, storage });
259
- createdNode = created;
260
- node = created;
261
- const repo = (created as Libp2pNodeWithRepo).coordinatedRepo;
262
- if (!repo) {
263
- throw new Error('coordinatedRepo not available on created libp2p node');
264
- }
265
- coordinatedRepo = repo;
266
- log('Created libp2p node (port: %d, fretProfile: %s, storage=%s)', port, fretProfile, !!storage);
267
- }
268
-
269
- collectionFactory.registerLibp2pNode(networkName, node, coordinatedRepo);
270
- log('Registered libp2p node with collection factory');
271
- }
272
-
273
- // 4. Set optimystic as default vtab so `declare schema` tables use it.
274
- db.setDefaultVtabName('optimystic');
275
- db.setDefaultVtabArgs({
276
- networkName,
277
- transactor: resolvedTransactor,
278
- keyNetwork: 'libp2p',
279
- });
280
- log('Set default vtab to optimystic (networkName=%s, transactor=%s)', networkName, resolvedTransactor);
281
-
282
- // 5. Hydrate Quereus's catalog from persisted optimystic vtab schemas
283
- // BEFORE applying the sApp schema. Without this, a warm restart diffs the
284
- // wrapped DDL against an empty catalog and re-emits CREATE TABLE / CREATE
285
- // INDEX for every persisted object — each round-tripping through optimystic
286
- // storage (the measured ~160s warm-start regression). No-op on first launch.
287
- const t0 = performance.now();
288
- hydrated = await pluginResult.hydrate(db);
289
- timing('[strand:%s] hydrate: %dms (tables=%d, indexes=%d)',
290
- strandId, Math.round(performance.now() - t0), hydrated.tables, hydrated.indexes);
291
- log('Hydrated catalog for strand %s (tables=%d, indexes=%d)', strandId, hydrated.tables, hydrated.indexes);
292
-
293
- // 6. Apply the strand membership/RBAC schema (`schemas/strand.qsql`,
294
- // embedded as STRAND_SCHEMA) UNCONDITIONALLY — every strand has membership
295
- // semantics whether or not an sApp schema is supplied. There are no
296
- // cross-schema references between `Strand` and `App`, so order is
297
- // irrelevant; apply `Strand` first for clarity. Because this runs AFTER
298
- // hydrate, a warm restart diffs the membership DDL against the already-
299
- // hydrated catalog and re-emits nothing (same fix as the sApp apply).
300
- //
301
- // This ticket only makes the tables present and their constraints active;
302
- // nothing here writes membership rows. Founder bootstrap (Header, founding
303
- // Manager/Member, invite/peer flows) is owned by the lifecycle ticket
304
- // `strand-membership-lifecycle-population`.
305
- log('Applying Strand membership schema for strand %s', strandId);
306
- await db.exec(`
307
- declare schema Strand {
308
- ${STRAND_SCHEMA}
309
- }
310
- apply schema Strand;
311
- `);
312
- log('Strand membership schema applied');
313
-
314
- // 7. Apply the sApp schema, if provided.
315
- //
316
- // SEAM: this and the Strand apply above are the single composition point
317
- // where strand-level declarative DDL is applied. Keep new schema wiring
318
- // here so it stays a one-location edit across Node, browser, and cadre-core.
319
- //
320
- // NOTE: neither apply is retried — a failure tears the strand down and throws,
321
- // and a later connect builds a fresh `Database` and re-hydrates from storage. A
322
- // failed apply is normally taken back whole (Quereus unwinds its migration steps
323
- // and verifies the catalog against a pre-apply fingerprint), so the next connect
324
- // diffs against the state the apply started from. The sApp schema is the one
325
- // supplied by the EMBEDDER, and the steps that DISCARD data — dropping a table,
326
- // dropping a column, narrowing a column's type — are irreversible to Quereus's
327
- // differ: once one has run, any later step that fails leaves the schema partially
328
- // migrated instead of restored. No schema in this repo evolves that way, so this
329
- // is a note for whoever first ships a migration that does.
330
- if (schema) {
331
- log('Applying sApp schema for strand %s', strandId);
332
- await applyAppSchema(db, schema);
333
- log('sApp schema applied');
334
- }
335
- } catch (err) {
336
- // Clean up resources if setup fails after partial initialization.
337
- await collectionFactory.shutdown();
338
- if (createdNode) {
339
- await createdNode.stop();
340
- }
341
- await releaseStorageCache();
342
- throw err;
343
- }
344
-
345
- // 8. Return result with hydrate counts + shutdown handler. `shutdown` tears
346
- // down the collection factory, and `collectionFactory.shutdown()` stops EVERY
347
- // node registered with it — including an injected one (registered via
348
- // `registerLibp2pNode`). So an injected node IS stopped on `shutdown`; the
349
- // `createdNode` guard below only avoids a redundant second `stop()` on a node
350
- // we created ourselves, it is not what spares an injected node. A caller that
351
- // needs an injected node to outlive `shutdown` must re-create it (cadre-core's
352
- // `StrandInstanceManager` instead re-stops it, which is idempotent).
353
- return {
354
- vtables: [],
355
- functions: [],
356
- collations: [],
357
- hydrated,
358
- transactor: resolvedTransactor,
359
- async shutdown() {
360
- log('Shutting down strand connection %s', strandId);
361
- await collectionFactory.shutdown();
362
- if (createdNode) {
363
- await createdNode.stop();
364
- }
365
- await releaseStorageCache();
366
- log('Strand connection %s shut down', strandId);
367
- },
368
- };
369
- }
370
-
371
- /**
372
- * Step 7 of {@link composeStrand}: declare the sApp's schema as `App` and apply it — a
373
- * declarative diff against the catalog, so re-applying a schema that is already in place
374
- * emits nothing. The one site that applies an `App` schema: `composeStrand` calls it at
375
- * bring-up, and cadre-core's `StrandDatabase.attachAppSchema` calls it on a live storage
376
- * replica when an app claims the strand. The database must already be composed (optimystic
377
- * set as the default vtab), or the tables land in memory instead of the strand.
378
- *
379
- * Refuses a schema holding an item the Quereus parser skipped: it keeps any item whose leading
380
- * keyword it does not model (`create unique index …`, a misspelled `tabel`, `domain`) as an
381
- * opaque placeholder that apply ignores, so the app would otherwise run without that item.
382
- *
383
- * Refuses a schema holding a `seed` item: the apply runs on every node of the strand at every
384
- * connect, so seed inserts would put a network write on each bring-up and two nodes inserting
385
- * the same key at once would collide. Rows an app needs at birth are the founding app's to write.
386
- */
387
- export async function applyAppSchema(db: Database, schema: string): Promise<void> {
388
- await db.exec(`
389
- declare schema App {
390
- ${schema}
391
- }
392
- `);
393
- assertNoIgnoredItems(db);
394
- assertNoSeedItems(db);
395
- await db.exec('apply schema App;');
396
- }
397
-
398
- function declaredAppItems(db: Database) {
399
- return db.declaredSchemaManager.getDeclaredSchema('App')?.items ?? [];
400
- }
401
-
402
- function assertNoIgnoredItems(db: Database): void {
403
- const ignored = declaredAppItems(db).filter(item => item.type === 'declareIgnored').length;
404
- if (ignored > 0) {
405
- // Only a count: Quereus leaves an ignored item's source text empty.
406
- throw new Error(`sApp schema has ${ignored} item(s) the parser does not recognize (a \`create …\` prefix or a misspelled item keyword such as \`tabel\`); items are \`table\`, \`index\`, \`unique index\`, \`view\`, \`materialized view\` and \`assertion\``);
407
- }
408
- }
409
-
410
- function assertNoSeedItems(db: Database): void {
411
- // Quereus refuses a second seed for one table at declare, so these names are distinct.
412
- const tables = declaredAppItems(db).flatMap(item => item.type === 'declaredSeed' ? [item.tableName] : []);
413
- if (tables.length > 0) {
414
- throw new Error(`sApp schema has seed item(s) for table(s) ${tables.join(', ')}; Sereus does not apply seed rows to a strand — insert the rows an app needs at birth from the app when it founds the strand`);
415
- }
416
- }
417
-
418
- /**
419
- * Steps 1-2 of {@link composeStrand}: register the crypto plugin, then the optimystic
420
- * plugin carrying this strand's transactor defaults. Split out so the composition can
421
- * guard the pair against the storage-cache claim it has already taken.
422
- */
423
- async function registerStrandPlugins(
424
- db: Database,
425
- platform: StrandPlatform,
426
- strand: {
427
- resolvedTransactor: StrandTransactor;
428
- networkName: string;
429
- enableCache: boolean;
430
- storage: IRawStorage | undefined;
431
- },
432
- ): Promise<OptimysticPluginResult> {
433
- await platform.registerCrypto(db);
434
- log('Registered crypto plugin');
435
-
436
- // On the local transactor with persistent storage, hand the same instance to the
437
- // plugin so DML persists on the host backend (not in-memory).
438
- const pluginConfig: Record<string, unknown> = {
439
- default_transactor: strand.resolvedTransactor,
440
- default_key_network: 'libp2p',
441
- default_network_name: strand.networkName,
442
- enable_cache: strand.enableCache,
443
- };
444
- if (strand.resolvedTransactor === 'local' && strand.storage) {
445
- const storage = strand.storage;
446
- pluginConfig.rawStorageFactory = () => storage;
447
- }
448
- // The plugin's published signature is `Record<string, SqlValue>` but it also reads
449
- // `rawStorageFactory` (a function reference) from the same map. Cast through unknown
450
- // rather than widen the public type.
451
- const pluginResult = optimysticPlugin(
452
- db,
453
- pluginConfig as unknown as Parameters<typeof optimysticPlugin>[1],
454
- ) as unknown as OptimysticPluginResult;
455
- applyRegistrations(db, pluginResult);
456
- log('Registered optimystic vtables and functions');
457
- return pluginResult;
458
- }
1
+ import debug from 'debug';
2
+ import type { Database, VTablePluginInfo, FunctionPluginInfo, CollationPluginInfo } from '@quereus/quereus';
3
+ import optimysticPlugin from '@optimystic/quereus-plugin-optimystic/plugin';
4
+ import type { SqlCommitDurabilityListener, CommitFullyHeldListener } from '@optimystic/quereus-plugin-optimystic/plugin';
5
+ import type { Libp2p } from '@libp2p/interface';
6
+ import type { IRepo } from '@optimystic/db-core';
7
+ import type { IRawStorage, NodeOptions } from '@optimystic/db-p2p';
8
+ import type { StrandConnectionOptions, SereusPluginResult, StrandTransactor, Libp2pNodeWithRepo } from './types.js';
9
+ import { STRAND_SCHEMA } from './strand-schema.js';
10
+ import { resolveStrandClusterSize, STRAND_CLUSTER_POLICY } from './cluster-size.js';
11
+ import { wrapStorageWithCache, disposeStorageCache } from './cached-storage.js';
12
+
13
+ const log = debug('sereus:plugin:strand');
14
+ const timing = debug('sereus:plugin:strand:timing');
15
+
16
+ /**
17
+ * Minimal interface for the CollectionFactory returned by the optimystic plugin.
18
+ * We only need the methods this composition actually uses.
19
+ */
20
+ interface CollectionFactory {
21
+ registerLibp2pNode(networkName: string, node: Libp2p, coordinatedRepo: IRepo): void;
22
+ /** Releases the read caches and network watches; leaves every registered node running. */
23
+ dispose(): Promise<void>;
24
+ /** {@link dispose}, after stopping every registered node. */
25
+ shutdown(): Promise<void>;
26
+ }
27
+
28
+ /**
29
+ * Release a composition's hold on the optimystic plugin. The transaction bridge goes first: its
30
+ * durability follower subscribes to the node's `StorageRepo`, and cadre-core keeps a strand's node
31
+ * across a database rebuild, so a bridge never disposed stays subscribed once per rebuild. Then the
32
+ * collection factory: a node the composition created is stopped with it; an injected node is left
33
+ * running, because it belongs to the caller — cadre-core keeps a strand's node up across a failed
34
+ * database build and retries the build on it.
35
+ *
36
+ * Disposes the bridge itself rather than calling the plugin's `dispose()`, which would release the
37
+ * factory a second time on the created-node path (`shutdown()` ends with the same `dispose()`).
38
+ */
39
+ async function releaseOptimysticPlugin(plugin: OptimysticPluginResult, createdNode: Libp2p | null): Promise<void> {
40
+ // NOTE: the bridge's follower re-subscribes at the start of any transaction, so a write begun on
41
+ // this `Database` after release (before its owner closes it) subscribes again and nothing
42
+ // releases it. cadre-core stops its own writers before releasing; if an app write in flight at
43
+ // close ever shows up as a leaked subscription, close the `Database` before releasing.
44
+ plugin.txnBridge.dispose();
45
+ const { collectionFactory } = plugin;
46
+ if (!createdNode) {
47
+ await collectionFactory.dispose();
48
+ return;
49
+ }
50
+ await collectionFactory.shutdown();
51
+ // Stopped again for a node created but not yet registered (a missing `coordinatedRepo`);
52
+ // a second stop of a registered one is a no-op.
53
+ await createdNode.stop();
54
+ }
55
+
56
+ /**
57
+ * Result of registering the optimystic plugin. The published plugin signature
58
+ * does not surface `collectionFactory` / `hydrate`, so this is the local shim
59
+ * over what `@optimystic/quereus-plugin-optimystic/plugin` actually returns
60
+ * (see its `plugin.ts` — `hydrate` delegates to `optimysticModule.hydrateCatalog`).
61
+ */
62
+ interface OptimysticPluginResult {
63
+ collectionFactory: CollectionFactory;
64
+ vtables: VTablePluginInfo[];
65
+ functions: FunctionPluginInfo[];
66
+ collations?: CollationPluginInfo[];
67
+ /**
68
+ * Hydrate Quereus's in-memory catalog from persisted optimystic vtab schemas.
69
+ * Must run BEFORE `apply schema App;` on a warm restart so the declarative
70
+ * diff sees the existing tables and skips re-emitting CREATE TABLE / CREATE
71
+ * INDEX for each persisted object. Idempotent; `{ tables: 0, indexes: 0 }`
72
+ * on cold start.
73
+ */
74
+ hydrate: (db: Database) => Promise<{ tables: number; indexes: number }>;
75
+ onCommitDurability: (listener: SqlCommitDurabilityListener) => () => void;
76
+ onCommitFullyHeld: (listener: CommitFullyHeldListener) => () => void;
77
+ /** Holds the durability follower's subscription to the node's `StorageRepo`; see {@link releaseOptimysticPlugin}. */
78
+ txnBridge: { dispose(): void };
79
+ [key: string]: unknown;
80
+ }
81
+
82
+ /** Registration shape shared by the crypto and optimystic plugin results. */
83
+ interface PluginRegistrations {
84
+ vtables?: VTablePluginInfo[];
85
+ functions?: FunctionPluginInfo[];
86
+ collations?: CollationPluginInfo[];
87
+ }
88
+
89
+ /**
90
+ * Inline equivalent of `@quereus/quereus`'s `registerPlugin` for plugins that
91
+ * only return functions/vtables/collations. Used directly for the optimystic
92
+ * result (so the composition can read `collectionFactory` off the same object),
93
+ * and by the browser crypto strategy to keep the browser bundle from pulling a
94
+ * duplicate `@quereus/quereus` next to the host's instance.
95
+ */
96
+ export function applyRegistrations(db: Database, result: PluginRegistrations): void {
97
+ for (const vtable of result.vtables ?? []) {
98
+ db.registerModule(vtable.name, vtable.module, vtable.auxData);
99
+ }
100
+ for (const func of result.functions ?? []) {
101
+ db.registerFunction(func.schema);
102
+ }
103
+ for (const collation of result.collations ?? []) {
104
+ db.registerCollation(collation.name, collation.func, collation.normalizer);
105
+ }
106
+ }
107
+
108
+ /** Context passed to the platform's node-creation strategy. */
109
+ export interface CreateNodeContext {
110
+ networkName: string;
111
+ bootstrapNodes: string[];
112
+ fretProfile: 'edge' | 'core';
113
+ /** libp2p listening port (Node TCP). Browser transports ignore it. */
114
+ port: number;
115
+ /**
116
+ * Resolved replication cluster size. Already defaulted — pass it to
117
+ * `createLibp2pNode` verbatim; omitting it falls back to Optimystic's own
118
+ * default of 10, which gates writes on any smaller party.
119
+ */
120
+ clusterSize: number;
121
+ /**
122
+ * The strand cluster policy, resolved here so no platform has to remember it.
123
+ * Pass it to `createLibp2pNode` verbatim.
124
+ *
125
+ * Omitting it does NOT fall back to something equivalent: Optimystic then computes
126
+ * its read-repair corroboration floor against `clusterSize` instead of against
127
+ * {@link STRAND_CLUSTER_POLICY}'s `assumedClusterSize`, which is a stricter demand
128
+ * than a two-machine strand can meet — so repair of a damaged block gives up where
129
+ * it would otherwise succeed. That is the same defect class that took the *control*
130
+ * network's e2e suite down before `42cd12c`; see `cluster-size.ts` for the reasoning
131
+ * written out in full.
132
+ */
133
+ clusterPolicy: NonNullable<NodeOptions['clusterPolicy']>;
134
+ /** Resolved persistent storage to back the node, if any. */
135
+ storage?: IRawStorage;
136
+ }
137
+
138
+ /** Context passed to the platform's storage-resolution strategy. */
139
+ export interface ResolveStorageContext {
140
+ strandId: string;
141
+ resolvedTransactor: StrandTransactor;
142
+ /** The storage the caller passed in `options.storage`, if any. */
143
+ requestedStorage?: IRawStorage;
144
+ }
145
+
146
+ /**
147
+ * Platform-specific seams the shared composition delegates to. Everything else
148
+ * (transactor resolution, plugin config, registration, default vtab, hydrate,
149
+ * schema apply, cleanup/shutdown) is identical across Node and browser and lives
150
+ * in {@link composeStrand}.
151
+ */
152
+ export interface StrandPlatform {
153
+ /**
154
+ * Register the crypto plugin against `db`. Node uses `@quereus/quereus`'s
155
+ * `registerPlugin`; the browser inlines `applyRegistrations` to avoid bundling
156
+ * a second `@quereus/quereus`.
157
+ */
158
+ registerCrypto(db: Database): void | Promise<void>;
159
+ /**
160
+ * Resolve the persistent storage to use. Optional — when omitted the caller's
161
+ * `options.storage` is used verbatim (Node). The browser supplies this to
162
+ * default to IndexedDB so a reload survives.
163
+ */
164
+ resolveStorage?(ctx: ResolveStorageContext): Promise<IRawStorage | undefined> | IRawStorage | undefined;
165
+ /**
166
+ * Create a libp2p node. Called only when no node is injected and the
167
+ * transactor needs one. Node creates a TCP node via `@optimystic/db-p2p`;
168
+ * the browser creates a WebSockets + circuit-relay node via the `/rn` entry.
169
+ * Returns the created node (its `coordinatedRepo` is read by the composition).
170
+ */
171
+ createNode(ctx: CreateNodeContext): Promise<Libp2p>;
172
+ }
173
+
174
+ /**
175
+ * Connect a Quereus `Database` to a Sereus strand. This is the single shared
176
+ * SQL-surface composition: `connectToStrand` (Node), `connectToStrandBrowser`
177
+ * (browser), and `cadre-core`'s `StrandDatabase` all flow through here, so the
178
+ * hydrate-before-apply fix and any future schema wiring land in one place.
179
+ *
180
+ * Steps: resolve transactor + storage, register crypto + optimystic plugins,
181
+ * acquire (inject or create) the libp2p node, set optimystic as the default
182
+ * vtab, hydrate the catalog, then apply the sApp schema.
183
+ */
184
+ export async function composeStrand(
185
+ db: Database,
186
+ options: StrandConnectionOptions,
187
+ platform: StrandPlatform,
188
+ ): Promise<SereusPluginResult> {
189
+ const {
190
+ strandId,
191
+ bootstrapNodes = [],
192
+ schema,
193
+ port = 0,
194
+ enableCache = true,
195
+ fretProfile = 'edge',
196
+ transactor = 'network',
197
+ } = options;
198
+
199
+ // Resolve (and validate) up front so a nonsense value fails before any plugin
200
+ // registration or node creation has happened.
201
+ const clusterSize = resolveStrandClusterSize(options.clusterSize);
202
+
203
+ // The storage engine every write and read below goes through — see
204
+ // `StrandConnectionOptions.transactor`. Named separately from the option so
205
+ // the default is applied exactly once and reported on the result.
206
+ const resolvedTransactor: StrandTransactor = transactor;
207
+
208
+ const networkName = `strand-${strandId}`;
209
+ log('Connecting to strand %s (network: %s, transactor=%s)',
210
+ strandId, networkName, resolvedTransactor);
211
+
212
+ // Resolve storage. Node passes `options.storage` through; the browser
213
+ // defaults to IndexedDB. The resolved instance feeds BOTH the local
214
+ // transactor's `rawStorageFactory` and node creation, so it must be decided
215
+ // up front (before pluginConfig is built). Wrapped in the write-through
216
+ // raw-storage cache (cached-storage.ts) — idempotent, so a caller that
217
+ // already wrapped (cadre-core's seams) passes through unchanged, with this
218
+ // composition counted as one more holder of that same cache.
219
+ const resolvedStorage = platform.resolveStorage
220
+ ? await platform.resolveStorage({ strandId, resolvedTransactor, requestedStorage: options.storage })
221
+ : options.storage;
222
+ const storage = resolvedStorage ? wrapStorageWithCache(resolvedStorage, strandId) : resolvedStorage;
223
+
224
+ /**
225
+ * Release THIS composition's claim on the storage cache — the matching half of the
226
+ * wrap above, which took one. `wrapStorageWithCache` counts holders, so this is
227
+ * unconditionally correct whoever created the wrapper: the cache stays live while
228
+ * cadre-core (or any other seam over the same store) still holds a claim, and is
229
+ * emptied and unregistered from the shared cache pool when the last one goes. Holding
230
+ * a claim past the strand's life is not free — the registration claims the backing
231
+ * store in the pool, and a claim outliving its strand makes the next connect over that
232
+ * same directory throw.
233
+ *
234
+ * Latched, so a caller that calls `shutdown()` twice releases one claim, not two — a
235
+ * second release would consume another scope's claim.
236
+ */
237
+ let storageCacheReleased = false;
238
+ const releaseStorageCache = async (): Promise<void> => {
239
+ if (storageCacheReleased || !storage) {
240
+ return;
241
+ }
242
+ storageCacheReleased = true;
243
+ await disposeStorageCache(storage);
244
+ };
245
+
246
+ // 1-2. Register the crypto and optimystic plugins. Guarded on its own, because the
247
+ // wrap above has ALREADY taken a holder claim: a registration that throws must release
248
+ // it, or the claim — and the backing store's registration in the shared cache pool —
249
+ // outlives the failed connect for the process lifetime, and no other scope can retire
250
+ // the cache. Separate from the larger try below, which has a collection factory and
251
+ // possibly a node to tear down as well; neither exists yet here.
252
+ let pluginResult: OptimysticPluginResult;
253
+ try {
254
+ pluginResult = await registerStrandPlugins(db, platform, {
255
+ resolvedTransactor,
256
+ networkName,
257
+ enableCache,
258
+ storage,
259
+ });
260
+ } catch (err) {
261
+ await releaseStorageCache();
262
+ throw err;
263
+ }
264
+
265
+ const { collectionFactory } = pluginResult;
266
+
267
+ let createdNode: Libp2p | null = null;
268
+ let hydrated: { tables: number; indexes: number } | undefined;
269
+
270
+ try {
271
+ // 3. Acquire the libp2p node. Skip only when this is the unit-test fake
272
+ // transactor with no injected node — every real path needs a node.
273
+ if (resolvedTransactor !== 'test' || options.libp2pNode) {
274
+ let node: Libp2p;
275
+ let coordinatedRepo: IRepo;
276
+
277
+ if (options.libp2pNode) {
278
+ node = options.libp2pNode;
279
+ if (!options.coordinatedRepo) {
280
+ throw new Error('coordinatedRepo is required when libp2pNode is provided');
281
+ }
282
+ coordinatedRepo = options.coordinatedRepo;
283
+ log('Using injected libp2p node');
284
+ } else {
285
+ // NOTE: this path takes the frozen policy whole, so it takes COHORT_READ_DEADLINE_MS
286
+ // with no way to override it — `StrandConnectionOptions` has no counterpart to
287
+ // cadre-core's `NetworkConfig.cohortQueryTimeoutMs` or `linkRoundTripMs`. Fine
288
+ // today: every production strand comes up through cadre-core, which does thread
289
+ // both, and this path is the plugin's own connect/e2e route, where the constant —
290
+ // two link round trips at cadre's default declared link — is the right answer
291
+ // anyway. If a plugin embedder ever needs a different deadline,
292
+ // `strandClusterPolicy(clusterSize, { cohortQueryTimeoutMs: <ms> })` is the call to
293
+ // reach for here.
294
+ const created = await platform.createNode({ networkName, bootstrapNodes, fretProfile, port, clusterSize, clusterPolicy: STRAND_CLUSTER_POLICY, storage });
295
+ createdNode = created;
296
+ node = created;
297
+ const repo = (created as Libp2pNodeWithRepo).coordinatedRepo;
298
+ if (!repo) {
299
+ throw new Error('coordinatedRepo not available on created libp2p node');
300
+ }
301
+ coordinatedRepo = repo;
302
+ log('Created libp2p node (port: %d, fretProfile: %s, storage=%s)', port, fretProfile, !!storage);
303
+ }
304
+
305
+ collectionFactory.registerLibp2pNode(networkName, node, coordinatedRepo);
306
+ log('Registered libp2p node with collection factory');
307
+ }
308
+
309
+ // 4. Set optimystic as default vtab so `declare schema` tables use it.
310
+ db.setDefaultVtabName('optimystic');
311
+ db.setDefaultVtabArgs({
312
+ networkName,
313
+ transactor: resolvedTransactor,
314
+ keyNetwork: 'libp2p',
315
+ });
316
+ log('Set default vtab to optimystic (networkName=%s, transactor=%s)', networkName, resolvedTransactor);
317
+
318
+ // 5. Hydrate Quereus's catalog from persisted optimystic vtab schemas
319
+ // BEFORE applying the sApp schema. Without this, a warm restart diffs the
320
+ // wrapped DDL against an empty catalog and re-emits CREATE TABLE / CREATE
321
+ // INDEX for every persisted object — each round-tripping through optimystic
322
+ // storage (the measured ~160s warm-start regression). No-op on first launch.
323
+ const t0 = performance.now();
324
+ hydrated = await pluginResult.hydrate(db);
325
+ timing('[strand:%s] hydrate: %dms (tables=%d, indexes=%d)',
326
+ strandId, Math.round(performance.now() - t0), hydrated.tables, hydrated.indexes);
327
+ log('Hydrated catalog for strand %s (tables=%d, indexes=%d)', strandId, hydrated.tables, hydrated.indexes);
328
+
329
+ // 6. Apply the strand membership/RBAC schema (`schemas/strand.qsql`,
330
+ // embedded as STRAND_SCHEMA) UNCONDITIONALLY — every strand has membership
331
+ // semantics whether or not an sApp schema is supplied. There are no
332
+ // cross-schema references between `Strand` and `App`, so order is
333
+ // irrelevant; apply `Strand` first for clarity. Because this runs AFTER
334
+ // hydrate, a warm restart diffs the membership DDL against the already-
335
+ // hydrated catalog and re-emits nothing (same fix as the sApp apply).
336
+ //
337
+ // This ticket only makes the tables present and their constraints active;
338
+ // nothing here writes membership rows. Founder bootstrap (Header, founding
339
+ // Manager/Member, invite/peer flows) is owned by the lifecycle ticket
340
+ // `strand-membership-lifecycle-population`.
341
+ log('Applying Strand membership schema for strand %s', strandId);
342
+ await db.exec(`
343
+ declare schema Strand {
344
+ ${STRAND_SCHEMA}
345
+ }
346
+ apply schema Strand;
347
+ `);
348
+ log('Strand membership schema applied');
349
+
350
+ // 7. Apply the sApp schema, if provided.
351
+ //
352
+ // SEAM: this and the Strand apply above are the single composition point
353
+ // where strand-level declarative DDL is applied. Keep new schema wiring
354
+ // here so it stays a one-location edit across Node, browser, and cadre-core.
355
+ //
356
+ // NOTE: neither apply is retried here — a failure releases this composition and throws,
357
+ // and a later connect builds a fresh `Database` and re-hydrates from storage. A
358
+ // failed apply is normally taken back whole (Quereus unwinds its migration steps
359
+ // and verifies the catalog against a pre-apply fingerprint), so the next connect
360
+ // diffs against the state the apply started from. The sApp schema is the one
361
+ // supplied by the EMBEDDER, and the steps that DISCARD data — dropping a table,
362
+ // dropping a column, narrowing a column's type — are irreversible to Quereus's
363
+ // differ: once one has run, any later step that fails leaves the schema partially
364
+ // migrated instead of restored. No schema in this repo evolves that way, so this
365
+ // is a note for whoever first ships a migration that does.
366
+ if (schema) {
367
+ log('Applying sApp schema for strand %s', strandId);
368
+ await applyAppSchema(db, schema);
369
+ log('sApp schema applied');
370
+ }
371
+ } catch (err) {
372
+ // Clean up resources if setup fails after partial initialization.
373
+ await releaseOptimysticPlugin(pluginResult, createdNode);
374
+ await releaseStorageCache();
375
+ throw err;
376
+ }
377
+
378
+ // 8. Return result with hydrate counts, the commit-durability listeners + shutdown handler.
379
+ // `shutdown` releases the plugin and stops the node only if this composition created it.
380
+ return {
381
+ vtables: [],
382
+ functions: [],
383
+ collations: [],
384
+ hydrated,
385
+ transactor: resolvedTransactor,
386
+ onCommitDurability: listener => pluginResult.onCommitDurability(listener),
387
+ onCommitFullyHeld: listener => pluginResult.onCommitFullyHeld(listener),
388
+ async shutdown() {
389
+ log('Shutting down strand connection %s', strandId);
390
+ await releaseOptimysticPlugin(pluginResult, createdNode);
391
+ await releaseStorageCache();
392
+ log('Strand connection %s shut down', strandId);
393
+ },
394
+ };
395
+ }
396
+
397
+ /**
398
+ * Step 7 of {@link composeStrand}: declare the sApp's schema as `App` and apply it — a
399
+ * declarative diff against the catalog, so re-applying a schema that is already in place
400
+ * emits nothing. The one site that applies an `App` schema: `composeStrand` calls it at
401
+ * bring-up, and cadre-core's `StrandDatabase.attachAppSchema` calls it on a live storage
402
+ * replica when an app claims the strand. The database must already be composed (optimystic
403
+ * set as the default vtab), or the tables land in memory instead of the strand.
404
+ *
405
+ * Refuses a schema holding an item the Quereus parser skipped: it keeps any item whose leading
406
+ * keyword it does not model (`create unique index …`, a misspelled `tabel`, `domain`) as an
407
+ * opaque placeholder that apply ignores, so the app would otherwise run without that item.
408
+ *
409
+ * Refuses a schema holding a `seed` item: the apply runs on every node of the strand at every
410
+ * connect, so seed inserts would put a network write on each bring-up and two nodes inserting
411
+ * the same key at once would collide. Rows an app needs at birth are the founding app's to write.
412
+ */
413
+ export async function applyAppSchema(db: Database, schema: string): Promise<void> {
414
+ await db.exec(`
415
+ declare schema App {
416
+ ${schema}
417
+ }
418
+ `);
419
+ assertNoIgnoredItems(db);
420
+ assertNoSeedItems(db);
421
+ await db.exec('apply schema App;');
422
+ }
423
+
424
+ function declaredAppItems(db: Database) {
425
+ return db.declaredSchemaManager.getDeclaredSchema('App')?.items ?? [];
426
+ }
427
+
428
+ function assertNoIgnoredItems(db: Database): void {
429
+ const ignored = declaredAppItems(db).filter(item => item.type === 'declareIgnored').length;
430
+ if (ignored > 0) {
431
+ // Only a count: Quereus leaves an ignored item's source text empty.
432
+ throw new Error(`sApp schema has ${ignored} item(s) the parser does not recognize (a \`create …\` prefix or a misspelled item keyword such as \`tabel\`); items are \`table\`, \`index\`, \`unique index\`, \`view\`, \`materialized view\` and \`assertion\``);
433
+ }
434
+ }
435
+
436
+ function assertNoSeedItems(db: Database): void {
437
+ // Quereus refuses a second seed for one table at declare, so these names are distinct.
438
+ const tables = declaredAppItems(db).flatMap(item => item.type === 'declaredSeed' ? [item.tableName] : []);
439
+ if (tables.length > 0) {
440
+ throw new Error(`sApp schema has seed item(s) for table(s) ${tables.join(', ')}; Sereus does not apply seed rows to a strand — insert the rows an app needs at birth from the app when it founds the strand`);
441
+ }
442
+ }
443
+
444
+ /**
445
+ * Steps 1-2 of {@link composeStrand}: register the crypto plugin, then the optimystic
446
+ * plugin carrying this strand's transactor defaults. Split out so the composition can
447
+ * guard the pair against the storage-cache claim it has already taken.
448
+ */
449
+ async function registerStrandPlugins(
450
+ db: Database,
451
+ platform: StrandPlatform,
452
+ strand: {
453
+ resolvedTransactor: StrandTransactor;
454
+ networkName: string;
455
+ enableCache: boolean;
456
+ storage: IRawStorage | undefined;
457
+ },
458
+ ): Promise<OptimysticPluginResult> {
459
+ await platform.registerCrypto(db);
460
+ log('Registered crypto plugin');
461
+
462
+ // On the local transactor with persistent storage, hand the same instance to the
463
+ // plugin so DML persists on the host backend (not in-memory).
464
+ const pluginConfig: Record<string, unknown> = {
465
+ default_transactor: strand.resolvedTransactor,
466
+ default_key_network: 'libp2p',
467
+ default_network_name: strand.networkName,
468
+ enable_cache: strand.enableCache,
469
+ };
470
+ if (strand.resolvedTransactor === 'local' && strand.storage) {
471
+ const storage = strand.storage;
472
+ pluginConfig.rawStorageFactory = () => storage;
473
+ }
474
+ // The plugin's published signature is `Record<string, SqlValue>` but it also reads
475
+ // `rawStorageFactory` (a function reference) from the same map. Cast through unknown
476
+ // rather than widen the public type.
477
+ const pluginResult = optimysticPlugin(
478
+ db,
479
+ pluginConfig as unknown as Parameters<typeof optimysticPlugin>[1],
480
+ ) as unknown as OptimysticPluginResult;
481
+ applyRegistrations(db, pluginResult);
482
+ log('Registered optimystic vtables and functions');
483
+ return pluginResult;
484
+ }