@serfab/quereus-plugin-sereus 0.8.0

package/src/compose-strand.ts

@@ -0,0 +1,314 @@
+import debug from 'debug';
+import type { Database } from '@quereus/quereus';
+import optimysticPlugin from '@optimystic/quereus-plugin-optimystic/plugin';
+import type { Libp2p } from '@libp2p/interface';
+import type { IRepo } from '@optimystic/db-core';
+import type { IRawStorage } from '@optimystic/db-p2p';
+import type { StrandConnectionOptions, SereusPluginResult, Libp2pNodeWithRepo } from './types.js';
+import { STRAND_SCHEMA } from './strand-schema.js';
+
+const log = debug('sereus:plugin:strand');
+const timing = debug('sereus:plugin:strand:timing');
+
+/**
+ * Minimal interface for the CollectionFactory returned by the optimystic plugin.
+ * We only need the methods this composition actually uses.
+ */
+interface CollectionFactory {
+	registerLibp2pNode(networkName: string, node: Libp2p, coordinatedRepo: IRepo): void;
+	shutdown(): Promise<void>;
+}
+
+/**
+ * Result of registering the optimystic plugin. The published plugin signature
+ * does not surface `collectionFactory` / `hydrate`, so this is the local shim
+ * over what `@optimystic/quereus-plugin-optimystic/plugin` actually returns
+ * (see its `plugin.ts` — `hydrate` delegates to `optimysticModule.hydrateCatalog`).
+ */
+interface OptimysticPluginResult {
+	collectionFactory: CollectionFactory;
+	vtables: Array<{ name: string; module: unknown; auxData: unknown }>;
+	functions: Array<{ schema: unknown }>;
+	collations?: Array<{ name: string; func: unknown; normalizer?: unknown }>;
+	/**
+	 * Hydrate Quereus's in-memory catalog from persisted optimystic vtab schemas.
+	 * Must run BEFORE `apply schema App;` on a warm restart so the declarative
+	 * diff sees the existing tables and skips re-emitting CREATE TABLE / CREATE
+	 * INDEX for each persisted object. Idempotent; `{ tables: 0, indexes: 0 }`
+	 * on cold start.
+	 */
+	hydrate: (db: Database) => Promise<{ tables: number; indexes: number }>;
+	[key: string]: unknown;
+}
+
+/** Registration shape shared by the crypto and optimystic plugin results. */
+interface PluginRegistrations {
+	vtables?: Array<{ name: string; module: unknown; auxData: unknown }>;
+	functions?: Array<{ schema: unknown }>;
+	collations?: Array<{ name: string; func: unknown; normalizer?: unknown }>;
+}
+
+/**
+ * Inline equivalent of `@quereus/quereus`'s `registerPlugin` for plugins that
+ * only return functions/vtables/collations. Used directly for the optimystic
+ * result (so the composition can read `collectionFactory` off the same object),
+ * and by the browser crypto strategy to keep the browser bundle from pulling a
+ * duplicate `@quereus/quereus` next to the host's instance.
+ */
+export function applyRegistrations(db: Database, result: PluginRegistrations): void {
+	for (const vtable of result.vtables ?? []) {
+		db.registerModule(vtable.name, vtable.module as any, vtable.auxData);
+	}
+	for (const func of result.functions ?? []) {
+		db.registerFunction(func.schema as any);
+	}
+	for (const collation of result.collations ?? []) {
+		db.registerCollation(collation.name, collation.func as any, collation.normalizer as any);
+	}
+}
+
+/** Context passed to the platform's node-creation strategy. */
+export interface CreateNodeContext {
+	networkName: string;
+	bootstrapNodes: string[];
+	fretProfile: 'edge' | 'core';
+	/** libp2p listening port (Node TCP). Browser transports ignore it. */
+	port: number;
+	/** Resolved persistent storage to back the node, if any. */
+	storage?: IRawStorage;
+}
+
+/** Context passed to the platform's storage-resolution strategy. */
+export interface ResolveStorageContext {
+	strandId: string;
+	resolvedTransactor: string;
+	/** The storage the caller passed in `options.storage`, if any. */
+	requestedStorage?: IRawStorage;
+}
+
+/**
+ * Platform-specific seams the shared composition delegates to. Everything else
+ * (transactor resolution, plugin config, registration, default vtab, hydrate,
+ * schema apply, cleanup/shutdown) is identical across Node and browser and lives
+ * in {@link composeStrand}.
+ */
+export interface StrandPlatform {
+	/**
+	 * Register the crypto plugin against `db`. Node uses `@quereus/quereus`'s
+	 * `registerPlugin`; the browser inlines `applyRegistrations` to avoid bundling
+	 * a second `@quereus/quereus`.
+	 */
+	registerCrypto(db: Database): void | Promise<void>;
+	/**
+	 * Resolve the persistent storage to use. Optional — when omitted the caller's
+	 * `options.storage` is used verbatim (Node). The browser supplies this to
+	 * default to IndexedDB so a reload survives.
+	 */
+	resolveStorage?(ctx: ResolveStorageContext): Promise<IRawStorage | undefined> | IRawStorage | undefined;
+	/**
+	 * Create a libp2p node. Called only when no node is injected and the
+	 * transactor needs one. Node creates a TCP node via `@optimystic/db-p2p`;
+	 * the browser creates a WebSockets + circuit-relay node via the `/rn` entry.
+	 * Returns the created node (its `coordinatedRepo` is read by the composition).
+	 */
+	createNode(ctx: CreateNodeContext): Promise<Libp2p>;
+}
+
+/**
+ * Connect a Quereus `Database` to a Sereus strand. This is the single shared
+ * SQL-surface composition: `connectToStrand` (Node), `connectToStrandBrowser`
+ * (browser), and `cadre-core`'s `StrandDatabase` all flow through here, so the
+ * hydrate-before-apply fix and any future schema wiring land in one place.
+ *
+ * Steps: resolve transactor + storage, register crypto + optimystic plugins,
+ * acquire (inject or create) the libp2p node, set optimystic as the default
+ * vtab, hydrate the catalog, then apply the sApp schema.
+ */
+export async function composeStrand(
+	db: Database,
+	options: StrandConnectionOptions,
+	platform: StrandPlatform,
+): Promise<SereusPluginResult> {
+	const {
+		strandId,
+		bootstrapNodes = [],
+		schema,
+		port = 0,
+		enableCache = true,
+		fretProfile = 'edge',
+		mode,
+	} = options;
+
+	// Resolve the transactor. `mode` is the public knob: bootstrap -> local,
+	// networked -> network. The legacy `transactor` override (used by unit
+	// tests with `'test'`) only applies when `mode` is unspecified.
+	let resolvedTransactor: string;
+	if (mode !== undefined) {
+		resolvedTransactor = mode === 'bootstrap' ? 'local' : 'network';
+	} else if (options.transactor !== undefined) {
+		resolvedTransactor = options.transactor;
+	} else {
+		resolvedTransactor = 'network';
+	}
+
+	const networkName = `strand-${strandId}`;
+	log('Connecting to strand %s (network: %s, mode=%s, transactor=%s)',
+		strandId, networkName, mode ?? '(default)', resolvedTransactor);
+
+	// Resolve storage. Node passes `options.storage` through; the browser
+	// defaults to IndexedDB. The resolved instance feeds BOTH the plugin's
+	// bootstrap `rawStorageFactory` and node creation, so it must be decided up
+	// front (before pluginConfig is built).
+	const storage = platform.resolveStorage
+		? await platform.resolveStorage({ strandId, resolvedTransactor, requestedStorage: options.storage })
+		: options.storage;
+
+	// 1. Register crypto plugin (digest, sign, verify, etc.)
+	await platform.registerCrypto(db);
+	log('Registered crypto plugin');
+
+	// 2. Register optimystic plugin with transactor defaults. In local mode with
+	// persistent storage, hand the same instance to the plugin so the local
+	// transactor persists DML on the host backend (not in-memory).
+	const pluginConfig: Record<string, unknown> = {
+		default_transactor: resolvedTransactor,
+		default_key_network: 'libp2p',
+		default_network_name: networkName,
+		enable_cache: enableCache,
+	};
+	if (resolvedTransactor === 'local' && storage) {
+		pluginConfig.rawStorageFactory = () => storage;
+	}
+	// The plugin's published signature is `Record<string, SqlValue>` but it
+	// also reads `rawStorageFactory` (a function reference) from the same map.
+	// Cast through unknown rather than widen the public type.
+	const pluginResult = optimysticPlugin(
+		db,
+		pluginConfig as unknown as Parameters<typeof optimysticPlugin>[1],
+	) as OptimysticPluginResult;
+	applyRegistrations(db, pluginResult);
+	log('Registered optimystic vtables and functions');
+
+	const { collectionFactory } = pluginResult;
+
+	let createdNode: Libp2p | null = null;
+	let hydrated: { tables: number; indexes: number } | undefined;
+
+	try {
+		// 3. Acquire the libp2p node. Skip only when this is the unit-test fake
+		// transactor with no injected node — every real path needs a node.
+		if (resolvedTransactor !== 'test' || options.libp2pNode) {
+			let node: Libp2p;
+			let coordinatedRepo: IRepo;
+
+			if (options.libp2pNode) {
+				node = options.libp2pNode;
+				if (!options.coordinatedRepo) {
+					throw new Error('coordinatedRepo is required when libp2pNode is provided');
+				}
+				coordinatedRepo = options.coordinatedRepo;
+				log('Using injected libp2p node');
+			} else {
+				const created = await platform.createNode({ networkName, bootstrapNodes, fretProfile, port, storage });
+				createdNode = created;
+				node = created;
+				const repo = (created as Libp2pNodeWithRepo).coordinatedRepo;
+				if (!repo) {
+					throw new Error('coordinatedRepo not available on created libp2p node');
+				}
+				coordinatedRepo = repo;
+				log('Created libp2p node (port: %d, fretProfile: %s, storage=%s)', port, fretProfile, !!storage);
+			}
+
+			collectionFactory.registerLibp2pNode(networkName, node, coordinatedRepo);
+			log('Registered libp2p node with collection factory');
+		}
+
+		// 4. Set optimystic as default vtab so `declare schema` tables use it.
+		db.setDefaultVtabName('optimystic');
+		db.setDefaultVtabArgs({
+			networkName,
+			transactor: resolvedTransactor,
+			keyNetwork: 'libp2p',
+		});
+		log('Set default vtab to optimystic (networkName=%s, transactor=%s)', networkName, resolvedTransactor);
+
+		// 5. Hydrate Quereus's catalog from persisted optimystic vtab schemas
+		// BEFORE applying the sApp schema. Without this, a warm restart diffs the
+		// wrapped DDL against an empty catalog and re-emits CREATE TABLE / CREATE
+		// INDEX for every persisted object — each round-tripping through optimystic
+		// storage (the measured ~160s warm-start regression). No-op on first launch.
+		const t0 = performance.now();
+		hydrated = await pluginResult.hydrate(db);
+		timing('[strand:%s] hydrate: %dms (tables=%d, indexes=%d)',
+			strandId, Math.round(performance.now() - t0), hydrated.tables, hydrated.indexes);
+		log('Hydrated catalog for strand %s (tables=%d, indexes=%d)', strandId, hydrated.tables, hydrated.indexes);
+
+		// 6. Apply the strand membership/RBAC schema (`schemas/strand.qsql`,
+		// embedded as STRAND_SCHEMA) UNCONDITIONALLY — every strand has membership
+		// semantics whether or not an sApp schema is supplied. There are no
+		// cross-schema references between `Strand` and `App`, so order is
+		// irrelevant; apply `Strand` first for clarity. Because this runs AFTER
+		// hydrate, a warm restart diffs the membership DDL against the already-
+		// hydrated catalog and re-emits nothing (same fix as the sApp apply).
+		//
+		// This ticket only makes the tables present and their constraints active;
+		// nothing here writes membership rows. Founder bootstrap (Header, founding
+		// Authority/Member, invite/peer flows) is owned by the lifecycle ticket
+		// `strand-membership-lifecycle-population`.
+		log('Applying Strand membership schema for strand %s', strandId);
+		await db.exec(`
+			declare schema Strand {
+				${STRAND_SCHEMA}
+			}
+			apply schema Strand;
+		`);
+		log('Strand membership schema applied');
+
+		// 7. Apply the sApp schema, if provided.
+		//
+		// SEAM: this and the Strand apply above are the single composition point
+		// where strand-level declarative DDL is applied. Keep new schema wiring
+		// here so it stays a one-location edit across Node, browser, and cadre-core.
+		if (schema) {
+			log('Applying sApp schema for strand %s', strandId);
+			await db.exec(`
+				declare schema App {
+					${schema}
+				}
+				apply schema App;
+			`);
+			log('sApp schema applied');
+		}
+	} catch (err) {
+		// Clean up resources if setup fails after partial initialization.
+		await collectionFactory.shutdown();
+		if (createdNode) {
+			await createdNode.stop();
+		}
+		throw err;
+	}
+
+	// 8. Return result with hydrate counts + shutdown handler. `shutdown` tears
+	// down the collection factory, and `collectionFactory.shutdown()` stops EVERY
+	// node registered with it — including an injected one (registered via
+	// `registerLibp2pNode`). So an injected node IS stopped on `shutdown`; the
+	// `createdNode` guard below only avoids a redundant second `stop()` on a node
+	// we created ourselves, it is not what spares an injected node. A caller that
+	// needs an injected node to outlive `shutdown` must re-create it (cadre-core's
+	// `StrandInstanceManager` instead re-stops it, which is idempotent).
+	return {
+		vtables: [],
+		functions: [],
+		collations: [],
+		hydrated,
+		async shutdown() {
+			log('Shutting down strand connection %s', strandId);
+			await collectionFactory.shutdown();
+			if (createdNode) {
+				await createdNode.stop();
+			}
+			log('Strand connection %s shut down', strandId);
+		},
+	};
+}

package/src/connect-browser.ts

@@ -0,0 +1,56 @@
+import type { Database } from '@quereus/quereus';
+import cryptoPlugin from '@optimystic/quereus-plugin-crypto/plugin';
+import { createLibp2pNode } from '@optimystic/db-p2p/rn';
+import { webSockets } from '@libp2p/websockets';
+import { circuitRelayTransport } from '@libp2p/circuit-relay-v2';
+import { IndexedDBRawStorage, openOptimysticWebDb } from '@optimystic/db-p2p-storage-web';
+import type { Libp2p } from '@libp2p/interface';
+import type { StrandConnectionOptions, SereusPluginResult } from './types.js';
+import { composeStrand, applyRegistrations } from './compose-strand.js';
+
+/** The subset of the crypto plugin result we register inline in the browser. */
+interface CryptoPluginResult {
+	vtables: Array<{ name: string; module: unknown; auxData: unknown }>;
+	functions: Array<{ schema: unknown }>;
+	collations: Array<{ name: string; func: unknown; normalizer?: unknown }>;
+}
+
+/**
+ * Connect a Quereus Database to a Sereus strand from a browser/worker.
+ *
+ * Thin adapter over the shared {@link composeStrand} composition, supplying the
+ * browser platform seams:
+ *  - crypto registered inline (no runtime `@quereus/quereus` import, so the
+ *    bundle does not duplicate the host's instance),
+ *  - storage defaulting to `IndexedDBRawStorage` keyed by `sereus-strand-<id>`,
+ *  - a node created via the TCP-free `@optimystic/db-p2p/rn` entry with explicit
+ *    `webSockets()` + `circuitRelayTransport()` transports.
+ */
+export async function connectToStrandBrowser(
+	db: Database,
+	options: StrandConnectionOptions,
+): Promise<SereusPluginResult> {
+	return composeStrand(db, options, {
+		registerCrypto(database) {
+			applyRegistrations(database, cryptoPlugin(database, {}) as CryptoPluginResult);
+		},
+		async resolveStorage({ strandId, resolvedTransactor, requestedStorage }) {
+			if (requestedStorage) return requestedStorage;
+			// Browsers always default to IndexedDB so a reload survives; the
+			// unit-test fake transactor needs no storage.
+			if (resolvedTransactor === 'test') return undefined;
+			const dbHandle = await openOptimysticWebDb(`sereus-strand-${strandId}`);
+			return new IndexedDBRawStorage(dbHandle);
+		},
+		async createNode({ networkName, bootstrapNodes, fretProfile, storage }) {
+			return createLibp2pNode({
+				transports: [webSockets(), circuitRelayTransport()],
+				listenAddrs: [],
+				bootstrapNodes,
+				networkName,
+				fretProfile,
+				...(storage && { storage }),
+			}) as Promise<Libp2p>;
+		},
+	});
+}

package/src/connect.ts

@@ -0,0 +1,37 @@
+import type { Database } from '@quereus/quereus';
+import { registerPlugin } from '@quereus/quereus';
+import cryptoPlugin from '@optimystic/quereus-plugin-crypto/plugin';
+import type { Libp2p } from '@libp2p/interface';
+import type { StrandConnectionOptions, SereusPluginResult } from './types.js';
+import { composeStrand } from './compose-strand.js';
+
+/**
+ * Connect a Quereus Database to a Sereus strand (Node).
+ *
+ * Thin adapter over the shared {@link composeStrand} composition, supplying the
+ * Node platform seams: crypto via `@quereus/quereus`'s `registerPlugin`, and a
+ * TCP libp2p node created (only when not injected) via `@optimystic/db-p2p`.
+ */
+export async function connectToStrand(
+	db: Database,
+	options: StrandConnectionOptions,
+): Promise<SereusPluginResult> {
+	return composeStrand(db, options, {
+		async registerCrypto(database) {
+			await registerPlugin(database, cryptoPlugin);
+		},
+		async createNode({ networkName, bootstrapNodes, fretProfile, port, storage }) {
+			// Dynamically import to keep the module cross-platform friendly: this
+			// pulls the Node-only `@optimystic/db-p2p` (TCP) entry, and is only
+			// reached when actually creating a node (never for injected nodes).
+			const { createLibp2pNode } = await import('@optimystic/db-p2p');
+			return createLibp2pNode({
+				port,
+				bootstrapNodes,
+				networkName,
+				fretProfile,
+				...(storage && { storage }),
+			}) as Promise<Libp2p>;
+		},
+	});
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export { connectToStrand } from './connect.js';
+export type { StrandConnectionOptions, SereusPluginResult } from './types.js';

package/src/parse-config.ts

@@ -0,0 +1,55 @@
+import type { SqlValue } from '@quereus/quereus';
+import type { StrandConnectionOptions } from './types.js';
+
+/**
+ * Parse the plugin-loader SqlValue config into typed StrandConnectionOptions.
+ * Shared by the Node (`plugin.ts`) and browser (`plugin-browser.ts`) entries.
+ */
+export function parseConfig(config: Record<string, SqlValue>): StrandConnectionOptions {
+	const strandId = config.strand_id;
+	if (typeof strandId !== 'string' || !strandId) {
+		throw new Error('quereus-plugin-sereus: strand_id is required');
+	}
+
+	const bootstrapNodesRaw = config.bootstrap_nodes;
+	const bootstrapNodes = typeof bootstrapNodesRaw === 'string' && bootstrapNodesRaw
+		? bootstrapNodesRaw.split(',').map(s => s.trim()).filter(Boolean)
+		: [];
+
+	const schema = typeof config.schema === 'string' ? config.schema : undefined;
+	const sAppId = typeof config.sapp_id === 'string' ? config.sapp_id : 'unknown';
+	const sAppVersion = typeof config.sapp_version === 'string' ? config.sapp_version : '1.0.0';
+	const port = typeof config.port === 'number' ? config.port : 0;
+	const enableCache = config.enable_cache !== false && config.enable_cache !== 0;
+	const fretProfile = config.fret_profile === 'core' ? 'core' as const : 'edge' as const;
+
+	// Lifecycle mode: selects bootstrap (local transactor) vs networked. Only the
+	// two known values are honored; anything else falls through to the default.
+	const mode = config.mode === 'bootstrap' || config.mode === 'networked'
+		? config.mode
+		: undefined;
+
+	// Internal transactor override (e.g. `'test'`). Applies only when `mode` is
+	// unset — see `StrandConnectionOptions.transactor`.
+	const transactor = typeof config.transactor === 'string' && config.transactor
+		? config.transactor
+		: undefined;
+
+	// NOTE: persistent storage cannot ride a `Record<string, SqlValue>` — an
+	// `IRawStorage` is not an `SqlValue`. The Node loader (`plugin.ts`) reads the
+	// platform-only `storage_path` key directly and resolves it to a concrete
+	// `FileRawStorage`; it is intentionally NOT parsed into the typed options here.
+
+	return {
+		strandId,
+		bootstrapNodes,
+		schema,
+		sAppId,
+		sAppVersion,
+		port,
+		enableCache,
+		fretProfile,
+		...(mode && { mode }),
+		...(transactor && { transactor }),
+	};
+}

package/src/plugin-browser.ts

@@ -0,0 +1,29 @@
+/**
+ * Quereus plugin entry point for Sereus strand connections (browser/worker).
+ *
+ * This is the ESM-bundled counterpart to `./plugin`. Load via Quoomb-web's
+ * plugin URL field or `plugins[].source` in `quoomb.config.json`. Defaults
+ * storage to IndexedDB and uses WebSockets + circuit-relay transports.
+ */
+
+import type { Database, SqlValue } from '@quereus/quereus';
+import { connectToStrandBrowser } from './connect-browser.js';
+import { parseConfig } from './parse-config.js';
+import type { SereusPluginResult } from './types.js';
+
+export { parseConfig };
+
+/**
+ * Default export: Quereus plugin registration function (browser).
+ *
+ * Note: the `storage_path` setting is Node-only (it resolves to a filesystem
+ * `FileRawStorage` in `./plugin`). The browser entry ignores it and defaults
+ * storage to IndexedDB via `connectToStrandBrowser`.
+ */
+export default async function register(
+	db: Database,
+	config: Record<string, SqlValue> = {},
+): Promise<SereusPluginResult> {
+	const options = parseConfig(config);
+	return connectToStrandBrowser(db, options);
+}

package/src/plugin.ts

@@ -0,0 +1,39 @@
+/**
+ * Quereus plugin entry point for Sereus strand connections (Node).
+ *
+ * Loaded via Quoomb's `.plugin install` or config file, or via `registerPlugin()`.
+ * Parses SqlValue config and delegates to `connectToStrand`.
+ *
+ * Browser/Worker consumers should load `./plugin-browser` instead, which uses
+ * the TCP-free libp2p entry and IndexedDB-backed default storage.
+ */
+
+import type { Database, SqlValue } from '@quereus/quereus';
+import { connectToStrand } from './connect.js';
+import { parseConfig } from './parse-config.js';
+import type { SereusPluginResult } from './types.js';
+
+export { parseConfig };
+
+/**
+ * Default export: Quereus plugin registration function.
+ */
+export default async function register(
+	db: Database,
+	config: Record<string, SqlValue> = {},
+): Promise<SereusPluginResult> {
+	const options = parseConfig(config);
+
+	// `storage_path` is a Node-only loader convenience: a `Record<string, SqlValue>`
+	// cannot carry an `IRawStorage`, so the path is resolved to a concrete
+	// `FileRawStorage` here (not in the platform-agnostic `parseConfig`). The
+	// dynamic import keeps `@optimystic/db-p2p-storage-fs` (and its `node:fs`
+	// dependency) out of the browser/RN module graph.
+	const storagePath = config.storage_path;
+	if (typeof storagePath === 'string' && storagePath) {
+		const { FileRawStorage } = await import('@optimystic/db-p2p-storage-fs');
+		options.storage = new FileRawStorage(storagePath);
+	}
+
+	return connectToStrand(db, options);
+}