@serfab/quereus-plugin-sereus 1.16.0 → 1.18.0

package/dist/types.d.ts

@@ -108,6 +108,20 @@ export interface SereusPluginResult {
      * fires for a forgotten one. Passed through like {@link onCommitDurability}.
      */
     onCommitFullyHeld: (listener: CommitFullyHeldListener) => () => void;
+    /**
+     * Commit an empty collection for every table declared in the `Strand` and `App` schemas that
+     * has none yet (main, index and unique-enforcement trees), and resolve to how many were
+     * committed. A founder calls it before its founding write, so a read of a table nobody has
+     * written is answered from local storage rather than by the cohort proving absence.
+     *
+     * `except` names qualified tables (`'Strand.Header'`) the caller is about to write, whose
+     * collections that write creates; every tree of those tables is left out. A name matching no
+     * declared table throws.
+     *
+     * Run it only while no other writer can reach this connection's `Database`: each collection's
+     * `sync()` commits whatever is staged in it, outside any SQL transaction.
+     */
+    createEmptyCollections: (except?: readonly string[]) => Promise<number>;
     /** Shuts down the libp2p node and collection factory. Call when done. */
     shutdown: () => Promise<void>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@serfab/quereus-plugin-sereus",
-  "version": "1.16.0",
+  "version": "1.18.0",
   "type": "module",
   "description": "Quereus plugin that connects to a Sereus strand for SQL access",
   "main": "dist/index.js",
@@ -68,13 +68,13 @@
     "@libp2p/circuit-relay-v2": "^4.2.13",
     "@libp2p/interface": "^3.3.0",
     "@libp2p/websockets": "^10.1.21",
-    "@optimystic/db-core": "^1.15.0",
-    "@optimystic/db-p2p": "^1.15.0",
-    "@optimystic/db-p2p-storage-fs": "^1.15.0",
-    "@optimystic/db-p2p-storage-web": "^1.15.0",
-    "@optimystic/quereus-plugin-crypto": "^1.15.0",
-    "@optimystic/quereus-plugin-optimystic": "^1.15.0",
-    "@quereus/quereus": "^4.20.1",
+    "@optimystic/db-core": "^1.18.0",
+    "@optimystic/db-p2p": "^1.18.0",
+    "@optimystic/db-p2p-storage-fs": "^1.18.0",
+    "@optimystic/db-p2p-storage-web": "^1.18.0",
+    "@optimystic/quereus-plugin-crypto": "^1.18.0",
+    "@optimystic/quereus-plugin-optimystic": "^1.18.0",
+    "@quereus/quereus": "^4.20.3",
     "debug": "^4.4.3"
   },
   "quereus": {

package/src/compose-strand.ts

@@ -9,10 +9,14 @@ import type { StrandConnectionOptions, SereusPluginResult, StrandTransactor, Lib
 import { STRAND_SCHEMA } from './strand-schema.js';
 import { resolveStrandClusterSize, STRAND_CLUSTER_POLICY } from './cluster-size.js';
 import { wrapStorageWithCache, disposeStorageCache } from './cached-storage.js';
+import { createEmptyCollections, type CollectionRegistrySource } from './empty-collections.js';
 
 const log = debug('sereus:plugin:strand');
 const timing = debug('sereus:plugin:strand:timing');
 
+/** The schemas whose tables a strand founder gives a collection: strand membership, then the sApp's. */
+const FOUNDED_SCHEMAS = ['Strand', 'App'] as const;
+
 /**
  * Minimal interface for the CollectionFactory returned by the optimystic plugin.
  * We only need the methods this composition actually uses.
@@ -74,8 +78,12 @@ interface OptimysticPluginResult {
 	hydrate: (db: Database) => Promise<{ tables: number; indexes: number }>;
 	onCommitDurability: (listener: SqlCommitDurabilityListener) => () => void;
 	onCommitFullyHeld: (listener: CommitFullyHeldListener) => () => void;
-	/** Holds the durability follower's subscription to the node's `StorageRepo`; see {@link releaseOptimysticPlugin}. */
-	txnBridge: { dispose(): void };
+	/**
+	 * Holds the durability follower's subscription to the node's `StorageRepo` (see
+	 * {@link releaseOptimysticPlugin}) and the registry of every collection a table opened (see
+	 * {@link createEmptyCollections}).
+	 */
+	txnBridge: { dispose(): void } & CollectionRegistrySource;
 	[key: string]: unknown;
 }
 
@@ -375,7 +383,8 @@ export async function composeStrand(
 		throw err;
 	}
 
-	// 8. Return result with hydrate counts, the commit-durability listeners + shutdown handler.
+	// 8. Return result with hydrate counts, the commit-durability listeners, the founder's
+	// empty-collection step + shutdown handler.
 	// `shutdown` releases the plugin and stops the node only if this composition created it.
 	return {
 		vtables: [],
@@ -385,6 +394,7 @@ export async function composeStrand(
 		transactor: resolvedTransactor,
 		onCommitDurability: listener => pluginResult.onCommitDurability(listener),
 		onCommitFullyHeld: listener => pluginResult.onCommitFullyHeld(listener),
+		createEmptyCollections: except => createEmptyCollections(db, pluginResult.txnBridge, FOUNDED_SCHEMAS, except),
 		async shutdown() {
 			log('Shutting down strand connection %s', strandId);
 			await releaseOptimysticPlugin(pluginResult, createdNode);

package/src/empty-collections.ts

@@ -0,0 +1,139 @@
+import debug from 'debug';
+import type { Database, TableSchema } from '@quereus/quereus';
+import type { Collection, CollectionId } from '@optimystic/db-core';
+import { defaultCollectionUri } from '@optimystic/quereus-plugin-optimystic';
+
+const log = debug('sereus:plugin:empty-collections');
+
+/**
+ * The part of the Optimystic plugin's transaction bridge this module reads: the live registry of
+ * every collection a table opened (its main tree, index trees and unique-enforcement trees),
+ * filled as each table initializes.
+ */
+export interface CollectionRegistrySource {
+	getCollectionRegistry(): Map<CollectionId, Collection<unknown>>;
+}
+
+/**
+ * Commit an empty collection for every Optimystic table declared in `schemaNames` that has none
+ * yet, except the tables named in `except`, and resolve to how many collections were committed.
+ *
+ * Opening a table whose collection was never committed stages a fresh empty one in this node's
+ * tracker and writes nothing (`Collection.createOrOpen`'s invented branch), and a SQL commit
+ * flushes only the trees it wrote to. So the step opens every table, which registers each of its
+ * trees with the bridge, then syncs every registered collection still in that invented state.
+ *
+ * `except` holds qualified table names (`'Strand.Header'`) whose collections the caller is about
+ * to create by writing rows: an insert writes every tree of its table, so each of those trees is
+ * left out whether or not it was opened here. A name that matches no declared Optimystic table
+ * throws, so a renamed table cannot quietly fall back to committing everything.
+ *
+ * Commits outside any SQL transaction whatever is staged in those collections, so it is only
+ * sound while nothing else writes through `db`.
+ */
+export async function createEmptyCollections(
+	db: Database,
+	bridge: CollectionRegistrySource,
+	schemaNames: readonly string[],
+	except: readonly string[] = []
+): Promise<number> {
+	const declared = optimysticTables(db, schemaNames);
+	const excluded = excludedTables(declared, except, schemaNames);
+	const tables = declared.filter(table => !excluded.includes(table));
+	await openTables(db, tables);
+	const registry = bridge.getCollectionRegistry();
+	assertMainCollectionsRegistered(registry, tables, schemaNames);
+	return await syncInventedCollections(registry, excluded.map(mainCollectionId));
+}
+
+/**
+ * Every Optimystic table the named schemas declare. A table reaches Optimystic either through its
+ * own `using` clause (an sApp's) or through the database's default vtab (the control tables), and
+ * both record `vtabModuleName`. None found means the filter no longer matches how those tables
+ * are backed, and carrying on would give no table a collection while appearing to work.
+ */
+function optimysticTables(db: Database, schemaNames: readonly string[]): TableSchema[] {
+	const tables = schemaNames.flatMap(name => {
+		const schema = db.schemaManager.getSchema(name);
+		return schema
+			? Array.from(schema.getAllTables()).filter(table => !table.isView && table.vtabModuleName === 'optimystic')
+			: [];
+	});
+	if (tables.length === 0) {
+		throw new Error(`Schemas ${schemaNames.join(', ')} declare no optimystic table; cannot give their tables an empty collection`);
+	}
+	return tables;
+}
+
+/** The declared tables `except` names, compared case-insensitively as SQL identifiers are. */
+function excludedTables(declared: readonly TableSchema[], except: readonly string[], schemaNames: readonly string[]): TableSchema[] {
+	return except.map(name => {
+		const table = declared.find(candidate => `${candidate.schemaName}.${candidate.name}`.toLowerCase() === name.toLowerCase());
+		if (!table) {
+			throw new Error(`Cannot leave ${name} out of the empty collections: schemas ${schemaNames.join(', ')} declare no optimystic table by that name`);
+		}
+		return table;
+	});
+}
+
+/** A read initializes the table's vtab, which is what opens and registers its collections. */
+async function openTables(db: Database, tables: readonly TableSchema[]): Promise<void> {
+	for (const table of tables) {
+		for await (const _row of db.eval(`select 1 from ${table.schemaName}."${table.name}" limit 1`)) {
+			break;
+		}
+	}
+}
+
+/**
+ * A declared table whose main collection is not registered means the plugin's open or
+ * registration path has changed under this module; founding on would leave that table with no
+ * collection while appearing to have given every table one.
+ */
+function assertMainCollectionsRegistered(
+	registry: ReadonlyMap<CollectionId, unknown>,
+	tables: readonly TableSchema[],
+	schemaNames: readonly string[]
+): void {
+	const unregistered = tables.filter(table => !registry.has(mainCollectionId(table)));
+	if (unregistered.length > 0) {
+		const names = unregistered.map(table => `${table.schemaName}.${table.name} (${mainCollectionId(table)})`).join(', ');
+		throw new Error(`Opening the tables of ${schemaNames.join(', ')} registered no collection for ${names}; cannot give those tables an empty collection`);
+	}
+}
+
+/** The collection id the Optimystic vtab stores `table` under: its URI without the `tree://` scheme. */
+function mainCollectionId(table: TableSchema): CollectionId {
+	const explicitUri = table.vtabArgs?.['0'];
+	const uri = typeof explicitUri === 'string' && explicitUri !== '' ? explicitUri : defaultCollectionUri(table.schemaName, table.name);
+	return (uri.startsWith('tree://') ? uri.slice('tree://'.length) : uri) as CollectionId;
+}
+
+async function syncInventedCollections(
+	registry: ReadonlyMap<CollectionId, Collection<unknown>>,
+	excludedMainIds: readonly CollectionId[]
+): Promise<number> {
+	let committed = 0;
+	for (const [id, collection] of registry) {
+		if (belongsToExcludedTable(id, excludedMainIds)) {
+			continue;
+		}
+		// `committedRevision() === undefined` is the invented STATE, which its doc says is safe to branch on.
+		if (collection.committedRevision() === undefined && collection.hasUnsyncedChanges()) {
+			await collection.sync();
+			log('Committed empty collection %s', id);
+			committed++;
+		}
+	}
+	return committed;
+}
+
+/**
+ * Whether `id` is one of an excluded table's trees: its main collection, or an index or
+ * unique-enforcement tree, which the Optimystic plugin keeps at `<main uri>/index/<name>`.
+ * Matched by id rather than by "opened here", because the caller's own reads of an excluded
+ * table registered it before this step ran.
+ */
+function belongsToExcludedTable(id: CollectionId, excludedMainIds: readonly CollectionId[]): boolean {
+	return excludedMainIds.some(mainId => id === mainId || id.startsWith(`${mainId}/index/`));
+}

package/src/index.ts

@@ -1,5 +1,6 @@
 export { connectToStrand } from './connect.js';
 export { applyAppSchema } from './compose-strand.js';
+export { createEmptyCollections, type CollectionRegistrySource } from './empty-collections.js';
 export {
 	MIN_CLUSTER_SIZE,
 	COHORT_READ_DEADLINE_MS,

package/src/types.ts

@@ -1,121 +1,135 @@
-import type { Libp2p } from '@libp2p/interface';
-import type { IRepo } from '@optimystic/db-core';
-import type { IRawStorage } from '@optimystic/db-p2p';
-import type { SqlCommitDurabilityListener, CommitFullyHeldListener } from '@optimystic/quereus-plugin-optimystic/plugin';
-
-/**
- * Which Optimystic transactor a strand connection commits through — a choice of
- * storage engine, nothing more. See {@link StrandConnectionOptions.transactor}.
- *
- * Deliberately a closed union, and narrower than what Optimystic's
- * `collection-factory.ts` accepts: that factory also switches on `'mesh-test'`
- * (its production transactor stack over a one-node mock mesh) and falls through
- * to a host-registered custom transactor for any other name. Neither is offered
- * here — `composeStrand`'s node gate special-cases only `'test'`, so
- * `'mesh-test'` would get a real libp2p node it does not want, and nothing in
- * this repo registers a custom transactor. A closed union lets `parseConfig`
- * reject a typo (`'locl'`) instead of handing it downstream; widen it (and the
- * node gate with it) when one of those values is actually needed.
- */
-export type StrandTransactor = 'local' | 'network' | 'test';
-
-export interface StrandConnectionOptions {
-	/** UUID of the strand to connect to */
-	strandId: string;
-	/** Bootstrap multiaddrs for peer discovery */
-	bootstrapNodes?: string[];
-	/** sApp schema DDL to apply (optional - omit if schema already exists on strand) */
-	schema?: string;
-	/** sApp author public key */
-	sAppId?: string;
-	/** sApp version */
-	sAppVersion?: string;
-	/** libp2p listening port (default: 0 = random) */
-	port?: number;
-	/** Enable optimystic caching (default: true) */
-	enableCache?: boolean;
-	/** FRET profile (default: 'edge') */
-	fretProfile?: 'edge' | 'core';
-	/**
-	 * Number of nodes Optimystic is told this strand's replication cluster should
-	 * have. Ignored when {@link libp2pNode} is injected (the injected node was
-	 * built with its own value). Every peer on the strand should use the same
-	 * number — see `resolveStrandClusterSize`. Defaults to
-	 * `DEFAULT_STRAND_CLUSTER_SIZE` (4 — the smallest breadth whose 0.75
-	 * super-majority still commits with one holder offline).
-	 */
-	clusterSize?: number;
-	/** Inject an existing libp2p node instead of creating one */
-	libp2pNode?: Libp2p;
-	/** Required when libp2pNode is provided */
-	coordinatedRepo?: IRepo;
-	/**
-	 * Persistent raw storage. When provided:
-	 *  - it is passed to `createLibp2pNode` as `storage` so the libp2p data path uses it,
-	 *  - with `transactor: 'local'` it is also handed to the optimystic plugin as
-	 *    `rawStorageFactory: () => storage` so the local transactor persists DML
-	 *    on the same instance (avoids cache divergence between the two consumers).
-	 *
-	 * The plugin treats `storage` as borrowed — it is NOT closed on `shutdown()`.
-	 */
-	storage?: IRawStorage;
-	/**
-	 * Which Optimystic transactor writes and reads go through (default
-	 * `'network'`):
-	 *
-	 *  - `'network'` — transactions go through the strand's libp2p cohort. A node
-	 *    that is alone coordinates for itself, so this works with one peer or
-	 *    many; it is the only value an application should use.
-	 *  - `'local'` — transactions go straight to this process's raw storage, no
-	 *    peers consulted. For in-process tests and tooling; pair it with a
-	 *    persistent {@link storage} to survive restart.
-	 *  - `'test'` — Optimystic's in-memory fake. No libp2p node is created unless
-	 *    one is injected. Unit tests only.
-	 */
-	transactor?: StrandTransactor;
-}
-
-export interface SereusPluginResult {
-	vtables: [];
-	functions: [];
-	collations: [];
-	/**
-	 * Catalog-hydration counts from the warm-restart hydrate that runs before
-	 * `apply schema App;`. On a cold start (empty storage) both are 0; on a warm
-	 * restart they report the persisted tables/indexes the catalog was primed
-	 * with — so a test can assert hydration actually ran. `undefined` only if
-	 * composition threw before hydrate.
-	 */
-	hydrated?: { tables: number; indexes: number };
-	/**
-	 * The transactor this connection actually resolved to — the RESOLVED value,
-	 * not the requested one, so a spec whose whole point is the arm it runs on can
-	 * assert it rather than assume it. Equals `options.transactor` when one was
-	 * given, `'network'` otherwise.
-	 */
-	transactor: StrandTransactor;
-	/**
-	 * Be told, after every commit on this connection that wrote something, who holds what it
-	 * wrote. Called once per such commit, synchronously, before the commit returns. The Optimystic
-	 * plugin's listener of the same name, passed through; what the report means is in that
-	 * plugin's README § "A commit's durability". Returns an idempotent unsubscribe.
-	 */
-	onCommitDurability: (listener: SqlCommitDurabilityListener) => () => void;
-	/**
-	 * Be told once when a commit reported `followed` below full replication becomes held by
-	 * every member of its cohort. Can fire inside the commit, right after its durability report.
-	 * Followed commits live in memory: {@link shutdown} or a restart forgets them, and nothing
-	 * fires for a forgotten one. Passed through like {@link onCommitDurability}.
-	 */
-	onCommitFullyHeld: (listener: CommitFullyHeldListener) => () => void;
-	/** Shuts down the libp2p node and collection factory. Call when done. */
-	shutdown: () => Promise<void>;
-}
-
-/**
- * Extended Libp2p node type with coordinatedRepo attached by createLibp2pNode.
- * @internal
- */
-export interface Libp2pNodeWithRepo extends Libp2p {
-	coordinatedRepo: IRepo;
-}
+import type { Libp2p } from '@libp2p/interface';
+import type { IRepo } from '@optimystic/db-core';
+import type { IRawStorage } from '@optimystic/db-p2p';
+import type { SqlCommitDurabilityListener, CommitFullyHeldListener } from '@optimystic/quereus-plugin-optimystic/plugin';
+
+/**
+ * Which Optimystic transactor a strand connection commits through — a choice of
+ * storage engine, nothing more. See {@link StrandConnectionOptions.transactor}.
+ *
+ * Deliberately a closed union, and narrower than what Optimystic's
+ * `collection-factory.ts` accepts: that factory also switches on `'mesh-test'`
+ * (its production transactor stack over a one-node mock mesh) and falls through
+ * to a host-registered custom transactor for any other name. Neither is offered
+ * here — `composeStrand`'s node gate special-cases only `'test'`, so
+ * `'mesh-test'` would get a real libp2p node it does not want, and nothing in
+ * this repo registers a custom transactor. A closed union lets `parseConfig`
+ * reject a typo (`'locl'`) instead of handing it downstream; widen it (and the
+ * node gate with it) when one of those values is actually needed.
+ */
+export type StrandTransactor = 'local' | 'network' | 'test';
+
+export interface StrandConnectionOptions {
+	/** UUID of the strand to connect to */
+	strandId: string;
+	/** Bootstrap multiaddrs for peer discovery */
+	bootstrapNodes?: string[];
+	/** sApp schema DDL to apply (optional - omit if schema already exists on strand) */
+	schema?: string;
+	/** sApp author public key */
+	sAppId?: string;
+	/** sApp version */
+	sAppVersion?: string;
+	/** libp2p listening port (default: 0 = random) */
+	port?: number;
+	/** Enable optimystic caching (default: true) */
+	enableCache?: boolean;
+	/** FRET profile (default: 'edge') */
+	fretProfile?: 'edge' | 'core';
+	/**
+	 * Number of nodes Optimystic is told this strand's replication cluster should
+	 * have. Ignored when {@link libp2pNode} is injected (the injected node was
+	 * built with its own value). Every peer on the strand should use the same
+	 * number — see `resolveStrandClusterSize`. Defaults to
+	 * `DEFAULT_STRAND_CLUSTER_SIZE` (4 — the smallest breadth whose 0.75
+	 * super-majority still commits with one holder offline).
+	 */
+	clusterSize?: number;
+	/** Inject an existing libp2p node instead of creating one */
+	libp2pNode?: Libp2p;
+	/** Required when libp2pNode is provided */
+	coordinatedRepo?: IRepo;
+	/**
+	 * Persistent raw storage. When provided:
+	 *  - it is passed to `createLibp2pNode` as `storage` so the libp2p data path uses it,
+	 *  - with `transactor: 'local'` it is also handed to the optimystic plugin as
+	 *    `rawStorageFactory: () => storage` so the local transactor persists DML
+	 *    on the same instance (avoids cache divergence between the two consumers).
+	 *
+	 * The plugin treats `storage` as borrowed — it is NOT closed on `shutdown()`.
+	 */
+	storage?: IRawStorage;
+	/**
+	 * Which Optimystic transactor writes and reads go through (default
+	 * `'network'`):
+	 *
+	 *  - `'network'` — transactions go through the strand's libp2p cohort. A node
+	 *    that is alone coordinates for itself, so this works with one peer or
+	 *    many; it is the only value an application should use.
+	 *  - `'local'` — transactions go straight to this process's raw storage, no
+	 *    peers consulted. For in-process tests and tooling; pair it with a
+	 *    persistent {@link storage} to survive restart.
+	 *  - `'test'` — Optimystic's in-memory fake. No libp2p node is created unless
+	 *    one is injected. Unit tests only.
+	 */
+	transactor?: StrandTransactor;
+}
+
+export interface SereusPluginResult {
+	vtables: [];
+	functions: [];
+	collations: [];
+	/**
+	 * Catalog-hydration counts from the warm-restart hydrate that runs before
+	 * `apply schema App;`. On a cold start (empty storage) both are 0; on a warm
+	 * restart they report the persisted tables/indexes the catalog was primed
+	 * with — so a test can assert hydration actually ran. `undefined` only if
+	 * composition threw before hydrate.
+	 */
+	hydrated?: { tables: number; indexes: number };
+	/**
+	 * The transactor this connection actually resolved to — the RESOLVED value,
+	 * not the requested one, so a spec whose whole point is the arm it runs on can
+	 * assert it rather than assume it. Equals `options.transactor` when one was
+	 * given, `'network'` otherwise.
+	 */
+	transactor: StrandTransactor;
+	/**
+	 * Be told, after every commit on this connection that wrote something, who holds what it
+	 * wrote. Called once per such commit, synchronously, before the commit returns. The Optimystic
+	 * plugin's listener of the same name, passed through; what the report means is in that
+	 * plugin's README § "A commit's durability". Returns an idempotent unsubscribe.
+	 */
+	onCommitDurability: (listener: SqlCommitDurabilityListener) => () => void;
+	/**
+	 * Be told once when a commit reported `followed` below full replication becomes held by
+	 * every member of its cohort. Can fire inside the commit, right after its durability report.
+	 * Followed commits live in memory: {@link shutdown} or a restart forgets them, and nothing
+	 * fires for a forgotten one. Passed through like {@link onCommitDurability}.
+	 */
+	onCommitFullyHeld: (listener: CommitFullyHeldListener) => () => void;
+	/**
+	 * Commit an empty collection for every table declared in the `Strand` and `App` schemas that
+	 * has none yet (main, index and unique-enforcement trees), and resolve to how many were
+	 * committed. A founder calls it before its founding write, so a read of a table nobody has
+	 * written is answered from local storage rather than by the cohort proving absence.
+	 *
+	 * `except` names qualified tables (`'Strand.Header'`) the caller is about to write, whose
+	 * collections that write creates; every tree of those tables is left out. A name matching no
+	 * declared table throws.
+	 *
+	 * Run it only while no other writer can reach this connection's `Database`: each collection's
+	 * `sync()` commits whatever is staged in it, outside any SQL transaction.
+	 */
+	createEmptyCollections: (except?: readonly string[]) => Promise<number>;
+	/** Shuts down the libp2p node and collection factory. Call when done. */
+	shutdown: () => Promise<void>;
+}
+
+/**
+ * Extended Libp2p node type with coordinatedRepo attached by createLibp2pNode.
+ * @internal
+ */
+export interface Libp2pNodeWithRepo extends Libp2p {
+	coordinatedRepo: IRepo;
+}