npm - @rocicorp/zero - Versions diffs - 1.11.0-canary.0 → 1.11.0-canary.2 - Mend

@rocicorp/zero 1.11.0-canary.0 → 1.11.0-canary.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (166) hide show

package/out/zero-cache/src/services/replicator/write-worker.js CHANGED Viewed

@@ -1,3 +1,4 @@
+import { assert } from "../../../../shared/src/asserts.js";
 import { must } from "../../../../shared/src/must.js";
 import { Database } from "../../../../zqlite/src/db.js";
 import { getSubscriptionState } from "./schema/replication-state.js";
@@ -6,7 +7,9 @@ import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics, registerSQLiteCorru
 import { createLogContext } from "../../server/logging.js";
 import { deleteLiteDB } from "../../db/delete-lite-db.js";
 import { ChangeProcessor } from "./change-processor.js";
+import { LitestreamController } from "../litestream/litestream-controller.js";
 import { applyPragmas, serializeError } from "./write-worker-client.js";
+import { LitestreamCheckpointer } from "../litestream/litestream-checkpointer.js";
 import { parentPort } from "node:worker_threads";
 //#region ../zero-cache/src/services/replicator/write-worker.ts
 if (!parentPort) throw new Error("write-worker must be run as a worker thread");
@@ -19,6 +22,8 @@ function createAPI() {
 	let lc;
 	let replicaDbPath;
 	let unregisterCorruptionDiagnosticTargets = [];
+	let checkpointerConfig;
+	let checkpointer;
 	function unregisterCorruptionDiagnostics() {
 		unregisterCorruptionDiagnosticTargets.forEach((unregister) => unregister());
 		unregisterCorruptionDiagnosticTargets = [];
@@ -39,8 +44,14 @@ function createAPI() {
 			port.postMessage({ writeError: serializeError(err) });
 		});
 	}
+	function createCheckpointer() {
+		if (checkpointerConfig) {
+			assert(lc && db && replicaDbPath, `not initialized`);
+			checkpointer = new LitestreamCheckpointer(lc, db, new LitestreamController(lc, replicaDbPath), checkpointerConfig);
+		}
+	}
 	return {
-		init(dbPath, cpMode, pragmas, logConfig) {
+		init(dbPath, cpMode, pragmas, logConfig, checkpointConfig) {
 			replicaDbPath = dbPath;
 			lc = createLogContext({ log: logConfig }, "write-worker");
 			unregisterCorruptionDiagnostics();
@@ -53,7 +64,9 @@ function createAPI() {
 				applyPragmas(db, pragmas);
 				runner = new StatementRunner(db);
 				mode = cpMode;
+				checkpointerConfig = checkpointConfig ?? void 0;
 				createProcessor();
+				createCheckpointer();
 			} catch (e) {
 				handleCorruptedDb(e);
 				throw e;
@@ -67,9 +80,17 @@ function createAPI() {
 				throw e;
 			}
 		},
-		processMessage(downstream) {
+		async processMessages(downstream) {
 			try {
-				return must(processor).processMessage(must(lc), downstream);
+				let commitResult = null;
+				for (const message of downstream) {
+					const committed = must(processor).processMessage(must(lc), message);
+					if (committed) {
+						commitResult = committed;
+						if (checkpointer) await checkpointer.maybeCheckpoint();
+					}
+				}
+				return commitResult;
 			} catch (e) {
 				handleCorruptedDb(e);
 				throw e;
@@ -77,9 +98,13 @@ function createAPI() {
 		},
 		abort() {
 			must(processor).abort(must(lc));
+			checkpointer?.close();
 			createProcessor();
+			createCheckpointer();
 		},
 		stop() {
+			checkpointer?.close();
+			checkpointer = void 0;
 			db?.close();
 			db = void 0;
 			runner = void 0;
@@ -90,9 +115,9 @@ function createAPI() {
 	};
 }
 var api = createAPI();
-port.on("message", (msg) => {
+port.on("message", async (msg) => {
 	try {
-		const result = api[msg.method](...msg.args);
+		const result = await api[msg.method](...msg.args);
 		if (msg.method !== "abort") port.postMessage({
 			method: msg.method,
 			result

package/out/zero-cache/src/services/replicator/write-worker.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n registerSQLiteCorruptionDiagnosticTarget,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n applyPragmas,\n serializeError,\n type ArgsMap,\n type Method,\n type PragmaConfig,\n type Request,\n type Response,\n type ResultMap,\n type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n let db: Database \| undefined;\n let runner: StatementRunner \| undefined;\n let processor: ChangeProcessor \| undefined;\n let mode: ChangeProcessorMode \| undefined;\n let lc: LogContext \| undefined;\n let replicaDbPath: string \| undefined;\n let unregisterCorruptionDiagnosticTargets: (() => void)[] = [];\n\n function unregisterCorruptionDiagnostics() {\n unregisterCorruptionDiagnosticTargets.forEach(unregister => unregister());\n unregisterCorruptionDiagnosticTargets = [];\n }\n\n function handleCorruptedDb(err: unknown) {\n if (!lc \|\| !replicaDbPath \|\| !isSQLiteCorruption(err)) {\n return;\n }\n logSQLiteCorruptionDiagnostics(lc, 'write-worker', replicaDbPath, err);\n try {\n lc.warn?.(`deleting corrupted db at ${replicaDbPath}`);\n deleteLiteDB(replicaDbPath);\n } catch (e) {\n lc.warn?.(`error deleting corrupted db at ${replicaDbPath}`, e);\n }\n }\n\n function createProcessor() {\n processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n handleCorruptedDb(err);\n port.postMessage({\n writeError: serializeError(err),\n } satisfies WriteError);\n });\n }\n\n return {\n init(\n dbPath: string,\n cpMode: ChangeProcessorMode,\n pragmas: PragmaConfig,\n logConfig: LogConfig,\n ): void {\n replicaDbPath = dbPath;\n lc = createLogContext({log: logConfig}, 'write-worker');\n unregisterCorruptionDiagnostics();\n unregisterCorruptionDiagnosticTargets.push(\n registerSQLiteCorruptionDiagnosticTarget({\n debugName: 'write-worker',\n dbPath,\n }),\n );\n try {\n db = new Database(lc, dbPath);\n applyPragmas(db, pragmas);\n runner = new StatementRunner(db);\n mode = cpMode;\n createProcessor();\n } catch (e) {\n handleCorruptedDb(e);\n throw e;\n }\n },\n\n getSubscriptionState() {\n try {\n return getSubscriptionState(must(runner));\n } catch (e) {\n handleCorruptedDb(e);\n throw e;\n }\n },\n\n ~~processMessage~~(downstream: ChangeStreamData) {\n try {\n ~~return~~ must(processor).processMessage(must(lc), ~~downstream~~);\n } catch (e) {\n handleCorruptedDb(e);\n throw e;\n }\n },\n\n abort() {\n must(processor).abort(must(lc));\n createProcessor();\n },\n\n stop() {\n db?.close();\n db = undefined;\n runner = undefined;\n processor = undefined;\n replicaDbPath = undefined;\n unregisterCorruptionDiagnostics();\n },\n };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n try {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n const result = (api[msg.method] as (...args: any[]) => unknown)(\n ...msg.args,\n );\n // abort is fire-and-forget — no pending slot on the client side.\n if (msg.method !== 'abort') {\n port.postMessage({method: msg.method, result} as Response);\n }\n } catch (e) {\n if (msg.method !== 'abort') {\n port.postMessage({\n method: msg.method,\n error: serializeError(e),\n } as Response);\n }\n }\n});\n"],"mappings":"~~;;;;;;;;;;;AA4BA~~,IAAI,CAAC,YACH,MAAM,IAAI,MAAM,6CAA6C;AAG/D,IAAM,OAAO;~~AAIb~~,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI,wCAAwD,CAAC;~~CAE7D~~,SAAS,kCAAkC;EACzC,sCAAsC,SAAQ,eAAc,WAAW,CAAC;EACxE,wCAAwC,CAAC;CAC3C;CAEA,SAAS,kBAAkB,KAAc;EACvC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,mBAAmB,GAAG,GAClD;EAEF,+BAA+B,IAAI,gBAAgB,eAAe,GAAG;EACrE,IAAI;GACF,GAAG,OAAO,4BAA4B,eAAe;GACrD,aAAa,aAAa;EAC5B,SAAS,GAAG;GACV,GAAG,OAAO,kCAAkC,iBAAiB,CAAC;EAChE;CACF;CAEA,SAAS,kBAAkB;EACzB,YAAY,IAAI,gBAAgB,KAAK,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,QAAQ;GACtE,kBAAkB,GAAG;GACrB,KAAK,YAAY,EACf,YAAY,eAAe,GAAG,EAChC,CAAsB;EACxB,CAAC;CACH;CAEA,OAAO;EACL,KACE,QACA,QACA,SACA,~~WACM~~;GACN,gBAAgB;GAChB,KAAK,iBAAiB,EAAC,KAAK,UAAS,GAAG,cAAc;GACtD,gCAAgC;GAChC,sCAAsC,KACpC,yCAAyC;IACvC,WAAW;IACX;GACF,CAAC,CACH;GACA,IAAI;IACF,KAAK,IAAI,SAAS,IAAI,MAAM;IAC5B,aAAa,IAAI,OAAO;IACxB,SAAS,IAAI,gBAAgB,EAAE;IAC/B,OAAO;IACP,gBAAgB;~~GAClB~~,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,uBAAuB;GACrB,IAAI;IACF,OAAO,qBAAqB,KAAK,MAAM,CAAC;GAC1C,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,~~eAAe~~,~~YAA8B~~;~~GAC3C~~,IAAI;IACF,~~OAAO~~,KAAK,SAAS,EAAE,eAAe,KAAK,EAAE,GAAG,~~UAAU~~;~~GAC5D~~,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,QAAQ;GACN,KAAK,SAAS,EAAE,MAAM,KAAK,EAAE,CAAC;GAC9B,gBAAgB;~~EAClB~~;EAEA,OAAO;GACL,IAAI,MAAM;GACV,KAAK,KAAA;GACL,SAAS,KAAA;GACT,YAAY,KAAA;GACZ,gBAAgB,KAAA;GAChB,gCAAgC;EAClC;CACF;AACF;AAEA,IAAM,MAAM,UAAU;AAEtB,KAAK,GAAG,~~YAAY~~,QAAiB;~~CACnC~~,IAAI;EAEF,MAAM,~~SAAU~~,IAAI,IAAI,~~QACtB~~,GAAG,IAAI,IACT;EAEA,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;EAAM,CAAa;CAE7D,SAAS,GAAG;EACV,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GACf,QAAQ,IAAI;GACZ,OAAO,eAAe,CAAC;EACzB,CAAa;CAEjB;AACF,CAAC"}
1	+ {"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import {parentPort} from 'node:worker_threads';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n registerSQLiteCorruptionDiagnosticTarget,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {LitestreamCheckpointer} from '../litestream/litestream-checkpointer.ts';\nimport {LitestreamController} from '../litestream/litestream-controller.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n applyPragmas,\n serializeError,\n type ArgsMap,\n type ForceCheckpointConfig,\n type Method,\n type PragmaConfig,\n type Request,\n type Response,\n type ResultMap,\n type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {\n [M in Method]: (...args: ArgsMap[M]) => ResultMap[M] \| Promise<ResultMap[M]>;\n};\n\nfunction createAPI(): API {\n let db: Database \| undefined;\n let runner: StatementRunner \| undefined;\n let processor: ChangeProcessor \| undefined;\n let mode: ChangeProcessorMode \| undefined;\n let lc: LogContext \| undefined;\n let replicaDbPath: string \| undefined;\n let unregisterCorruptionDiagnosticTargets: (() => void)[] = [];\n\n // Set when write-path checkpoint backpressure is enabled (backup replicator\n // on litestream v5). `undefined` disables the feature.\n let checkpointerConfig: ForceCheckpointConfig \| undefined;\n let checkpointer: LitestreamCheckpointer \| undefined;\n\n function unregisterCorruptionDiagnostics() {\n unregisterCorruptionDiagnosticTargets.forEach(unregister => unregister());\n unregisterCorruptionDiagnosticTargets = [];\n }\n\n function handleCorruptedDb(err: unknown) {\n if (!lc \|\| !replicaDbPath \|\| !isSQLiteCorruption(err)) {\n return;\n }\n logSQLiteCorruptionDiagnostics(lc, 'write-worker', replicaDbPath, err);\n try {\n lc.warn?.(`deleting corrupted db at ${replicaDbPath}`);\n deleteLiteDB(replicaDbPath);\n } catch (e) {\n lc.warn?.(`error deleting corrupted db at ${replicaDbPath}`, e);\n }\n }\n\n function createProcessor() {\n processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n handleCorruptedDb(err);\n port.postMessage({\n writeError: serializeError(err),\n } satisfies WriteError);\n });\n }\n\n function createCheckpointer() {\n if (checkpointerConfig) {\n assert(lc && db && replicaDbPath, `not initialized`);\n checkpointer = new LitestreamCheckpointer(\n lc,\n db,\n new LitestreamController(lc, replicaDbPath),\n checkpointerConfig,\n );\n }\n }\n\n return {\n init(\n dbPath: string,\n cpMode: ChangeProcessorMode,\n pragmas: PragmaConfig,\n logConfig: LogConfig,\n checkpointConfig: ForceCheckpointConfig \| null,\n ): void {\n replicaDbPath = dbPath;\n lc = createLogContext({log: logConfig}, 'write-worker');\n unregisterCorruptionDiagnostics();\n unregisterCorruptionDiagnosticTargets.push(\n registerSQLiteCorruptionDiagnosticTarget({\n debugName: 'write-worker',\n dbPath,\n }),\n );\n try {\n db = new Database(lc, dbPath);\n applyPragmas(db, pragmas);\n runner = new StatementRunner(db);\n mode = cpMode;\n checkpointerConfig = checkpointConfig ?? undefined;\n createProcessor();\n createCheckpointer();\n } catch (e) {\n handleCorruptedDb(e);\n throw e;\n }\n },\n\n getSubscriptionState() {\n try {\n return getSubscriptionState(must(runner));\n } catch (e) {\n handleCorruptedDb(e);\n throw e;\n }\n },\n\n async processMessages(downstream: readonly ChangeStreamData[]) {\n try {\n let commitResult = null;\n for (const message of downstream) {\n const committed = must(processor).processMessage(must(lc), message);\n if (committed) {\n commitResult = committed;\n if (checkpointer) {\n await checkpointer.maybeCheckpoint();\n }\n }\n }\n return commitResult;\n } catch (e) {\n handleCorruptedDb(e);\n throw e;\n }\n },\n\n abort() {\n must(processor).abort(must(lc));\n checkpointer?.close();\n createProcessor();\n createCheckpointer();\n },\n\n stop() {\n checkpointer?.close();\n checkpointer = undefined;\n db?.close();\n db = undefined;\n runner = undefined;\n processor = undefined;\n replicaDbPath = undefined;\n unregisterCorruptionDiagnostics();\n },\n };\n}\n\nconst api = createAPI();\n\nport.on('message', async (msg: Request) => {\n try {\n // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n const result = await (api[msg.method] as (...args: any[]) => unknown)(\n ...msg.args,\n );\n // abort is fire-and-forget — no pending slot on the client side.\n if (msg.method !== 'abort') {\n port.postMessage({method: msg.method, result} as Response);\n }\n } catch (e) {\n if (msg.method !== 'abort') {\n port.postMessage({\n method: msg.method,\n error: serializeError(e),\n } as Response);\n }\n }\n});\n"],"mappings":";;;;;;;;;;;;;;AAgCA,IAAI,CAAC,YACH,MAAM,IAAI,MAAM,6CAA6C;AAG/D,IAAM,OAAO;AAMb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI,wCAAwD,CAAC;CAI7D,IAAI;CACJ,IAAI;CAEJ,SAAS,kCAAkC;EACzC,sCAAsC,SAAQ,eAAc,WAAW,CAAC;EACxE,wCAAwC,CAAC;CAC3C;CAEA,SAAS,kBAAkB,KAAc;EACvC,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,mBAAmB,GAAG,GAClD;EAEF,+BAA+B,IAAI,gBAAgB,eAAe,GAAG;EACrE,IAAI;GACF,GAAG,OAAO,4BAA4B,eAAe;GACrD,aAAa,aAAa;EAC5B,SAAS,GAAG;GACV,GAAG,OAAO,kCAAkC,iBAAiB,CAAC;EAChE;CACF;CAEA,SAAS,kBAAkB;EACzB,YAAY,IAAI,gBAAgB,KAAK,MAAM,GAAG,KAAK,IAAI,IAAI,KAAK,QAAQ;GACtE,kBAAkB,GAAG;GACrB,KAAK,YAAY,EACf,YAAY,eAAe,GAAG,EAChC,CAAsB;EACxB,CAAC;CACH;CAEA,SAAS,qBAAqB;EAC5B,IAAI,oBAAoB;GACtB,OAAO,MAAM,MAAM,eAAe,iBAAiB;GACnD,eAAe,IAAI,uBACjB,IACA,IACA,IAAI,qBAAqB,IAAI,aAAa,GAC1C,kBACF;EACF;CACF;CAEA,OAAO;EACL,KACE,QACA,QACA,SACA,WACA,kBACM;GACN,gBAAgB;GAChB,KAAK,iBAAiB,EAAC,KAAK,UAAS,GAAG,cAAc;GACtD,gCAAgC;GAChC,sCAAsC,KACpC,yCAAyC;IACvC,WAAW;IACX;GACF,CAAC,CACH;GACA,IAAI;IACF,KAAK,IAAI,SAAS,IAAI,MAAM;IAC5B,aAAa,IAAI,OAAO;IACxB,SAAS,IAAI,gBAAgB,EAAE;IAC/B,OAAO;IACP,qBAAqB,oBAAoB,KAAA;IACzC,gBAAgB;IAChB,mBAAmB;GACrB,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,uBAAuB;GACrB,IAAI;IACF,OAAO,qBAAqB,KAAK,MAAM,CAAC;GAC1C,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,MAAM,gBAAgB,YAAyC;GAC7D,IAAI;IACF,IAAI,eAAe;IACnB,KAAK,MAAM,WAAW,YAAY;KAChC,MAAM,YAAY,KAAK,SAAS,EAAE,eAAe,KAAK,EAAE,GAAG,OAAO;KAClE,IAAI,WAAW;MACb,eAAe;MACf,IAAI,cACF,MAAM,aAAa,gBAAgB;KAEvC;IACF;IACA,OAAO;GACT,SAAS,GAAG;IACV,kBAAkB,CAAC;IACnB,MAAM;GACR;EACF;EAEA,QAAQ;GACN,KAAK,SAAS,EAAE,MAAM,KAAK,EAAE,CAAC;GAC9B,cAAc,MAAM;GACpB,gBAAgB;GAChB,mBAAmB;EACrB;EAEA,OAAO;GACL,cAAc,MAAM;GACpB,eAAe,KAAA;GACf,IAAI,MAAM;GACV,KAAK,KAAA;GACL,SAAS,KAAA;GACT,YAAY,KAAA;GACZ,gBAAgB,KAAA;GAChB,gCAAgC;EAClC;CACF;AACF;AAEA,IAAM,MAAM,UAAU;AAEtB,KAAK,GAAG,WAAW,OAAO,QAAiB;CACzC,IAAI;EAEF,MAAM,SAAS,MAAO,IAAI,IAAI,QAC5B,GAAG,IAAI,IACT;EAEA,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;EAAM,CAAa;CAE7D,SAAS,GAAG;EACV,IAAI,IAAI,WAAW,SACjB,KAAK,YAAY;GACf,QAAQ,IAAI;GACZ,OAAO,eAAe,CAAC;EACzB,CAAa;CAEjB;AACF,CAAC"}

package/out/zero-cache/src/types/streams.d.ts CHANGED Viewed

@@ -53,10 +53,10 @@ type PipeOptions<T> = {
     bufferMessages?: number;
 };
 export declare function pipe<T>({ source, sink, parse, bufferMessages }: PipeOptions<T>): void;
-/** A parsed JSON value paired with the exact source text it was parsed from. */
-export type ParsedJSON<T> = {
+/** A parsed value paired with its approximate serialized transport size. */
+export type Sized<T> = {
     data: T;
-    json: string;
+    size: number;
 };
 export declare function streamOut<T extends JSONValue>(lc: LogContext, source: Source<T>, sink: WebSocket): Promise<void>;
 /**
@@ -65,9 +65,9 @@ export declare function streamOut<T extends JSONValue>(lc: LogContext, source: S
 export declare function streamOutStringified(lc: LogContext, source: Source<string>, sink: WebSocket): Promise<void>;
 export declare function streamIn<T extends JSONValue>(lc: LogContext, source: WebSocket, schema: v.Type<T>): Promise<Source<T>>;
 /**
- * Streams in messages sent by {@link streamOutStringified}, preserving the
- * exact application-level JSON alongside its parsed and validated value.
+ * Streams in parsed messages while retaining only the transport-frame size.
+ * The size bounds downstream batching without keeping or copying the JSON.
  */
-export declare function streamInStringified<T extends JSONValue>(lc: LogContext, source: WebSocket, schema: v.Type<T>): Promise<Source<ParsedJSON<T>>>;
+export declare function streamInWithSize<T extends JSONValue>(lc: LogContext, source: WebSocket, schema: v.Type<T>): Promise<Source<Sized<T>>>;
 export {};
 //# sourceMappingURL=streams.d.ts.map

package/out/zero-cache/src/types/streams.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAE9B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,IAAI,CAAC,CAAC,IAAI;IACpB,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;CACxB,CAAC;AAEF;;;GAGG;AAIH,wBAAgB,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,SAAS,EAChE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EACpB,UAAU,GAAE,OAAO,CAAC,GAAG,CAAM,EAC7B,SAAS,GAAE,OAAO,CAAC,EAAE,CAAM,EAC3B,aAAa,GAAE,aAAkB,GAChC;IAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;CAAC,CAwE9C;AAED,KAAK,WAAW,CAAC,CAAC,IAAI;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAsD5E;AAUD,~~gFAAgF~~;~~AAChF~~,MAAM,MAAM,~~UAAU~~,CAAC,CAAC,IAAI;~~IAC1B~~,IAAI,EAAE,CAAC,CAAC;IACR,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,wBAAgB,SAAS,CAAC,CAAC,SAAS,SAAS,EAC3C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EACjB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EACtB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAmED,wBAAgB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAC1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAEpB;AAED;;;GAGG;AACH,wBAAgB,~~mBAAmB~~,CAAC,CAAC,SAAS,SAAS,~~EACrD~~,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,~~UAAU~~,CAAC,CAAC,CAAC,CAAC,CAAC,~~CAKhC~~"}
1	+ {"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/types/streams.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,QAAQ,EAGR,KAAK,aAAa,EACnB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAKL,KAAK,SAAS,EACf,MAAM,IAAI,CAAC;AAEZ,OAAO,EAAa,KAAK,SAAS,EAAC,MAAM,oCAAoC,CAAC;AAE9E,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,EAAC,YAAY,EAAE,KAAK,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAW7D,MAAM,MAAM,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IACzC;;;;;;OAMG;IACH,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;IAE9B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAE7B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC;QAAC,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;KAAC,CAAC,GAAG,SAAS,CAAC;CACxE,CAAC;AAEF,MAAM,MAAM,IAAI,CAAC,CAAC,IAAI;IACpB,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC;CACxB,CAAC;AAEF;;;GAGG;AAIH,wBAAgB,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,SAAS,EAChE,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,SAAS,EACb,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EACpB,UAAU,GAAE,OAAO,CAAC,GAAG,CAAM,EAC7B,SAAS,GAAE,OAAO,CAAC,EAAE,CAAM,EAC3B,aAAa,GAAE,aAAkB,GAChC;IAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;CAAC,CAwE9C;AAED,KAAK,WAAW,CAAC,CAAC,IAAI;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACtB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,IAAI,CAAC,CAAC,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,cAAc,EAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAsD5E;AAUD,4EAA4E;AAC5E,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI;IACrB,IAAI,EAAE,CAAC,CAAC;IACR,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,wBAAgB,SAAS,CAAC,CAAC,SAAS,SAAS,EAC3C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EACjB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EACtB,IAAI,EAAE,SAAS,GACd,OAAO,CAAC,IAAI,CAAC,CAEf;AAmED,wBAAgB,QAAQ,CAAC,CAAC,SAAS,SAAS,EAC1C,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAEpB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,SAAS,SAAS,EAClD,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAK3B"}

package/out/zero-cache/src/types/streams.js CHANGED Viewed

@@ -158,13 +158,13 @@ function streamIn(lc, source, schema) {
 	return streamInInternal(lc, source, schema, (data) => data);
 }
 /**
-* Streams in messages sent by {@link streamOutStringified}, preserving the
-* exact application-level JSON alongside its parsed and validated value.
+* Streams in parsed messages while retaining only the transport-frame size.
+* The size bounds downstream batching without keeping or copying the JSON.
 */
-function streamInStringified(lc, source, schema) {
-	return streamInInternal(lc, source, schema, (data, frame, id) => ({
+function streamInWithSize(lc, source, schema) {
+	return streamInInternal(lc, source, schema, (data, frame) => ({
 		data,
-		json: extractStringifiedMessage(frame, id)
+		size: frame.length
 	}));
 }
 async function streamInInternal(lc, source, schema, transform) {
@@ -197,11 +197,6 @@ async function streamInInternal(lc, source, schema, transform) {
 	await closer.connected;
 	return sink;
 }
-function extractStringifiedMessage(frame, id) {
-	const prefix = `{"id":${id},"msg":`;
-	assert(frame.startsWith(prefix) && frame.endsWith("}"), `invalid streamed message frame`);
-	return frame.slice(prefix.length, -1);
-}
 var WebSocketCloser = class WebSocketCloser {
 	#lc;
 	#ws;
@@ -267,6 +262,6 @@ var WebSocketCloser = class WebSocketCloser {
 	}
 };
 //#endregion
-export { pipe, stream, streamIn, streamInStringified, streamOut, streamOutStringified };
+export { pipe, stream, streamIn, streamInWithSize, streamOut, streamOutStringified };
 //# sourceMappingURL=streams.js.map

package/out/zero-cache/src/types/streams.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"streams.js","names":["#lc","#ws","#closeStream","#messageHandler","#connected","#handleOpen","#handleClose","#handleError","#conn"],"sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import {\n pipeline,\n Readable,\n Transform,\n Writable,\n type DuplexOptions,\n} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n createWebSocketStream,\n type CloseEvent,\n type ErrorEvent,\n type MessageEvent,\n type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n closeWithError,\n expectPingsForLiveness,\n sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n /*\n Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n * from yielding any values.\n \n @param err Terminate the iteration by throwing the `err` instead.\n /\n cancel: (err?: Error) => void;\n\n /\n An AbortSignal that can be used to listen for termination or\n * race a short-lived promise against the termination of the iterable\n * (via `promiseOrAbort()`)\n /\n readonly signal: AbortSignal;\n\n /\n The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n * to be overridden.\n \n This is suitable for transport layers that serialize messages across processes, such\n * as the {@link streamOut()} method; pipelining allows the transport to send messages\n * as they arrive without waiting for the previous message to be acked, streaming\n * them to the receiving process where they are presumably queued and processed without\n * a per-message ack delay. The receiving end of the transport then responds with acks\n * asynchronously as the receiving end processes the messages.\n /\n pipeline?: AsyncIterable<{value: T; consumed: () => void}> \| undefined;\n};\n\nexport type Sink<T> = {\n push(message: T): void;\n};\n\n/\n Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n /\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n// under the covers so that internal communication is also\n// responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n lc: LogContext,\n ws: WebSocket,\n inSchema: v.Type<In>,\n outOptions: Options<Out> = {},\n inOptions: Options<In> = {},\n streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n const endpoint = ws.url ?? 'client';\n function close(err?: unknown) {\n if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n if (err) {\n closeWithError(lc, ws, err);\n } else {\n lc.info?.(`closing connection to ${endpoint}`);\n ws.close();\n }\n }\n }\n\n const instream = Subscription.create<In>({\n ...inOptions,\n cleanup: (unconsumed, err) => {\n inOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n const outstream = Subscription.create<Out>({\n ...outOptions,\n cleanup: (unconsumed, err) => {\n outOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n\n const duplex = createWebSocketStream(ws, {\n ...streamOptions,\n decodeStrings: false,\n });\n\n // Outgoing transform.\n function streamOut() {\n // Mainly used for verifying that back-pressure kicks in tests.\n duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n pipeline(\n Readable.from(outstream),\n new Transform({\n objectMode: true,\n transform: (msg, _encoding, callback) =>\n callback(null, BigIntJSON.stringify(msg)),\n }),\n duplex,\n err => (err ? outstream.fail(err) : outstream.cancel()),\n );\n }\n\n if (ws.readyState === ws.CONNECTING) {\n ws.on('open', () => {\n lc.info?.(`connected to ${endpoint}`);\n streamOut();\n });\n } else {\n streamOut();\n }\n\n // Incoming transform.\n pipe({\n source: duplex,\n sink: instream,\n parse: chunk => {\n const json = BigIntJSON.parse(chunk.toString());\n return v.parse(json, inSchema, 'passthrough');\n },\n });\n\n sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n source: Readable;\n sink: Subscription<T>;\n parse: (buffer: Buffer) => T \| null;\n bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n bufferMessages ??= 0;\n assert(bufferMessages >= 0, 'bufferMessages must be non-negative');\n const pending: Promise<unknown>[] = [];\n\n pipeline(\n source,\n new Writable({\n decodeStrings: false,\n write: (chunk, _encoding, callback) => {\n let msg: T \| null;\n try {\n if ((msg = parse(chunk)) === null) {\n callback();\n return;\n }\n } catch (err) {\n callback(ensureError(err));\n return;\n }\n // Inbound backpressure is exerted by unconsumed messages in the\n // subscription. A buffer can be used to allow messages to queue up in\n // in the Subscription object, which allows the consumer to \"peek\" at\n // whether there are more messages immediately available\n // (via {@link Subscription.queued}.\n const {result} = sink.push(msg);\n pending.push(result);\n void result.then(() => pending.shift());\n\n if (pending.length <= bufferMessages) {\n // immediately allow more messages\n callback();\n } else {\n // wait for the oldest result in the pending queue\n pending[0].then(\n () => callback(),\n err => callback(ensureError(err)),\n );\n }\n },\n destroy: (err, callback) => {\n if (err) {\n sink.fail(ensureError(err));\n }\n // Otherwise, final will handle the cancel.\n callback();\n },\n final: callback => {\n sink.cancel();\n callback();\n },\n }),\n err => (err ? sink.fail(err) : sink.cancel()),\n );\n}\n\nfunction ensureError(err: unknown) {\n return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\n/* A parsed JSON value paired with the exact source text it was parsed from. /\nexport type ParsedJSON<T> = {\n data: T;\n json: string;\n};\n\nexport function streamOut<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, BigIntJSON.stringify);\n}\n\n/\n Streams out a `Source` for which messages are already stringified JSON.\n /\nexport function streamOutStringified(\n lc: LogContext,\n source: Source<string>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, json => json);\n}\n\nasync function streamOutInternal<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n stringify: (payload: T) => string,\n): Promise<void> {\n sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n const closer = WebSocketCloser.forSource(lc, sink, source);\n\n const acks = new Queue<Ack>();\n sink.addEventListener('message', ({data}) => {\n try {\n if (typeof data !== 'string') {\n throw new Error('Expected string message');\n }\n acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n } catch (e) {\n lc.error?.(`error parsing ack`, e);\n closer.close(e);\n }\n });\n\n try {\n let nextID = 0;\n const {pipeline} = source;\n if (pipeline) {\n lc.debug?.(`started pipelined outbound stream`);\n for await (const {value: msg, consumed} of pipeline) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`pipelining`, data);\n sink.send(data);\n\n void (async () => {\n const {ack} = await acks.dequeue();\n // lc.debug?.(`received ack`, ack);\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n consumed();\n })();\n }\n } else {\n lc.debug?.(`started synchronous outbound stream`);\n for await (const msg of source) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`sending`, data);\n sink.send(data);\n\n const {ack} = await acks.dequeue();\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n }\n }\n closer.close();\n } catch (e) {\n closer.close(e);\n }\n}\n\nexport function streamIn<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<T>> {\n return streamInInternal(lc, source, schema, data => data);\n}\n\n/\n Streams in messages sent by {@link streamOutStringified}, preserving the\n * exact application-level JSON alongside its parsed and validated value.\n */\nexport function streamInStringified<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<ParsedJSON<T>>> {\n return streamInInternal(lc, source, schema, (data, frame, id) => ({\n data,\n json: extractStringifiedMessage(frame, id),\n }));\n}\n\nasync function streamInInternal<T extends JSONValue, Out>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n transform: (data: T, frame: string, id: number) => Out,\n): Promise<Source<Out>> {\n expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n const streamedSchema = v.object({\n msg: schema,\n id: v.number(),\n });\n\n const sink: Subscription<Out, {id: number; data: Out}> = new Subscription<\n Out,\n {id: number; data: Out}\n >(\n {\n consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n cleanup: () => closer.close(),\n },\n ({data}) => data,\n );\n\n const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n function handleMessage(event: MessageEvent) {\n const data = event.data.toString();\n if (!sink.active) {\n lc.warn?.('dropping ws message received after close', data);\n return;\n }\n try {\n const value = BigIntJSON.parse(data);\n const msg = v.parse(value, streamedSchema, 'passthrough');\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`received`, data);\n sink.push({id: msg.id, data: transform(msg.msg, data, msg.id)});\n } catch (e) {\n closer.close(e);\n }\n }\n\n await closer.connected;\n return sink;\n}\n\nfunction extractStringifiedMessage(frame: string, id: number): string {\n const prefix = `{\"id\":${id},\"msg\":`;\n assert(\n frame.startsWith(prefix) && frame.endsWith('}'),\n `invalid streamed message frame`,\n );\n return frame.slice(prefix.length, -1);\n}\n\nclass WebSocketCloser {\n readonly #lc: LogContext;\n readonly #ws: WebSocket;\n readonly #closeStream: () => void;\n readonly #messageHandler: ((e: MessageEvent) => void \| undefined) \| null;\n readonly #connected = resolver();\n\n get connected(): Promise<void> {\n return this.#connected.promise;\n }\n\n static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n // If the websocket is closed, call cancel() to notify the Source of\n // any unconsumed messages.\n return new WebSocketCloser(lc, ws, () => stream.cancel());\n }\n\n static forSink<T, Input>(\n lc: LogContext,\n ws: WebSocket,\n stream: Subscription<T, Input>,\n messageHandler: (e: MessageEvent) => void \| undefined,\n ) {\n // If the websocket is closed, call end() to allow the downstream Sink\n // to process any pending messages before closing the stream.\n return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n }\n\n private constructor(\n lc: LogContext,\n ws: WebSocket,\n closeStream: () => void,\n messageHandler?: (e: MessageEvent) => void \| undefined,\n ) {\n this.#lc = lc;\n this.#ws = ws;\n this.#closeStream = closeStream;\n this.#messageHandler = messageHandler ?? null;\n\n ws.addEventListener('open', this.#handleOpen);\n ws.addEventListener('close', this.#handleClose);\n ws.addEventListener('error', this.#handleError);\n if (this.#messageHandler) {\n ws.addEventListener('message', this.#messageHandler);\n }\n\n switch (ws.readyState) {\n case ws.CONNECTING:\n break; // expected for new connections. resolve or reject in handlers.\n case ws.OPEN:\n this.#connected.resolve();\n break;\n default:\n this.#connected.reject(\n new Error(`websocket already in state ${ws.readyState}`),\n );\n break;\n }\n }\n\n get #conn(): string {\n return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n }\n\n #handleOpen = () => {\n this.#lc.info?.(`${this.#conn} established`);\n this.#connected.resolve();\n };\n\n #handleClose = (e: CloseEvent) => {\n const {code, reason, wasClean} = e;\n this.#lc.info?.(`${this.#conn} closed`, {\n code,\n reason,\n wasClean,\n });\n this.close();\n this.#connected.reject(`${this.#conn} closed with code ${code}`);\n };\n\n #handleError = ({message, error}: ErrorEvent) => {\n if (this.#ws.readyState === this.#ws.OPEN) {\n this.#lc.error?.(`error in ${this.#conn}`, message, error);\n }\n this.#connected.reject(error);\n };\n\n close(err?: unknown) {\n if (err) {\n this.#lc.error?.(`closing stream with error`, err);\n }\n this.#closeStream();\n if (!this.closed()) {\n this.#ws.close();\n }\n }\n\n closed() {\n return (\n this.#ws.readyState === this.#ws.CLOSED \|\|\n this.#ws.readyState === this.#ws.CLOSING\n );\n }\n}\n"],"mappings":";;;;;;;;;;AA6BA,IAAM,mBAAmB;;;;;AA4CzB,SAAgB,OACd,IACA,IACA,UACA,aAA2B,CAAC,GAC5B,YAAyB,CAAC,GAC1B,gBAA+B,CAAC,GACc;CAC9C,MAAM,WAAW,GAAG,OAAO;CAC3B,SAAS,MAAM,KAAe;EAC5B,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SACtD,IAAI,KACF,eAAe,IAAI,IAAI,GAAG;OACrB;GACL,GAAG,OAAO,yBAAyB,UAAU;GAC7C,GAAG,MAAM;EACX;CAEJ;CAEA,MAAM,WAAW,aAAa,OAAW;EACvC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,UAAU,UAAU,YAAY,GAAG;GACnC,MAAM,GAAG;EACX;CACF,CAAC;CACD,MAAM,YAAY,aAAa,OAAY;EACzC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,WAAW,UAAU,YAAY,GAAG;GACpC,MAAM,GAAG;EACX;CACF,CAAC;CAED,MAAM,SAAS,sBAAsB,IAAI;EACvC,GAAG;EACH,eAAe;CACjB,CAAC;CAGD,SAAS,YAAY;EAEnB,OAAO,GAAG,eAAe,GAAG,QAAQ,uBAAuB,UAAU,CAAC;EAEtE,SACE,SAAS,KAAK,SAAS,GACvB,IAAI,UAAU;GACZ,YAAY;GACZ,YAAY,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;EAC5C,CAAC,GACD,SACA,QAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAO,CACvD;CACF;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,GAAG,cAAc;EAClB,GAAG,OAAO,gBAAgB,UAAU;EACpC,UAAU;CACZ,CAAC;MAED,UAAU;CAIZ,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,UAAS;GAEd,OAAO,MADM,WAAW,MAAM,MAAM,SAAS,CAC9B,GAAM,UAAU,aAAa;EAC9C;CACF,CAAC;CAED,qBAAqB,IAAI,IAAI,gBAAgB;CAE7C,OAAO;EAAC;EAAW;CAAQ;AAC7B;AASA,SAAgB,KAAQ,EAAC,QAAQ,MAAM,OAAO,kBAAiC;CAC7E,mBAAmB;CACnB,OAAO,kBAAkB,GAAG,qCAAqC;CACjE,MAAM,UAA8B,CAAC;CAErC,SACE,QACA,IAAI,SAAS;EACX,eAAe;EACf,QAAQ,OAAO,WAAW,aAAa;GACrC,IAAI;GACJ,IAAI;IACF,KAAK,MAAM,MAAM,KAAK,OAAO,MAAM;KACjC,SAAS;KACT;IACF;GACF,SAAS,KAAK;IACZ,SAAS,YAAY,GAAG,CAAC;IACzB;GACF;GAMA,MAAM,EAAC,WAAU,KAAK,KAAK,GAAG;GAC9B,QAAQ,KAAK,MAAM;GACnB,OAAY,WAAW,QAAQ,MAAM,CAAC;GAEtC,IAAI,QAAQ,UAAU,gBAEpB,SAAS;QAGT,QAAQ,GAAG,WACH,SAAS,IACf,QAAO,SAAS,YAAY,GAAG,CAAC,CAClC;EAEJ;EACA,UAAU,KAAK,aAAa;GAC1B,IAAI,KACF,KAAK,KAAK,YAAY,GAAG,CAAC;GAG5B,SAAS;EACX;EACA,QAAO,aAAY;GACjB,KAAK,OAAO;GACZ,SAAS;EACX;CACF,CAAC,IACD,QAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAO,CAC7C;AACF;AAEA,SAAS,YAAY,KAAc;CACjC,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,IAAM,YAAY,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;AAU5C,SAAgB,UACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,MAAM,WAAW,SAAS;AACjE;;;;AAKA,SAAgB,qBACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,OAAM,SAAQ,IAAI;AACzD;AAEA,eAAe,kBACb,IACA,QACA,MACA,WACe;CACf,qBAAqB,IAAI,MAAM,gBAAgB;CAE/C,MAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;CAEzD,MAAM,OAAO,IAAI,MAAW;CAC5B,KAAK,iBAAiB,YAAY,EAAC,WAAU;EAC3C,IAAI;GACF,IAAI,OAAO,SAAS,UAClB,MAAM,IAAI,MAAM,yBAAyB;GAE3C,KAAK,QAAQ,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;EACnD,SAAS,GAAG;GACV,GAAG,QAAQ,qBAAqB,CAAC;GACjC,OAAO,MAAM,CAAC;EAChB;CACF,CAAC;CAED,IAAI;EACF,IAAI,SAAS;EACb,MAAM,EAAC,aAAY;EACnB,IAAI,UAAU;GACZ,GAAG,QAAQ,mCAAmC;GAC9C,WAAW,MAAM,EAAC,OAAO,KAAK,cAAa,UAAU;IACnD,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,CAAM,YAAY;KAChB,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;KAEjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;KAEpD,SAAS;IACX,GAAG;GACL;EACF,OAAO;GACL,GAAG,QAAQ,qCAAqC;GAChD,WAAW,MAAM,OAAO,QAAQ;IAC9B,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;IACjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;GAEtD;EACF;EACA,OAAO,MAAM;CACf,SAAS,GAAG;EACV,OAAO,MAAM,CAAC;CAChB;AACF;AAEA,SAAgB,SACd,IACA,QACA,QACoB;CACpB,OAAO,iBAAiB,IAAI,QAAQ,SAAQ,SAAQ,IAAI;AAC1D;;;;;AAMA,SAAgB,oBACd,IACA,QACA,QACgC;CAChC,OAAO,iBAAiB,IAAI,QAAQ,SAAS,MAAM,OAAO,QAAQ;EAChE;EACA,MAAM,0BAA0B,OAAO,EAAE;CAC3C,EAAE;AACJ;AAEA,eAAe,iBACb,IACA,QACA,QACA,WACsB;CACtB,uBAAuB,IAAI,QAAQ,gBAAgB;CAEnD,MAAM,iBAAiB,eAAE,OAAO;EAC9B,KAAK;EACL,IAAI,eAAE,OAAO;CACf,CAAC;CAED,MAAM,OAAmD,IAAI,aAI3D;EACE,WAAW,EAAC,SAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAE,CAAe,CAAC;EACvE,eAAe,OAAO,MAAM;CAC9B,IACC,EAAC,WAAU,IACd;CAEA,MAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;CAEtE,SAAS,cAAc,OAAqB;EAC1C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,CAAC,KAAK,QAAQ;GAChB,GAAG,OAAO,4CAA4C,IAAI;GAC1D;EACF;EACA,IAAI;GAEF,MAAM,MAAM,MADE,WAAW,MAAM,IACX,GAAO,gBAAgB,aAAa;GAGxD,KAAK,KAAK;IAAC,IAAI,IAAI;IAAI,MAAM,UAAU,IAAI,KAAK,MAAM,IAAI,EAAE;GAAC,CAAC;EAChE,SAAS,GAAG;GACV,OAAO,MAAM,CAAC;EAChB;CACF;CAEA,MAAM,OAAO;CACb,OAAO;AACT;AAEA,SAAS,0BAA0B,OAAe,IAAoB;CACpE,MAAM,SAAS,SAAS,GAAG;CAC3B,OACE,MAAM,WAAW,MAAM,KAAK,MAAM,SAAS,GAAG,GAC9C,gCACF;CACA,OAAO,MAAM,MAAM,OAAO,QAAQ,EAAE;AACtC;AAEA,IAAM,kBAAN,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA,aAAsB,SAAS;CAE/B,IAAI,YAA2B;EAC7B,OAAO,KAAKI,WAAW;CACzB;CAEA,OAAO,UAAa,IAAgB,IAAe,QAAmB;EAGpE,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,OAAO,CAAC;CAC1D;CAEA,OAAO,QACL,IACA,IACA,QACA,gBACA;EAGA,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,IAAI,GAAG,cAAc;CACvE;CAEA,YACE,IACA,IACA,aACA,gBACA;EACA,KAAKJ,MAAM;EACX,KAAKC,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,kBAAkB,kBAAkB;EAEzC,GAAG,iBAAiB,QAAQ,KAAKE,WAAW;EAC5C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,IAAI,KAAKJ,iBACP,GAAG,iBAAiB,WAAW,KAAKA,eAAe;EAGrD,QAAQ,GAAG,YAAX;GACE,KAAK,GAAG,YACN;GACF,KAAK,GAAG;IACN,KAAKC,WAAW,QAAQ;IACxB;GACF;IACE,KAAKA,WAAW,uBACd,IAAI,MAAM,8BAA8B,GAAG,YAAY,CACzD;IACA;EACJ;CACF;CAEA,IAAII,QAAgB;EAClB,OAAO,gBAAgB,KAAKP,IAAI,MAAM,OAAO,KAAKA,IAAI,QAAQ;CAChE;CAEA,oBAAoB;EAClB,KAAKD,IAAI,OAAO,GAAG,KAAKQ,MAAM,aAAa;EAC3C,KAAKJ,WAAW,QAAQ;CAC1B;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,KAAKJ,IAAI,OAAO,GAAG,KAAKQ,MAAM,UAAU;GACtC;GACA;GACA;EACF,CAAC;EACD,KAAK,MAAM;EACX,KAAKJ,WAAW,OAAO,GAAG,KAAKI,MAAM,oBAAoB,MAAM;CACjE;CAEA,gBAAgB,EAAC,SAAS,YAAuB;EAC/C,IAAI,KAAKP,IAAI,eAAe,KAAKA,IAAI,MACnC,KAAKD,IAAI,QAAQ,YAAY,KAAKQ,SAAS,SAAS,KAAK;EAE3D,KAAKJ,WAAW,OAAO,KAAK;CAC9B;CAEA,MAAM,KAAe;EACnB,IAAI,KACF,KAAKJ,IAAI,QAAQ,6BAA6B,GAAG;EAEnD,KAAKE,aAAa;EAClB,IAAI,CAAC,KAAK,OAAO,GACf,KAAKD,IAAI,MAAM;CAEnB;CAEA,SAAS;EACP,OACE,KAAKA,IAAI,eAAe,KAAKA,IAAI,UACjC,KAAKA,IAAI,eAAe,KAAKA,IAAI;CAErC;AACF"}
1	+ {"version":3,"file":"streams.js","names":["#lc","#ws","#closeStream","#messageHandler","#connected","#handleOpen","#handleClose","#handleError","#conn"],"sources":["../../../../../zero-cache/src/types/streams.ts"],"sourcesContent":["import {\n pipeline,\n Readable,\n Transform,\n Writable,\n type DuplexOptions,\n} from 'node:stream';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {\n createWebSocketStream,\n type CloseEvent,\n type ErrorEvent,\n type MessageEvent,\n type WebSocket,\n} from 'ws';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {BigIntJSON, type JSONValue} from '../../../shared/src/bigint-json.ts';\nimport {Queue} from '../../../shared/src/queue.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Subscription, type Options} from './subscription.ts';\nimport {\n closeWithError,\n expectPingsForLiveness,\n sendPingsForLiveness,\n} from './ws.ts';\n\n// Consistent with Postgres keepalives, and shorter than the\n// commonly used default idle timeout of 1 minute.\nconst PING_INTERVAL_MS = 30_000;\n\nexport type Source<T> = AsyncIterable<T> & {\n /*\n Immediately terminates all current iterations (i.e. {@link AsyncIterator.next next()})\n * will return `{value: undefined, done: true}`), and prevents any subsequent iterations\n * from yielding any values.\n \n @param err Terminate the iteration by throwing the `err` instead.\n /\n cancel: (err?: Error) => void;\n\n /\n An AbortSignal that can be used to listen for termination or\n * race a short-lived promise against the termination of the iterable\n * (via `promiseOrAbort()`)\n /\n readonly signal: AbortSignal;\n\n /\n The presence of a `pipeline` iterable allows the usual \"consumed-on-iterate\" semantics\n * to be overridden.\n \n This is suitable for transport layers that serialize messages across processes, such\n * as the {@link streamOut()} method; pipelining allows the transport to send messages\n * as they arrive without waiting for the previous message to be acked, streaming\n * them to the receiving process where they are presumably queued and processed without\n * a per-message ack delay. The receiving end of the transport then responds with acks\n * asynchronously as the receiving end processes the messages.\n /\n pipeline?: AsyncIterable<{value: T; consumed: () => void}> \| undefined;\n};\n\nexport type Sink<T> = {\n push(message: T): void;\n};\n\n/\n Back-pressure-aware transformation of a WebSocket into\n * upstream and downstream {@link Subscription} objects.\n /\n// TODO: Change {@link streamIn} and {@link streamOut} to use this\n// under the covers so that internal communication is also\n// responsive to backpressure.\nexport function stream<In extends JSONValue, Out extends JSONValue>(\n lc: LogContext,\n ws: WebSocket,\n inSchema: v.Type<In>,\n outOptions: Options<Out> = {},\n inOptions: Options<In> = {},\n streamOptions: DuplexOptions = {},\n): {outstream: Sink<Out>; instream: Source<In>} {\n const endpoint = ws.url ?? 'client';\n function close(err?: unknown) {\n if (ws.readyState !== ws.CLOSED && ws.readyState !== ws.CLOSING) {\n if (err) {\n closeWithError(lc, ws, err);\n } else {\n lc.info?.(`closing connection to ${endpoint}`);\n ws.close();\n }\n }\n }\n\n const instream = Subscription.create<In>({\n ...inOptions,\n cleanup: (unconsumed, err) => {\n inOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n const outstream = Subscription.create<Out>({\n ...outOptions,\n cleanup: (unconsumed, err) => {\n outOptions.cleanup?.(unconsumed, err);\n close(err);\n },\n });\n\n const duplex = createWebSocketStream(ws, {\n ...streamOptions,\n decodeStrings: false,\n });\n\n // Outgoing transform.\n function streamOut() {\n // Mainly used for verifying that back-pressure kicks in tests.\n duplex.on('drain', () => lc.debug?.(`drained messages to ${endpoint}`));\n\n pipeline(\n Readable.from(outstream),\n new Transform({\n objectMode: true,\n transform: (msg, _encoding, callback) =>\n callback(null, BigIntJSON.stringify(msg)),\n }),\n duplex,\n err => (err ? outstream.fail(err) : outstream.cancel()),\n );\n }\n\n if (ws.readyState === ws.CONNECTING) {\n ws.on('open', () => {\n lc.info?.(`connected to ${endpoint}`);\n streamOut();\n });\n } else {\n streamOut();\n }\n\n // Incoming transform.\n pipe({\n source: duplex,\n sink: instream,\n parse: chunk => {\n const json = BigIntJSON.parse(chunk.toString());\n return v.parse(json, inSchema, 'passthrough');\n },\n });\n\n sendPingsForLiveness(lc, ws, PING_INTERVAL_MS);\n\n return {outstream, instream};\n}\n\ntype PipeOptions<T> = {\n source: Readable;\n sink: Subscription<T>;\n parse: (buffer: Buffer) => T \| null;\n bufferMessages?: number;\n};\n\nexport function pipe<T>({source, sink, parse, bufferMessages}: PipeOptions<T>) {\n bufferMessages ??= 0;\n assert(bufferMessages >= 0, 'bufferMessages must be non-negative');\n const pending: Promise<unknown>[] = [];\n\n pipeline(\n source,\n new Writable({\n decodeStrings: false,\n write: (chunk, _encoding, callback) => {\n let msg: T \| null;\n try {\n if ((msg = parse(chunk)) === null) {\n callback();\n return;\n }\n } catch (err) {\n callback(ensureError(err));\n return;\n }\n // Inbound backpressure is exerted by unconsumed messages in the\n // subscription. A buffer can be used to allow messages to queue up in\n // in the Subscription object, which allows the consumer to \"peek\" at\n // whether there are more messages immediately available\n // (via {@link Subscription.queued}.\n const {result} = sink.push(msg);\n pending.push(result);\n void result.then(() => pending.shift());\n\n if (pending.length <= bufferMessages) {\n // immediately allow more messages\n callback();\n } else {\n // wait for the oldest result in the pending queue\n pending[0].then(\n () => callback(),\n err => callback(ensureError(err)),\n );\n }\n },\n destroy: (err, callback) => {\n if (err) {\n sink.fail(ensureError(err));\n }\n // Otherwise, final will handle the cancel.\n callback();\n },\n final: callback => {\n sink.cancel();\n callback();\n },\n }),\n err => (err ? sink.fail(err) : sink.cancel()),\n );\n}\n\nfunction ensureError(err: unknown) {\n return err instanceof Error ? err : new Error(String(err));\n}\n\nconst ackSchema = v.object({ack: v.number()});\n\ntype Ack = v.Infer<typeof ackSchema>;\n\n/* A parsed value paired with its approximate serialized transport size. /\nexport type Sized<T> = {\n data: T;\n size: number;\n};\n\nexport function streamOut<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, BigIntJSON.stringify);\n}\n\n/\n Streams out a `Source` for which messages are already stringified JSON.\n /\nexport function streamOutStringified(\n lc: LogContext,\n source: Source<string>,\n sink: WebSocket,\n): Promise<void> {\n return streamOutInternal(lc, source, sink, json => json);\n}\n\nasync function streamOutInternal<T extends JSONValue>(\n lc: LogContext,\n source: Source<T>,\n sink: WebSocket,\n stringify: (payload: T) => string,\n): Promise<void> {\n sendPingsForLiveness(lc, sink, PING_INTERVAL_MS);\n\n const closer = WebSocketCloser.forSource(lc, sink, source);\n\n const acks = new Queue<Ack>();\n sink.addEventListener('message', ({data}) => {\n try {\n if (typeof data !== 'string') {\n throw new Error('Expected string message');\n }\n acks.enqueue(v.parse(JSON.parse(data), ackSchema));\n } catch (e) {\n lc.error?.(`error parsing ack`, e);\n closer.close(e);\n }\n });\n\n try {\n let nextID = 0;\n const {pipeline} = source;\n if (pipeline) {\n lc.debug?.(`started pipelined outbound stream`);\n for await (const {value: msg, consumed} of pipeline) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`pipelining`, data);\n sink.send(data);\n\n void (async () => {\n const {ack} = await acks.dequeue();\n // lc.debug?.(`received ack`, ack);\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n consumed();\n })();\n }\n } else {\n lc.debug?.(`started synchronous outbound stream`);\n for await (const msg of source) {\n const id = ++nextID;\n const data = `{\"id\":${id},\"msg\":${stringify(msg)}}`;\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`sending`, data);\n sink.send(data);\n\n const {ack} = await acks.dequeue();\n if (ack !== id) {\n throw new Error(`Unexpected ack for ${id}: ${ack}`);\n }\n }\n }\n closer.close();\n } catch (e) {\n closer.close(e);\n }\n}\n\nexport function streamIn<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<T>> {\n return streamInInternal(lc, source, schema, data => data);\n}\n\n/\n Streams in parsed messages while retaining only the transport-frame size.\n * The size bounds downstream batching without keeping or copying the JSON.\n */\nexport function streamInWithSize<T extends JSONValue>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n): Promise<Source<Sized<T>>> {\n return streamInInternal(lc, source, schema, (data, frame) => ({\n data,\n size: frame.length,\n }));\n}\n\nasync function streamInInternal<T extends JSONValue, Out>(\n lc: LogContext,\n source: WebSocket,\n schema: v.Type<T>,\n transform: (data: T, frame: string, id: number) => Out,\n): Promise<Source<Out>> {\n expectPingsForLiveness(lc, source, PING_INTERVAL_MS);\n\n const streamedSchema = v.object({\n msg: schema,\n id: v.number(),\n });\n\n const sink: Subscription<Out, {id: number; data: Out}> = new Subscription<\n Out,\n {id: number; data: Out}\n >(\n {\n consumed: ({id}) => source.send(JSON.stringify({ack: id} satisfies Ack)),\n cleanup: () => closer.close(),\n },\n ({data}) => data,\n );\n\n const closer = WebSocketCloser.forSink(lc, source, sink, handleMessage);\n\n function handleMessage(event: MessageEvent) {\n const data = event.data.toString();\n if (!sink.active) {\n lc.warn?.('dropping ws message received after close', data);\n return;\n }\n try {\n const value = BigIntJSON.parse(data);\n const msg = v.parse(value, streamedSchema, 'passthrough');\n // Enable for debugging. Otherwise too verbose.\n // lc.debug?.(`received`, data);\n sink.push({id: msg.id, data: transform(msg.msg, data, msg.id)});\n } catch (e) {\n closer.close(e);\n }\n }\n\n await closer.connected;\n return sink;\n}\n\nclass WebSocketCloser {\n readonly #lc: LogContext;\n readonly #ws: WebSocket;\n readonly #closeStream: () => void;\n readonly #messageHandler: ((e: MessageEvent) => void \| undefined) \| null;\n readonly #connected = resolver();\n\n get connected(): Promise<void> {\n return this.#connected.promise;\n }\n\n static forSource<T>(lc: LogContext, ws: WebSocket, stream: Source<T>) {\n // If the websocket is closed, call cancel() to notify the Source of\n // any unconsumed messages.\n return new WebSocketCloser(lc, ws, () => stream.cancel());\n }\n\n static forSink<T, Input>(\n lc: LogContext,\n ws: WebSocket,\n stream: Subscription<T, Input>,\n messageHandler: (e: MessageEvent) => void \| undefined,\n ) {\n // If the websocket is closed, call end() to allow the downstream Sink\n // to process any pending messages before closing the stream.\n return new WebSocketCloser(lc, ws, () => stream.end(), messageHandler);\n }\n\n private constructor(\n lc: LogContext,\n ws: WebSocket,\n closeStream: () => void,\n messageHandler?: (e: MessageEvent) => void \| undefined,\n ) {\n this.#lc = lc;\n this.#ws = ws;\n this.#closeStream = closeStream;\n this.#messageHandler = messageHandler ?? null;\n\n ws.addEventListener('open', this.#handleOpen);\n ws.addEventListener('close', this.#handleClose);\n ws.addEventListener('error', this.#handleError);\n if (this.#messageHandler) {\n ws.addEventListener('message', this.#messageHandler);\n }\n\n switch (ws.readyState) {\n case ws.CONNECTING:\n break; // expected for new connections. resolve or reject in handlers.\n case ws.OPEN:\n this.#connected.resolve();\n break;\n default:\n this.#connected.reject(\n new Error(`websocket already in state ${ws.readyState}`),\n );\n break;\n }\n }\n\n get #conn(): string {\n return 'connection' + (this.#ws.url ? ` to ${this.#ws.url}` : '');\n }\n\n #handleOpen = () => {\n this.#lc.info?.(`${this.#conn} established`);\n this.#connected.resolve();\n };\n\n #handleClose = (e: CloseEvent) => {\n const {code, reason, wasClean} = e;\n this.#lc.info?.(`${this.#conn} closed`, {\n code,\n reason,\n wasClean,\n });\n this.close();\n this.#connected.reject(`${this.#conn} closed with code ${code}`);\n };\n\n #handleError = ({message, error}: ErrorEvent) => {\n if (this.#ws.readyState === this.#ws.OPEN) {\n this.#lc.error?.(`error in ${this.#conn}`, message, error);\n }\n this.#connected.reject(error);\n };\n\n close(err?: unknown) {\n if (err) {\n this.#lc.error?.(`closing stream with error`, err);\n }\n this.#closeStream();\n if (!this.closed()) {\n this.#ws.close();\n }\n }\n\n closed() {\n return (\n this.#ws.readyState === this.#ws.CLOSED \|\|\n this.#ws.readyState === this.#ws.CLOSING\n );\n }\n}\n"],"mappings":";;;;;;;;;;AA6BA,IAAM,mBAAmB;;;;;AA4CzB,SAAgB,OACd,IACA,IACA,UACA,aAA2B,CAAC,GAC5B,YAAyB,CAAC,GAC1B,gBAA+B,CAAC,GACc;CAC9C,MAAM,WAAW,GAAG,OAAO;CAC3B,SAAS,MAAM,KAAe;EAC5B,IAAI,GAAG,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,SACtD,IAAI,KACF,eAAe,IAAI,IAAI,GAAG;OACrB;GACL,GAAG,OAAO,yBAAyB,UAAU;GAC7C,GAAG,MAAM;EACX;CAEJ;CAEA,MAAM,WAAW,aAAa,OAAW;EACvC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,UAAU,UAAU,YAAY,GAAG;GACnC,MAAM,GAAG;EACX;CACF,CAAC;CACD,MAAM,YAAY,aAAa,OAAY;EACzC,GAAG;EACH,UAAU,YAAY,QAAQ;GAC5B,WAAW,UAAU,YAAY,GAAG;GACpC,MAAM,GAAG;EACX;CACF,CAAC;CAED,MAAM,SAAS,sBAAsB,IAAI;EACvC,GAAG;EACH,eAAe;CACjB,CAAC;CAGD,SAAS,YAAY;EAEnB,OAAO,GAAG,eAAe,GAAG,QAAQ,uBAAuB,UAAU,CAAC;EAEtE,SACE,SAAS,KAAK,SAAS,GACvB,IAAI,UAAU;GACZ,YAAY;GACZ,YAAY,KAAK,WAAW,aAC1B,SAAS,MAAM,WAAW,UAAU,GAAG,CAAC;EAC5C,CAAC,GACD,SACA,QAAQ,MAAM,UAAU,KAAK,GAAG,IAAI,UAAU,OAAO,CACvD;CACF;CAEA,IAAI,GAAG,eAAe,GAAG,YACvB,GAAG,GAAG,cAAc;EAClB,GAAG,OAAO,gBAAgB,UAAU;EACpC,UAAU;CACZ,CAAC;MAED,UAAU;CAIZ,KAAK;EACH,QAAQ;EACR,MAAM;EACN,QAAO,UAAS;GAEd,OAAO,MADM,WAAW,MAAM,MAAM,SAAS,CAC9B,GAAM,UAAU,aAAa;EAC9C;CACF,CAAC;CAED,qBAAqB,IAAI,IAAI,gBAAgB;CAE7C,OAAO;EAAC;EAAW;CAAQ;AAC7B;AASA,SAAgB,KAAQ,EAAC,QAAQ,MAAM,OAAO,kBAAiC;CAC7E,mBAAmB;CACnB,OAAO,kBAAkB,GAAG,qCAAqC;CACjE,MAAM,UAA8B,CAAC;CAErC,SACE,QACA,IAAI,SAAS;EACX,eAAe;EACf,QAAQ,OAAO,WAAW,aAAa;GACrC,IAAI;GACJ,IAAI;IACF,KAAK,MAAM,MAAM,KAAK,OAAO,MAAM;KACjC,SAAS;KACT;IACF;GACF,SAAS,KAAK;IACZ,SAAS,YAAY,GAAG,CAAC;IACzB;GACF;GAMA,MAAM,EAAC,WAAU,KAAK,KAAK,GAAG;GAC9B,QAAQ,KAAK,MAAM;GACnB,OAAY,WAAW,QAAQ,MAAM,CAAC;GAEtC,IAAI,QAAQ,UAAU,gBAEpB,SAAS;QAGT,QAAQ,GAAG,WACH,SAAS,IACf,QAAO,SAAS,YAAY,GAAG,CAAC,CAClC;EAEJ;EACA,UAAU,KAAK,aAAa;GAC1B,IAAI,KACF,KAAK,KAAK,YAAY,GAAG,CAAC;GAG5B,SAAS;EACX;EACA,QAAO,aAAY;GACjB,KAAK,OAAO;GACZ,SAAS;EACX;CACF,CAAC,IACD,QAAQ,MAAM,KAAK,KAAK,GAAG,IAAI,KAAK,OAAO,CAC7C;AACF;AAEA,SAAS,YAAY,KAAc;CACjC,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAC3D;AAEA,IAAM,YAAY,eAAE,OAAO,EAAC,KAAK,eAAE,OAAO,EAAC,CAAC;AAU5C,SAAgB,UACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,MAAM,WAAW,SAAS;AACjE;;;;AAKA,SAAgB,qBACd,IACA,QACA,MACe;CACf,OAAO,kBAAkB,IAAI,QAAQ,OAAM,SAAQ,IAAI;AACzD;AAEA,eAAe,kBACb,IACA,QACA,MACA,WACe;CACf,qBAAqB,IAAI,MAAM,gBAAgB;CAE/C,MAAM,SAAS,gBAAgB,UAAU,IAAI,MAAM,MAAM;CAEzD,MAAM,OAAO,IAAI,MAAW;CAC5B,KAAK,iBAAiB,YAAY,EAAC,WAAU;EAC3C,IAAI;GACF,IAAI,OAAO,SAAS,UAClB,MAAM,IAAI,MAAM,yBAAyB;GAE3C,KAAK,QAAQ,MAAQ,KAAK,MAAM,IAAI,GAAG,SAAS,CAAC;EACnD,SAAS,GAAG;GACV,GAAG,QAAQ,qBAAqB,CAAC;GACjC,OAAO,MAAM,CAAC;EAChB;CACF,CAAC;CAED,IAAI;EACF,IAAI,SAAS;EACb,MAAM,EAAC,aAAY;EACnB,IAAI,UAAU;GACZ,GAAG,QAAQ,mCAAmC;GAC9C,WAAW,MAAM,EAAC,OAAO,KAAK,cAAa,UAAU;IACnD,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,CAAM,YAAY;KAChB,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;KAEjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;KAEpD,SAAS;IACX,GAAG;GACL;EACF,OAAO;GACL,GAAG,QAAQ,qCAAqC;GAChD,WAAW,MAAM,OAAO,QAAQ;IAC9B,MAAM,KAAK,EAAE;IACb,MAAM,OAAO,SAAS,GAAG,SAAS,UAAU,GAAG,EAAE;IAGjD,KAAK,KAAK,IAAI;IAEd,MAAM,EAAC,QAAO,MAAM,KAAK,QAAQ;IACjC,IAAI,QAAQ,IACV,MAAM,IAAI,MAAM,sBAAsB,GAAG,IAAI,KAAK;GAEtD;EACF;EACA,OAAO,MAAM;CACf,SAAS,GAAG;EACV,OAAO,MAAM,CAAC;CAChB;AACF;AAEA,SAAgB,SACd,IACA,QACA,QACoB;CACpB,OAAO,iBAAiB,IAAI,QAAQ,SAAQ,SAAQ,IAAI;AAC1D;;;;;AAMA,SAAgB,iBACd,IACA,QACA,QAC2B;CAC3B,OAAO,iBAAiB,IAAI,QAAQ,SAAS,MAAM,WAAW;EAC5D;EACA,MAAM,MAAM;CACd,EAAE;AACJ;AAEA,eAAe,iBACb,IACA,QACA,QACA,WACsB;CACtB,uBAAuB,IAAI,QAAQ,gBAAgB;CAEnD,MAAM,iBAAiB,eAAE,OAAO;EAC9B,KAAK;EACL,IAAI,eAAE,OAAO;CACf,CAAC;CAED,MAAM,OAAmD,IAAI,aAI3D;EACE,WAAW,EAAC,SAAQ,OAAO,KAAK,KAAK,UAAU,EAAC,KAAK,GAAE,CAAe,CAAC;EACvE,eAAe,OAAO,MAAM;CAC9B,IACC,EAAC,WAAU,IACd;CAEA,MAAM,SAAS,gBAAgB,QAAQ,IAAI,QAAQ,MAAM,aAAa;CAEtE,SAAS,cAAc,OAAqB;EAC1C,MAAM,OAAO,MAAM,KAAK,SAAS;EACjC,IAAI,CAAC,KAAK,QAAQ;GAChB,GAAG,OAAO,4CAA4C,IAAI;GAC1D;EACF;EACA,IAAI;GAEF,MAAM,MAAM,MADE,WAAW,MAAM,IACX,GAAO,gBAAgB,aAAa;GAGxD,KAAK,KAAK;IAAC,IAAI,IAAI;IAAI,MAAM,UAAU,IAAI,KAAK,MAAM,IAAI,EAAE;GAAC,CAAC;EAChE,SAAS,GAAG;GACV,OAAO,MAAM,CAAC;EAChB;CACF;CAEA,MAAM,OAAO;CACb,OAAO;AACT;AAEA,IAAM,kBAAN,MAAM,gBAAgB;CACpB;CACA;CACA;CACA;CACA,aAAsB,SAAS;CAE/B,IAAI,YAA2B;EAC7B,OAAO,KAAKI,WAAW;CACzB;CAEA,OAAO,UAAa,IAAgB,IAAe,QAAmB;EAGpE,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,OAAO,CAAC;CAC1D;CAEA,OAAO,QACL,IACA,IACA,QACA,gBACA;EAGA,OAAO,IAAI,gBAAgB,IAAI,UAAU,OAAO,IAAI,GAAG,cAAc;CACvE;CAEA,YACE,IACA,IACA,aACA,gBACA;EACA,KAAKJ,MAAM;EACX,KAAKC,MAAM;EACX,KAAKC,eAAe;EACpB,KAAKC,kBAAkB,kBAAkB;EAEzC,GAAG,iBAAiB,QAAQ,KAAKE,WAAW;EAC5C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,GAAG,iBAAiB,SAAS,KAAKC,YAAY;EAC9C,IAAI,KAAKJ,iBACP,GAAG,iBAAiB,WAAW,KAAKA,eAAe;EAGrD,QAAQ,GAAG,YAAX;GACE,KAAK,GAAG,YACN;GACF,KAAK,GAAG;IACN,KAAKC,WAAW,QAAQ;IACxB;GACF;IACE,KAAKA,WAAW,uBACd,IAAI,MAAM,8BAA8B,GAAG,YAAY,CACzD;IACA;EACJ;CACF;CAEA,IAAII,QAAgB;EAClB,OAAO,gBAAgB,KAAKP,IAAI,MAAM,OAAO,KAAKA,IAAI,QAAQ;CAChE;CAEA,oBAAoB;EAClB,KAAKD,IAAI,OAAO,GAAG,KAAKQ,MAAM,aAAa;EAC3C,KAAKJ,WAAW,QAAQ;CAC1B;CAEA,gBAAgB,MAAkB;EAChC,MAAM,EAAC,MAAM,QAAQ,aAAY;EACjC,KAAKJ,IAAI,OAAO,GAAG,KAAKQ,MAAM,UAAU;GACtC;GACA;GACA;EACF,CAAC;EACD,KAAK,MAAM;EACX,KAAKJ,WAAW,OAAO,GAAG,KAAKI,MAAM,oBAAoB,MAAM;CACjE;CAEA,gBAAgB,EAAC,SAAS,YAAuB;EAC/C,IAAI,KAAKP,IAAI,eAAe,KAAKA,IAAI,MACnC,KAAKD,IAAI,QAAQ,YAAY,KAAKQ,SAAS,SAAS,KAAK;EAE3D,KAAKJ,WAAW,OAAO,KAAK;CAC9B;CAEA,MAAM,KAAe;EACnB,IAAI,KACF,KAAKJ,IAAI,QAAQ,6BAA6B,GAAG;EAEnD,KAAKE,aAAa;EAClB,IAAI,CAAC,KAAK,OAAO,GACf,KAAKD,IAAI,MAAM;CAEnB;CAEA,SAAS;EACP,OACE,KAAKA,IAAI,eAAe,KAAKA,IAAI,UACjC,KAAKA,IAAI,eAAe,KAAKA,IAAI;CAErC;AACF"}

package/out/zero-cache/src/workers/connection.js CHANGED Viewed

@@ -78,9 +78,9 @@ var Connection = class {
 	* will only parse messages with schema(s) of supported protocol versions.
 	*/
 	init() {
-		if (this.#protocolVersion > 52 || this.#protocolVersion < 30) this.#closeWithError({
+		if (this.#protocolVersion > 53 || this.#protocolVersion < 30) this.#closeWithError({
 			kind: VersionNotSupported,
-			message: `server is at sync protocol v52 and does not support v${this.#protocolVersion}. The ${this.#protocolVersion > 52 ? "server" : "client"} must be updated to a newer release.`,
+			message: `server is at sync protocol v53 and does not support v${this.#protocolVersion}. The ${this.#protocolVersion > 53 ? "server" : "client"} must be updated to a newer release.`,
 			origin: ZeroCache
 		});
 		else {

package/out/zero-cache/src/workers/replicator.d.ts CHANGED Viewed

@@ -43,6 +43,7 @@ export declare function getPragmaConfig(mode: ReplicaFileMode): PragmaConfig;
 export declare function setupReplica(lc: LogContext, mode: ReplicaFileMode, replicaOptions: ReplicaOptions): Promise<{
     file: string;
     walMode: WalMode;
+    pageSize: number;
 }>;
 export declare function setUpMessageHandlers(lc: LogContext, replicator: Replicator, parent: Worker): void;
 export declare function handleSubscriptionsFrom(lc: LogContext, subscriber: Worker, notifier: ReplicaStateNotifier): void;

package/out/zero-cache/src/workers/replicator.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AAEnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0BAA0B,CAAC;AAI7D,OAAO,EAAC,QAAQ,EAAC,MAAM,oCAAoC,CAAC;AAC5D,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACX,MAAM,sCAAsC,CAAC;AAK9C,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,+CAA+C,CAAC;AACvD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAElD,eAAO,MAAM,qBAAqB,+CAIjC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,MAAM,MAAM,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;AAErC,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,UAEzE;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,+BAA+B,CAC7C,mBAAmB,EAAE,MAAM,GAC1B,OAAO,CAET;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,mBAAmB,EAAE,MAAM,EAC3B,MAAM,EAAE,MAAM,GACb,OAAO,CAMT;~~AA4FD~~;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,eAAe,GAAG,YAAY,CAMnE;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,eAAe,EACrB,cAAc,EAAE,cAAc;~~UAlGd~~,MAAM;aAAW,OAAO;~~GAiIzC~~;AAED,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,MAAM,QAGf;AAID,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,oBAAoB,QA8B/B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GACvC,QAAQ,CAOV;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,QAE1D"}
1	+ {"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/workers/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AAEnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0BAA0B,CAAC;AAI7D,OAAO,EAAC,QAAQ,EAAC,MAAM,oCAAoC,CAAC;AAC5D,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EACpB,UAAU,EACX,MAAM,sCAAsC,CAAC;AAK9C,OAAO,EAEL,KAAK,YAAY,EAClB,MAAM,+CAA+C,CAAC;AACvD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,uBAAuB,CAAC;AAElD,eAAO,MAAM,qBAAqB,+CAIjC,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,MAAM,MAAM,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;AAErC,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,UAEzE;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,+BAA+B,CAC7C,mBAAmB,EAAE,MAAM,GAC1B,OAAO,CAET;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,mBAAmB,EAAE,MAAM,EAC3B,MAAM,EAAE,MAAM,GACb,OAAO,CAMT;AA2FD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,eAAe,GAAG,YAAY,CAMnE;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,UAAU,EACd,IAAI,EAAE,eAAe,EACrB,cAAc,EAAE,cAAc;UAjGd,MAAM;aAAW,OAAO;cAAY,MAAM;GAgI3D;AAED,wBAAgB,oBAAoB,CAClC,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,UAAU,EACtB,MAAM,EAAE,MAAM,QAGf;AAID,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,oBAAoB,QA8B/B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,GACvC,QAAQ,CAOV;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,QAE1D"}

package/out/zero-cache/src/workers/replicator.js CHANGED Viewed

@@ -48,9 +48,10 @@ var MB = 1024 * 1024;
 async function prepare(lc, { file, vacuumIntervalHours }, walMode, mode) {
 	await upgradeReplica(lc, `${mode}-replica`, file);
 	const replica = new Database(lc, file);
+	let pageSize;
 	try {
 		await setJournalMode(lc, replica, "delete");
-		const [{ page_size: pageSize }] = replica.pragma("page_size");
+		[{page_size: pageSize}] = replica.pragma("page_size");
 		const [{ page_count: pageCount }] = replica.pragma("page_count");
 		const [{ freelist_count: freelistCount }] = replica.pragma("freelist_count");
 		const dbSize = (pageCount * pageSize / MB).toFixed(2);
@@ -80,7 +81,8 @@ async function prepare(lc, { file, vacuumIntervalHours }, walMode, mode) {
 	}
 	return {
 		file,
-		walMode
+		walMode,
+		pageSize
 	};
 }
 async function setJournalMode(lc, replica, mode) {

package/out/zero-cache/src/workers/replicator.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/workers/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {ReplicaOptions} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {deleteChangeLogDB} from '../services/replicator/change-log-db.ts';\nimport {Notifier} from '../services/replicator/notifier.ts';\nimport type {\n ReplicaState,\n ReplicaStateNotifier,\n Replicator,\n} from '../services/replicator/replicator.ts';\nimport {\n getAscendingEvents,\n recordEvent,\n} from '../services/replicator/schema/replication-state.ts';\nimport {\n applyPragmas,\n type PragmaConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport type {Worker} from '../types/processes.ts';\n\nexport const replicaFileModeSchema = v.literalUnion(\n 'serving',\n 'serving-copy',\n 'backup',\n);\n\nexport type ReplicaFileMode = v.Infer<typeof replicaFileModeSchema>;\n\nexport type WalMode = 'wal' \| 'wal2';\n\nexport function replicaFileName(replicaFile: string, mode: ReplicaFileMode) {\n return mode === 'serving-copy' ? `${replicaFile}-serving-copy` : replicaFile;\n}\n\n/*\n Whether a replicator worker deletes the change-log file beside its replica at\n * startup.\n \n The change log belongs to the change-streamer, so this only ever cleans up a\n * file that nothing is writing. The predicate is deliberately the config flag\n * and nothing else: keying it on anything derived per replica -- a file mode, or\n * the retired `logsChangeStream` -- would make every replicator start unlink the\n * change-streamer's live log, because no replicator writes it any more. Paths\n * coincide whenever `fileMode !== 'serving-copy'`, so that hazard is real in the\n * shipped no-`backupURL` configuration.\n \n Deleting is safe when nothing writes the log: it is excluded from the\n * litestream backup, only the writer purges it, and re-enabling the writer\n * reseeds at the resume watermark.\n /\nexport function replicatorDeletesStaleChangeLog(\n sqliteChangeLogMode: string,\n): boolean {\n return sqliteChangeLogMode === 'off';\n}\n\n/\n The `mode=off` cleanup itself: deletes the change-log file beside `dbPath`\n * when {@link replicatorDeletesStaleChangeLog} says nothing writes it, and\n * reports whether it did. The decision and the delete live in one function so\n * that the guard test covers the call the replicator actually makes, not just\n * the predicate.\n /\nexport function deleteStaleChangeLog(\n sqliteChangeLogMode: string,\n dbPath: string,\n): boolean {\n if (!replicatorDeletesStaleChangeLog(sqliteChangeLogMode)) {\n return false;\n }\n deleteChangeLogDB(dbPath);\n return true;\n}\n\nconst MILLIS_PER_HOUR = 1000 60 * 60;\nconst MB = 1024 * 1024;\n\nasync function prepare(\n lc: LogContext,\n {file, vacuumIntervalHours}: ReplicaOptions,\n walMode: WalMode,\n mode: ReplicaFileMode,\n): Promise<{file: string; walMode: WalMode}> {\n // Perform any upgrades to the replica in case the backup is an\n // earlier version.\n await upgradeReplica(lc, `${mode}-replica`, file);\n\n const replica = new Database(lc, file);\n try {\n // Start by folding any (e.g. restored) WAL(2) files into the main db.\n await setJournalMode(lc, replica, 'delete');\n\n const [{page_size: pageSize}] = replica.pragma<{page_size: number}>(\n 'page_size',\n );\n const [{page_count: pageCount}] = replica.pragma<{page_count: number}>(\n 'page_count',\n );\n const [{freelist_count: freelistCount}] = replica.pragma<{\n freelist_count: number;\n }>('freelist_count');\n\n const dbSize = ((pageCount * pageSize) / MB).toFixed(2);\n const freelistSize = ((freelistCount * pageSize) / MB).toFixed(2);\n\n // TODO: Consider adding a freelist size or ratio based vacuum trigger.\n lc.info?.(`Size of db ${file}: ${dbSize} MB (${freelistSize} MB freeable)`);\n\n // Check for the VACUUM threshold.\n const events = getAscendingEvents(replica);\n lc.debug?.(`Runtime events for db ${file}`, {events});\n if (vacuumIntervalHours !== undefined) {\n const millisSinceLastEvent =\n Date.now() - (events.at(-1)?.timestamp.getTime() ?? 0);\n if (millisSinceLastEvent / MILLIS_PER_HOUR > vacuumIntervalHours) {\n lc.info?.(`Performing maintenance cleanup on ${file}`);\n const t0 = performance.now();\n replica.unsafeMode(true);\n replica.pragma('journal_mode = OFF');\n replica.exec('VACUUM');\n recordEvent(replica, 'vacuum');\n replica.unsafeMode(false);\n const t1 = performance.now();\n lc.info?.(`VACUUM completed (${t1 - t0} ms)`);\n }\n }\n\n await setJournalMode(lc, replica, walMode);\n\n const pragmas = getPragmaConfig(mode);\n applyPragmas(replica, pragmas);\n\n replica.pragma('optimize = 0x10002');\n lc.info?.(`optimized ${file}`);\n } finally {\n replica.close();\n }\n return {file, walMode};\n}\n\n// Setting the journal_mode requires an exclusive lock on the replica.\n// Add resilience against random replica reads (for stats, etc.) by\n// retrying if the database is locked. Note that the busy_timeout doesn't\n// work here.\nasync function setJournalMode(\n lc: LogContext,\n replica: Database,\n mode: 'delete' \| 'wal' \| 'wal2',\n) {\n lc.info?.(`setting ${replica.name} to ${mode} mode`);\n let err: unknown;\n for (let i = 0; i < 5; i++) {\n try {\n replica.pragma(`journal_mode = ${mode}`);\n return;\n } catch (e) {\n lc.warn?.(`error setting journal_mode to ${mode} (attempt ${i + 1})`, e);\n err = e;\n }\n await sleep(500);\n }\n throw err;\n}\n\n/*\n Returns the PragmaConfig for a given replica file mode.\n * This is used by both the main thread (setupReplica) and\n * the write worker thread to apply the same pragma settings.\n /\nexport function getPragmaConfig(mode: ReplicaFileMode): PragmaConfig {\n return {\n busyTimeout: 30000,\n analysisLimit: 1000,\n walAutocheckpoint: mode === 'backup' ? 0 : undefined,\n };\n}\n\nexport function setupReplica(\n lc: LogContext,\n mode: ReplicaFileMode,\n replicaOptions: ReplicaOptions,\n) {\n lc.info?.(`setting up ${mode} replica`);\n\n switch (mode) {\n case 'backup':\n return prepare(lc, replicaOptions, 'wal', mode);\n\n case 'serving-copy': {\n // In 'serving-copy' mode, the original file is being used for 'backup'\n // mode, so we make a copy for servicing sync requests.\n const {file} = replicaOptions;\n const copyLocation = replicaFileName(file, mode);\n deleteLiteDB(copyLocation);\n\n const start = Date.now();\n lc.info?.(`copying ${file} to ${copyLocation}`);\n const replica = new Database(lc, file);\n replica.prepare(`VACUUM INTO ?`).run(copyLocation);\n replica.close();\n lc.info?.(`finished copy (${Date.now() - start} ms)`);\n\n return prepare(lc, {...replicaOptions, file: copyLocation}, 'wal2', mode);\n }\n\n case 'serving':\n return prepare(lc, replicaOptions, 'wal2', mode);\n\n default:\n throw new Error(`Invalid ReplicaMode ${mode}`);\n }\n}\n\nexport function setUpMessageHandlers(\n lc: LogContext,\n replicator: Replicator,\n parent: Worker,\n) {\n handleSubscriptionsFrom(lc, parent, replicator);\n}\n\ntype Notification = ['notify', ReplicaState];\n\nexport function handleSubscriptionsFrom(\n lc: LogContext,\n subscriber: Worker,\n notifier: ReplicaStateNotifier,\n) {\n subscriber.onMessageType('subscribe', async () => {\n const subscription = notifier.subscribe();\n\n subscriber.on('close', () => {\n lc.debug?.(`closing replication subscription from ${subscriber.pid}`);\n subscription.cancel();\n });\n\n for await (const msg of subscription) {\n try {\n subscriber.send<Notification>(['notify', msg]);\n } catch (e) {\n const log =\n e instanceof Error &&\n 'code' in e &&\n // This can happen in a race condition if the subscribing process\n // is closed before the 'close' message is processed.\n e.code === 'ERR_IPC_CHANNEL_CLOSED'\n ? 'warn'\n : 'error';\n\n lc[log]?.(\n `error sending replicator notification to ${subscriber.pid}: ${String(e)}`,\n e,\n );\n }\n }\n });\n}\n\n/\n Creates a Notifier to relay notifications the notifier of another Worker.\n * This does not send the initial subscription message. Use {@link subscribeTo}\n * to initiate the subscription.\n */\nexport function createNotifierFrom(\n _lc: LogContext,\n source: Worker,\n onNotify?: (state: ReplicaState) => void,\n): Notifier {\n const notifier = new Notifier();\n source.onMessageType<Notification>('notify', msg => {\n onNotify?.(msg);\n void notifier.notifySubscribers(msg);\n });\n return notifier;\n}\n\nexport function subscribeTo(_lc: LogContext, source: Worker) {\n source.send(['subscribe', {}]);\n}\n"],"mappings":";;;;;;;;;;AAwBA,IAAa,wBAAwB,aACnC,WACA,gBACA,QACF;AAMA,SAAgB,gBAAgB,aAAqB,MAAuB;CAC1E,OAAO,SAAS,iBAAiB,GAAG,YAAY,iBAAiB;AACnE;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gCACd,qBACS;CACT,OAAO,wBAAwB;AACjC;;;;;;;;AASA,SAAgB,qBACd,qBACA,QACS;CACT,IAAI,CAAC,gCAAgC,mBAAmB,GACtD,OAAO;CAET,kBAAkB,MAAM;CACxB,OAAO;AACT;AAEA,IAAM,kBAAkB,MAAO,KAAK;AACpC,IAAM,KAAK,OAAO;AAElB,eAAe,QACb,IACA,EAAC,MAAM,uBACP,SACA,MAC2C;CAG3C,MAAM,eAAe,IAAI,GAAG,KAAK,WAAW,IAAI;CAEhD,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;CACrC,IAAI;EAEF,MAAM,eAAe,IAAI,SAAS,QAAQ;EAE1C,MAAM,CAAC,EAAC,WAAW,cAAa,QAAQ,OACtC,WACF;EACA,MAAM,CAAC,EAAC,YAAY,eAAc,QAAQ,OACxC,YACF;EACA,MAAM,CAAC,EAAC,gBAAgB,mBAAkB,QAAQ,OAE/C,gBAAgB;EAEnB,MAAM,UAAW,YAAY,WAAY,IAAI,QAAQ,CAAC;EACtD,MAAM,gBAAiB,gBAAgB,WAAY,IAAI,QAAQ,CAAC;EAGhE,GAAG,OAAO,cAAc,KAAK,IAAI,OAAO,OAAO,aAAa,cAAc;EAG1E,MAAM,SAAS,mBAAmB,OAAO;EACzC,GAAG,QAAQ,yBAAyB,QAAQ,EAAC,OAAM,CAAC;EACpD,IAAI,wBAAwB,KAAA;QAExB,KAAK,IAAI,KAAK,OAAO,GAAG,EAAE,GAAG,UAAU,QAAQ,KAAK,MAC3B,kBAAkB,qBAAqB;IAChE,GAAG,OAAO,qCAAqC,MAAM;IACrD,MAAM,KAAK,YAAY,IAAI;IAC3B,QAAQ,WAAW,IAAI;IACvB,QAAQ,OAAO,oBAAoB;IACnC,QAAQ,KAAK,QAAQ;IACrB,YAAY,SAAS,QAAQ;IAC7B,QAAQ,WAAW,KAAK;IACxB,MAAM,KAAK,YAAY,IAAI;IAC3B,GAAG,OAAO,qBAAqB,KAAK,GAAG,KAAK;GAC9C;;EAGF,MAAM,eAAe,IAAI,SAAS,OAAO;EAGzC,aAAa,SADG,gBAAgB,IACV,CAAO;EAE7B,QAAQ,OAAO,oBAAoB;EACnC,GAAG,OAAO,aAAa,MAAM;CAC/B,UAAU;EACR,QAAQ,MAAM;CAChB;CACA,OAAO;EAAC;EAAM;CAAO;AACvB;AAMA,eAAe,eACb,IACA,SACA,MACA;CACA,GAAG,OAAO,WAAW,QAAQ,KAAK,MAAM,KAAK,MAAM;CACnD,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI;GACF,QAAQ,OAAO,kBAAkB,MAAM;GACvC;EACF,SAAS,GAAG;GACV,GAAG,OAAO,iCAAiC,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE,MAAM;EACR;EACA,MAAM,MAAM,GAAG;CACjB;CACA,MAAM;AACR;;;;;;AAOA,SAAgB,gBAAgB,MAAqC;CACnE,OAAO;EACL,aAAa;EACb,eAAe;EACf,mBAAmB,SAAS,WAAW,IAAI,KAAA;CAC7C;AACF;AAEA,SAAgB,aACd,IACA,MACA,gBACA;CACA,GAAG,OAAO,cAAc,KAAK,SAAS;CAEtC,QAAQ,MAAR;EACE,KAAK,UACH,OAAO,QAAQ,IAAI,gBAAgB,OAAO,IAAI;EAEhD,KAAK,gBAAgB;GAGnB,MAAM,EAAC,SAAQ;GACf,MAAM,eAAe,gBAAgB,MAAM,IAAI;GAC/C,aAAa,YAAY;GAEzB,MAAM,QAAQ,KAAK,IAAI;GACvB,GAAG,OAAO,WAAW,KAAK,MAAM,cAAc;GAC9C,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;GACrC,QAAQ,QAAQ,eAAe,EAAE,IAAI,YAAY;GACjD,QAAQ,MAAM;GACd,GAAG,OAAO,kBAAkB,KAAK,IAAI,IAAI,MAAM,KAAK;GAEpD,OAAO,QAAQ,IAAI;IAAC,GAAG;IAAgB,MAAM;GAAY,GAAG,QAAQ,IAAI;EAC1E;EAEA,KAAK,WACH,OAAO,QAAQ,IAAI,gBAAgB,QAAQ,IAAI;EAEjD,SACE,MAAM,IAAI,MAAM,uBAAuB,MAAM;CACjD;AACF;AAEA,SAAgB,qBACd,IACA,YACA,QACA;CACA,wBAAwB,IAAI,QAAQ,UAAU;AAChD;AAIA,SAAgB,wBACd,IACA,YACA,UACA;CACA,WAAW,cAAc,aAAa,YAAY;EAChD,MAAM,eAAe,SAAS,UAAU;EAExC,WAAW,GAAG,eAAe;GAC3B,GAAG,QAAQ,yCAAyC,WAAW,KAAK;GACpE,aAAa,OAAO;EACtB,CAAC;EAED,WAAW,MAAM,OAAO,cACtB,IAAI;GACF,WAAW,KAAmB,CAAC,UAAU,GAAG,CAAC;EAC/C,SAAS,GAAG;GAUV,GARE,aAAa,SACb,UAAU,KAGV,EAAE,SAAS,2BACP,SACA,WAGJ,4CAA4C,WAAW,IAAI,IAAI,OAAO,CAAC,KACvE,CACF;EACF;CAEJ,CAAC;AACH;;;;;;AAOA,SAAgB,mBACd,KACA,QACA,UACU;CACV,MAAM,WAAW,IAAI,SAAS;CAC9B,OAAO,cAA4B,WAAU,QAAO;EAClD,WAAW,GAAG;EACd,SAAc,kBAAkB,GAAG;CACrC,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,YAAY,KAAiB,QAAgB;CAC3D,OAAO,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;AAC/B"}
1	+ {"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/workers/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {ReplicaOptions} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {deleteChangeLogDB} from '../services/replicator/change-log-db.ts';\nimport {Notifier} from '../services/replicator/notifier.ts';\nimport type {\n ReplicaState,\n ReplicaStateNotifier,\n Replicator,\n} from '../services/replicator/replicator.ts';\nimport {\n getAscendingEvents,\n recordEvent,\n} from '../services/replicator/schema/replication-state.ts';\nimport {\n applyPragmas,\n type PragmaConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport type {Worker} from '../types/processes.ts';\n\nexport const replicaFileModeSchema = v.literalUnion(\n 'serving',\n 'serving-copy',\n 'backup',\n);\n\nexport type ReplicaFileMode = v.Infer<typeof replicaFileModeSchema>;\n\nexport type WalMode = 'wal' \| 'wal2';\n\nexport function replicaFileName(replicaFile: string, mode: ReplicaFileMode) {\n return mode === 'serving-copy' ? `${replicaFile}-serving-copy` : replicaFile;\n}\n\n/*\n Whether a replicator worker deletes the change-log file beside its replica at\n * startup.\n \n The change log belongs to the change-streamer, so this only ever cleans up a\n * file that nothing is writing. The predicate is deliberately the config flag\n * and nothing else: keying it on anything derived per replica -- a file mode, or\n * the retired `logsChangeStream` -- would make every replicator start unlink the\n * change-streamer's live log, because no replicator writes it any more. Paths\n * coincide whenever `fileMode !== 'serving-copy'`, so that hazard is real in the\n * shipped no-`backupURL` configuration.\n \n Deleting is safe when nothing writes the log: it is excluded from the\n * litestream backup, only the writer purges it, and re-enabling the writer\n * reseeds at the resume watermark.\n /\nexport function replicatorDeletesStaleChangeLog(\n sqliteChangeLogMode: string,\n): boolean {\n return sqliteChangeLogMode === 'off';\n}\n\n/\n The `mode=off` cleanup itself: deletes the change-log file beside `dbPath`\n * when {@link replicatorDeletesStaleChangeLog} says nothing writes it, and\n * reports whether it did. The decision and the delete live in one function so\n * that the guard test covers the call the replicator actually makes, not just\n * the predicate.\n /\nexport function deleteStaleChangeLog(\n sqliteChangeLogMode: string,\n dbPath: string,\n): boolean {\n if (!replicatorDeletesStaleChangeLog(sqliteChangeLogMode)) {\n return false;\n }\n deleteChangeLogDB(dbPath);\n return true;\n}\n\nconst MILLIS_PER_HOUR = 1000 60 * 60;\nconst MB = 1024 * 1024;\n\nasync function prepare(\n lc: LogContext,\n {file, vacuumIntervalHours}: ReplicaOptions,\n walMode: WalMode,\n mode: ReplicaFileMode,\n): Promise<{file: string; walMode: WalMode; pageSize: number}> {\n // Perform any upgrades to the replica in case the backup is an\n // earlier version.\n await upgradeReplica(lc, `${mode}-replica`, file);\n\n const replica = new Database(lc, file);\n let pageSize: number;\n try {\n // Start by folding any (e.g. restored) WAL(2) files into the main db.\n await setJournalMode(lc, replica, 'delete');\n\n [{page_size: pageSize}] = replica.pragma<{page_size: number}>('page_size');\n const [{page_count: pageCount}] = replica.pragma<{page_count: number}>(\n 'page_count',\n );\n const [{freelist_count: freelistCount}] = replica.pragma<{\n freelist_count: number;\n }>('freelist_count');\n\n const dbSize = ((pageCount * pageSize) / MB).toFixed(2);\n const freelistSize = ((freelistCount * pageSize) / MB).toFixed(2);\n\n // TODO: Consider adding a freelist size or ratio based vacuum trigger.\n lc.info?.(`Size of db ${file}: ${dbSize} MB (${freelistSize} MB freeable)`);\n\n // Check for the VACUUM threshold.\n const events = getAscendingEvents(replica);\n lc.debug?.(`Runtime events for db ${file}`, {events});\n if (vacuumIntervalHours !== undefined) {\n const millisSinceLastEvent =\n Date.now() - (events.at(-1)?.timestamp.getTime() ?? 0);\n if (millisSinceLastEvent / MILLIS_PER_HOUR > vacuumIntervalHours) {\n lc.info?.(`Performing maintenance cleanup on ${file}`);\n const t0 = performance.now();\n replica.unsafeMode(true);\n replica.pragma('journal_mode = OFF');\n replica.exec('VACUUM');\n recordEvent(replica, 'vacuum');\n replica.unsafeMode(false);\n const t1 = performance.now();\n lc.info?.(`VACUUM completed (${t1 - t0} ms)`);\n }\n }\n\n await setJournalMode(lc, replica, walMode);\n\n const pragmas = getPragmaConfig(mode);\n applyPragmas(replica, pragmas);\n\n replica.pragma('optimize = 0x10002');\n lc.info?.(`optimized ${file}`);\n } finally {\n replica.close();\n }\n return {file, walMode, pageSize};\n}\n\n// Setting the journal_mode requires an exclusive lock on the replica.\n// Add resilience against random replica reads (for stats, etc.) by\n// retrying if the database is locked. Note that the busy_timeout doesn't\n// work here.\nasync function setJournalMode(\n lc: LogContext,\n replica: Database,\n mode: 'delete' \| 'wal' \| 'wal2',\n) {\n lc.info?.(`setting ${replica.name} to ${mode} mode`);\n let err: unknown;\n for (let i = 0; i < 5; i++) {\n try {\n replica.pragma(`journal_mode = ${mode}`);\n return;\n } catch (e) {\n lc.warn?.(`error setting journal_mode to ${mode} (attempt ${i + 1})`, e);\n err = e;\n }\n await sleep(500);\n }\n throw err;\n}\n\n/*\n Returns the PragmaConfig for a given replica file mode.\n * This is used by both the main thread (setupReplica) and\n * the write worker thread to apply the same pragma settings.\n /\nexport function getPragmaConfig(mode: ReplicaFileMode): PragmaConfig {\n return {\n busyTimeout: 30000,\n analysisLimit: 1000,\n walAutocheckpoint: mode === 'backup' ? 0 : undefined,\n };\n}\n\nexport function setupReplica(\n lc: LogContext,\n mode: ReplicaFileMode,\n replicaOptions: ReplicaOptions,\n) {\n lc.info?.(`setting up ${mode} replica`);\n\n switch (mode) {\n case 'backup':\n return prepare(lc, replicaOptions, 'wal', mode);\n\n case 'serving-copy': {\n // In 'serving-copy' mode, the original file is being used for 'backup'\n // mode, so we make a copy for servicing sync requests.\n const {file} = replicaOptions;\n const copyLocation = replicaFileName(file, mode);\n deleteLiteDB(copyLocation);\n\n const start = Date.now();\n lc.info?.(`copying ${file} to ${copyLocation}`);\n const replica = new Database(lc, file);\n replica.prepare(`VACUUM INTO ?`).run(copyLocation);\n replica.close();\n lc.info?.(`finished copy (${Date.now() - start} ms)`);\n\n return prepare(lc, {...replicaOptions, file: copyLocation}, 'wal2', mode);\n }\n\n case 'serving':\n return prepare(lc, replicaOptions, 'wal2', mode);\n\n default:\n throw new Error(`Invalid ReplicaMode ${mode}`);\n }\n}\n\nexport function setUpMessageHandlers(\n lc: LogContext,\n replicator: Replicator,\n parent: Worker,\n) {\n handleSubscriptionsFrom(lc, parent, replicator);\n}\n\ntype Notification = ['notify', ReplicaState];\n\nexport function handleSubscriptionsFrom(\n lc: LogContext,\n subscriber: Worker,\n notifier: ReplicaStateNotifier,\n) {\n subscriber.onMessageType('subscribe', async () => {\n const subscription = notifier.subscribe();\n\n subscriber.on('close', () => {\n lc.debug?.(`closing replication subscription from ${subscriber.pid}`);\n subscription.cancel();\n });\n\n for await (const msg of subscription) {\n try {\n subscriber.send<Notification>(['notify', msg]);\n } catch (e) {\n const log =\n e instanceof Error &&\n 'code' in e &&\n // This can happen in a race condition if the subscribing process\n // is closed before the 'close' message is processed.\n e.code === 'ERR_IPC_CHANNEL_CLOSED'\n ? 'warn'\n : 'error';\n\n lc[log]?.(\n `error sending replicator notification to ${subscriber.pid}: ${String(e)}`,\n e,\n );\n }\n }\n });\n}\n\n/\n Creates a Notifier to relay notifications the notifier of another Worker.\n * This does not send the initial subscription message. Use {@link subscribeTo}\n * to initiate the subscription.\n */\nexport function createNotifierFrom(\n _lc: LogContext,\n source: Worker,\n onNotify?: (state: ReplicaState) => void,\n): Notifier {\n const notifier = new Notifier();\n source.onMessageType<Notification>('notify', msg => {\n onNotify?.(msg);\n void notifier.notifySubscribers(msg);\n });\n return notifier;\n}\n\nexport function subscribeTo(_lc: LogContext, source: Worker) {\n source.send(['subscribe', {}]);\n}\n"],"mappings":";;;;;;;;;;AAwBA,IAAa,wBAAwB,aACnC,WACA,gBACA,QACF;AAMA,SAAgB,gBAAgB,aAAqB,MAAuB;CAC1E,OAAO,SAAS,iBAAiB,GAAG,YAAY,iBAAiB;AACnE;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gCACd,qBACS;CACT,OAAO,wBAAwB;AACjC;;;;;;;;AASA,SAAgB,qBACd,qBACA,QACS;CACT,IAAI,CAAC,gCAAgC,mBAAmB,GACtD,OAAO;CAET,kBAAkB,MAAM;CACxB,OAAO;AACT;AAEA,IAAM,kBAAkB,MAAO,KAAK;AACpC,IAAM,KAAK,OAAO;AAElB,eAAe,QACb,IACA,EAAC,MAAM,uBACP,SACA,MAC6D;CAG7D,MAAM,eAAe,IAAI,GAAG,KAAK,WAAW,IAAI;CAEhD,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;CACrC,IAAI;CACJ,IAAI;EAEF,MAAM,eAAe,IAAI,SAAS,QAAQ;EAE1C,CAAC,CAAC,WAAW,aAAa,QAAQ,OAA4B,WAAW;EACzE,MAAM,CAAC,EAAC,YAAY,eAAc,QAAQ,OACxC,YACF;EACA,MAAM,CAAC,EAAC,gBAAgB,mBAAkB,QAAQ,OAE/C,gBAAgB;EAEnB,MAAM,UAAW,YAAY,WAAY,IAAI,QAAQ,CAAC;EACtD,MAAM,gBAAiB,gBAAgB,WAAY,IAAI,QAAQ,CAAC;EAGhE,GAAG,OAAO,cAAc,KAAK,IAAI,OAAO,OAAO,aAAa,cAAc;EAG1E,MAAM,SAAS,mBAAmB,OAAO;EACzC,GAAG,QAAQ,yBAAyB,QAAQ,EAAC,OAAM,CAAC;EACpD,IAAI,wBAAwB,KAAA;QAExB,KAAK,IAAI,KAAK,OAAO,GAAG,EAAE,GAAG,UAAU,QAAQ,KAAK,MAC3B,kBAAkB,qBAAqB;IAChE,GAAG,OAAO,qCAAqC,MAAM;IACrD,MAAM,KAAK,YAAY,IAAI;IAC3B,QAAQ,WAAW,IAAI;IACvB,QAAQ,OAAO,oBAAoB;IACnC,QAAQ,KAAK,QAAQ;IACrB,YAAY,SAAS,QAAQ;IAC7B,QAAQ,WAAW,KAAK;IACxB,MAAM,KAAK,YAAY,IAAI;IAC3B,GAAG,OAAO,qBAAqB,KAAK,GAAG,KAAK;GAC9C;;EAGF,MAAM,eAAe,IAAI,SAAS,OAAO;EAGzC,aAAa,SADG,gBAAgB,IACV,CAAO;EAE7B,QAAQ,OAAO,oBAAoB;EACnC,GAAG,OAAO,aAAa,MAAM;CAC/B,UAAU;EACR,QAAQ,MAAM;CAChB;CACA,OAAO;EAAC;EAAM;EAAS;CAAQ;AACjC;AAMA,eAAe,eACb,IACA,SACA,MACA;CACA,GAAG,OAAO,WAAW,QAAQ,KAAK,MAAM,KAAK,MAAM;CACnD,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI;GACF,QAAQ,OAAO,kBAAkB,MAAM;GACvC;EACF,SAAS,GAAG;GACV,GAAG,OAAO,iCAAiC,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE,MAAM;EACR;EACA,MAAM,MAAM,GAAG;CACjB;CACA,MAAM;AACR;;;;;;AAOA,SAAgB,gBAAgB,MAAqC;CACnE,OAAO;EACL,aAAa;EACb,eAAe;EACf,mBAAmB,SAAS,WAAW,IAAI,KAAA;CAC7C;AACF;AAEA,SAAgB,aACd,IACA,MACA,gBACA;CACA,GAAG,OAAO,cAAc,KAAK,SAAS;CAEtC,QAAQ,MAAR;EACE,KAAK,UACH,OAAO,QAAQ,IAAI,gBAAgB,OAAO,IAAI;EAEhD,KAAK,gBAAgB;GAGnB,MAAM,EAAC,SAAQ;GACf,MAAM,eAAe,gBAAgB,MAAM,IAAI;GAC/C,aAAa,YAAY;GAEzB,MAAM,QAAQ,KAAK,IAAI;GACvB,GAAG,OAAO,WAAW,KAAK,MAAM,cAAc;GAC9C,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;GACrC,QAAQ,QAAQ,eAAe,EAAE,IAAI,YAAY;GACjD,QAAQ,MAAM;GACd,GAAG,OAAO,kBAAkB,KAAK,IAAI,IAAI,MAAM,KAAK;GAEpD,OAAO,QAAQ,IAAI;IAAC,GAAG;IAAgB,MAAM;GAAY,GAAG,QAAQ,IAAI;EAC1E;EAEA,KAAK,WACH,OAAO,QAAQ,IAAI,gBAAgB,QAAQ,IAAI;EAEjD,SACE,MAAM,IAAI,MAAM,uBAAuB,MAAM;CACjD;AACF;AAEA,SAAgB,qBACd,IACA,YACA,QACA;CACA,wBAAwB,IAAI,QAAQ,UAAU;AAChD;AAIA,SAAgB,wBACd,IACA,YACA,UACA;CACA,WAAW,cAAc,aAAa,YAAY;EAChD,MAAM,eAAe,SAAS,UAAU;EAExC,WAAW,GAAG,eAAe;GAC3B,GAAG,QAAQ,yCAAyC,WAAW,KAAK;GACpE,aAAa,OAAO;EACtB,CAAC;EAED,WAAW,MAAM,OAAO,cACtB,IAAI;GACF,WAAW,KAAmB,CAAC,UAAU,GAAG,CAAC;EAC/C,SAAS,GAAG;GAUV,GARE,aAAa,SACb,UAAU,KAGV,EAAE,SAAS,2BACP,SACA,WAGJ,4CAA4C,WAAW,IAAI,IAAI,OAAO,CAAC,KACvE,CACF;EACF;CAEJ,CAAC;AACH;;;;;;AAOA,SAAgB,mBACd,KACA,QACA,UACU;CACV,MAAM,WAAW,IAAI,SAAS;CAC9B,OAAO,cAA4B,WAAU,QAAO;EAClD,WAAW,GAAG;EACd,SAAc,kBAAkB,GAAG;CACrC,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,YAAY,KAAiB,QAAgB;CAC3D,OAAO,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;AAC/B"}

package/out/zero-client/src/client/context.d.ts CHANGED Viewed

@@ -39,6 +39,21 @@ export declare class ZeroContext extends QueryDelegateBase {
     applyFiltersAnyway?: boolean | undefined;
     debug?: DebugDelegate | undefined;
     getSource(name: string): Source | undefined;
+    get pipelinesReady(): boolean;
+    onPipelinesReady(cb: () => void): () => void;
+    /**
+     * Stop building pipelines for materialized queries until
+     * {@link markPipelinesReady} is called. Views materialized in the meantime
+     * are empty and `unknown`, exactly as they would be over empty sources.
+     */
+    deferPipelines(): void;
+    /**
+     * Build and hydrate every pipeline deferred since {@link deferPipelines},
+     * in materialization order, as a single view-update batch. A pipeline that
+     * fails to build is logged and its view reports an error; the rest are
+     * still built.
+     */
+    markPipelinesReady(): void;
     mapAst(ast: AST): AST;
     decorateSourceInput(input: SourceInput, queryID: string): Input;
     onTransactionCommit(cb: CommitListener): () => void;

package/out/zero-client/src/client/context.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../../../zero-client/src/client/context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,uCAAuC,CAAC;AACvE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,iCAAiC,CAAC;AAE1D,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,4CAA4C,CAAC;AAC9E,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,kCAAkC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAE,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAExE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAChF,OAAO,EAAC,iBAAiB,EAAC,MAAM,+CAA+C,CAAC;AAChF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAC,KAAK,eAAe,EAAC,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AACjD,MAAM,MAAM,cAAc,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AAEvD,MAAM,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,iBAAiB,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,iBAAiB,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;AAE3D;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,iBAAiB;;IAShD,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,iBAAiB,EAAE,WAAW,CAAC;IACxC,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAC9C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,IAAI,CAAC;~~IAIvC~~,QAAQ,CAAC,qBAAqB,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,KAAK,IAAI,CAAC;IAE/D;;;OAGG;IACH,QAAQ,CAAC,oBAAoB,SAAS;IAEtC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC;IAEjD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,eAAe,EAC5B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,cAAc,EAC9B,WAAW,EAAE,WAAW,EACxB,iBAAiB,EAAE,iBAAiB,EACpC,iBAAiB,EAAE,MAAM,IAAI,EAC7B,gBAAgB,EAAE,CAAC,gBAAgB,EAAE,MAAM,IAAI,KAAK,IAAI,EACxD,SAAS,EAAE,eAAe,CAAC,WAAW,CAAC,EACvC,qBAAqB,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,KAAK,IAAI,EAatD;IAED,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAEzC,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAElC,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE1C;IAED,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAEpB;IAEQ,mBAAmB,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,KAAK,CAEvE;IAED,mBAAmB,CAAC,EAAE,EAAE,cAAc,GAAG,MAAM,IAAI,CAKlD;IAEQ,gBAAgB,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,GAWnC,CAAC,CACnB;IAED,cAAc,CACZ,YAAY,EAAE,IAAI,GAAG,SAAS,EAC9B,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,WAAW,QASrB;CAkBF"}
1	+ {"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../../../../zero-client/src/client/context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,uCAAuC,CAAC;AACvE,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,iCAAiC,CAAC;AAE1D,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,4CAA4C,CAAC;AAC9E,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,kCAAkC,CAAC;AAC5D,OAAO,KAAK,EAAC,MAAM,EAAE,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAExE,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAChF,OAAO,EAAC,iBAAiB,EAAC,MAAM,+CAA+C,CAAC;AAChF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAC,KAAK,eAAe,EAAC,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AACjD,MAAM,MAAM,cAAc,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;AAEvD,MAAM,MAAM,WAAW,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,iBAAiB,GAAG,YAAY,CAAC,cAAc,CAAC,CAAC;AAC7D,MAAM,MAAM,iBAAiB,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;AAE3D;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,iBAAiB;;IAShD,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC;IAClC,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,iBAAiB,EAAE,WAAW,CAAC;IACxC,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAC9C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,IAAI,CAAC;IAYvC,QAAQ,CAAC,qBAAqB,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,KAAK,IAAI,CAAC;IAE/D;;;OAGG;IACH,QAAQ,CAAC,oBAAoB,SAAS;IAEtC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC;IAEjD,YACE,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,eAAe,EAC5B,QAAQ,EAAE,QAAQ,EAClB,cAAc,EAAE,cAAc,EAC9B,WAAW,EAAE,WAAW,EACxB,iBAAiB,EAAE,iBAAiB,EACpC,iBAAiB,EAAE,MAAM,IAAI,EAC7B,gBAAgB,EAAE,CAAC,gBAAgB,EAAE,MAAM,IAAI,KAAK,IAAI,EACxD,SAAS,EAAE,eAAe,CAAC,WAAW,CAAC,EACvC,qBAAqB,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,KAAK,IAAI,EAatD;IAED,kBAAkB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAEzC,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAElC,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE1C;IAED,IAAa,cAAc,IAAI,OAAO,CAErC;IAEQ,gBAAgB,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CASpD;IAED;;;;OAIG;IACH,cAAc,IAAI,IAAI,CAErB;IAED;;;;;OAKG;IACH,kBAAkB,IAAI,IAAI,CA6BzB;IAED,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAEpB;IAEQ,mBAAmB,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,KAAK,CAEvE;IAED,mBAAmB,CAAC,EAAE,EAAE,cAAc,GAAG,MAAM,IAAI,CAKlD;IAEQ,gBAAgB,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,GAWnC,CAAC,CACnB;IAED,cAAc,CACZ,YAAY,EAAE,IAAI,GAAG,SAAS,EAC9B,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,WAAW,QASrB;CAkBF"}

package/out/zero-client/src/client/context.js CHANGED Viewed

@@ -19,6 +19,8 @@ var ZeroContext = class extends QueryDelegateBase {
 	flushQueryChanges;
 	#batchViewUpdates;
 	#commitListeners = /* @__PURE__ */ new Set();
+	#pipelinesReady = true;
+	#pendingPipelines = /* @__PURE__ */ new Set();
 	assertValidRunOptions;
 	/**
 	* Client-side queries start out as "unknown" and are then updated to
@@ -44,6 +46,48 @@ var ZeroContext = class extends QueryDelegateBase {
 	getSource(name) {
 		return this.#mainSources.getSource(name);
 	}
+	get pipelinesReady() {
+		return this.#pipelinesReady;
+	}
+	onPipelinesReady(cb) {
+		assert(!this.#pipelinesReady, "onPipelinesReady called while pipelines are ready");
+		this.#pendingPipelines.add(cb);
+		return () => {
+			this.#pendingPipelines.delete(cb);
+		};
+	}
+	/**
+	* Stop building pipelines for materialized queries until
+	* {@link markPipelinesReady} is called. Views materialized in the meantime
+	* are empty and `unknown`, exactly as they would be over empty sources.
+	*/
+	deferPipelines() {
+		this.#pipelinesReady = false;
+	}
+	/**
+	* Build and hydrate every pipeline deferred since {@link deferPipelines},
+	* in materialization order, as a single view-update batch. A pipeline that
+	* fails to build is logged and its view reports an error; the rest are
+	* still built.
+	*/
+	markPipelinesReady() {
+		if (this.#pipelinesReady) return;
+		this.#pipelinesReady = true;
+		if (this.#pendingPipelines.size === 0) return;
+		const pending = [...this.#pendingPipelines];
+		this.#pendingPipelines.clear();
+		this.batchViewUpdates(() => {
+			try {
+				for (const attach of pending) try {
+					attach();
+				} catch (e) {
+					this.#lc.error?.(Internal, "Failed to build a deferred query pipeline", e);
+				}
+			} finally {
+				this.#endTransaction();
+			}
+		});
+	}
 	mapAst(ast) {
 		return ast;
 	}

package/out/zero-client/src/client/context.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"context.js","names":["#lc","#mainSources","#batchViewUpdates","#commitListeners","#endTransaction"],"sources":["../../../../../zero-client/src/client/context.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NoIndexDiff} from '../../../replicache/src/btree/node.ts';\nimport type {Hash} from '../../../replicache/src/hash.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport type {DebugDelegate} from '../../../zql/src/builder/debug-delegate.ts';\nimport type {Input} from '../../../zql/src/ivm/operator.ts';\nimport type {Source, SourceInput} from '../../../zql/src/ivm/source.ts';\nimport {MeasurePushOperator} from '../../../zql/src/query/measure-push-operator.ts';\nimport type {MetricsDelegate} from '../../../zql/src/query/metrics-delegate.ts';\nimport {QueryDelegateBase} from '../../../zql/src/query/query-delegate-base.ts';\nimport type {CommitListener} from '../../../zql/src/query/query-delegate.ts';\nimport type {RunOptions} from '../../../zql/src/query/query.ts';\nimport {type IVMSourceBranch} from './ivm-branch.ts';\nimport type {QueryManager} from './query-manager.ts';\n\nexport type AddQuery = QueryManager['addLegacy'];\nexport type AddCustomQuery = QueryManager['addCustom'];\n\nexport type UpdateQuery = QueryManager['updateLegacy'];\nexport type UpdateCustomQuery = QueryManager['updateCustom'];\nexport type FlushQueryChanges = QueryManager['flushBatch'];\n\n/*\n ZeroContext glues together zql and Replicache. It listens to changes in\n * Replicache data and pushes them into IVM and on tells the server about new\n * queries.\n /\nexport class ZeroContext extends QueryDelegateBase {\n readonly #lc: LogContext;\n\n // It is a bummer to have to maintain separate MemorySources here and copy the\n // data in from the Replicache db. But we want the data to be accessible via\n // pipelines synchronously* and the core Replicache infra is all async. So\n // that needs to be fixed.\n readonly #mainSources: IVMSourceBranch;\n\n readonly addServerQuery: AddQuery;\n readonly addCustomQuery: AddCustomQuery;\n readonly updateServerQuery: UpdateQuery;\n readonly updateCustomQuery: UpdateCustomQuery;\n readonly flushQueryChanges: () => void;\n readonly #batchViewUpdates: (applyViewUpdates: () => void) => void;\n readonly #commitListeners: Set<CommitListener> = new Set();\n\n readonly assertValidRunOptions: (options?: RunOptions) => void;\n\n /*\n Client-side queries start out as \"unknown\" and are then updated to\n * \"complete\" once the server has sent back the query result.\n */\n readonly defaultQueryComplete = false;\n\n readonly addMetric: MetricsDelegate['addMetric'];\n\n constructor(\n lc: LogContext,\n mainSources: IVMSourceBranch,\n addQuery: AddQuery,\n addCustomQuery: AddCustomQuery,\n updateQuery: UpdateQuery,\n updateCustomQuery: UpdateCustomQuery,\n flushQueryChanges: () => void,\n batchViewUpdates: (applyViewUpdates: () => void) => void,\n addMetric: MetricsDelegate['addMetric'],\n assertValidRunOptions: (options?: RunOptions) => void,\n ) {\n super();\n this.#lc = lc;\n this.#mainSources = mainSources;\n this.addServerQuery = addQuery;\n this.updateServerQuery = updateQuery;\n this.updateCustomQuery = updateCustomQuery;\n this.#batchViewUpdates = batchViewUpdates;\n this.assertValidRunOptions = assertValidRunOptions;\n this.addCustomQuery = addCustomQuery;\n this.flushQueryChanges = flushQueryChanges;\n this.addMetric = addMetric;\n }\n\n applyFiltersAnyway?: boolean \| undefined;\n\n debug?: DebugDelegate \| undefined;\n\n getSource(name: string): Source \| undefined {\n return this.#mainSources.getSource(name);\n }\n\n mapAst(ast: AST): AST {\n return ast;\n }\n\n override decorateSourceInput(input: SourceInput, queryID: string): Input {\n return new MeasurePushOperator(input, queryID, this, 'query-update-client');\n }\n\n onTransactionCommit(cb: CommitListener): () => void {\n this.#commitListeners.add(cb);\n return () => {\n this.#commitListeners.delete(cb);\n };\n }\n\n override batchViewUpdates<T>(applyViewUpdates: () => T) {\n let result: T \| undefined;\n let viewChangesPerformed = false;\n this.#batchViewUpdates(() => {\n result = applyViewUpdates();\n viewChangesPerformed = true;\n });\n assert(\n viewChangesPerformed,\n 'batchViewUpdates must call applyViewUpdates synchronously.',\n );\n return result as T;\n }\n\n processChanges(\n expectedHead: Hash \| undefined,\n newHead: Hash,\n changes: NoIndexDiff,\n ) {\n this.batchViewUpdates(() => {\n try {\n this.#mainSources.advance(expectedHead, newHead, changes);\n } finally {\n this.#endTransaction();\n }\n });\n }\n\n #endTransaction() {\n for (const listener of this.#commitListeners) {\n try {\n listener();\n } catch (e) {\n // We should not fatal the inner-workings of Zero due to the user's application\n // code throwing an error.\n // Hence we wrap notifications in a try-catch block.\n this.#lc.error?.(\n ErrorKind.Internal,\n 'Failed notifying a commit listener of IVM updates',\n e,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;AA6BA,IAAa,cAAb,cAAiC,kBAAkB;CACjD;CAMA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA,mCAAiD,IAAI,IAAI;~~CAEzD~~;;;;;CAMA,uBAAgC;CAEhC;CAEA,YACE,IACA,aACA,UACA,gBACA,aACA,mBACA,mBACA,kBACA,WACA,uBACA;EACA,MAAM;EACN,KAAKA,MAAM;EACX,KAAKC,eAAe;EACpB,KAAK,iBAAiB;EACtB,KAAK,oBAAoB;EACzB,KAAK,oBAAoB;EACzB,KAAKC,oBAAoB;EACzB,KAAK,wBAAwB;EAC7B,KAAK,iBAAiB;EACtB,KAAK,oBAAoB;EACzB,KAAK,YAAY;CACnB;CAEA;CAEA;CAEA,UAAU,MAAkC;EAC1C,OAAO,KAAKD,aAAa,UAAU,IAAI;CACzC;CAEA,OAAO,KAAe;EACpB,OAAO;CACT;CAEA,oBAA6B,OAAoB,SAAwB;EACvE,OAAO,IAAI,oBAAoB,OAAO,SAAS,MAAM,qBAAqB;CAC5E;CAEA,oBAAoB,IAAgC;EAClD,~~KAAKE~~,iBAAiB,IAAI,EAAE;EAC5B,aAAa;GACX,KAAKA,iBAAiB,OAAO,EAAE;EACjC;CACF;CAEA,iBAA6B,kBAA2B;EACtD,IAAI;EACJ,IAAI,uBAAuB;EAC3B,KAAKD,wBAAwB;GAC3B,SAAS,iBAAiB;GAC1B,uBAAuB;EACzB,CAAC;EACD,OACE,sBACA,4DACF;EACA,OAAO;CACT;CAEA,eACE,cACA,SACA,SACA;EACA,KAAK,uBAAuB;GAC1B,IAAI;IACF,KAAKD,aAAa,QAAQ,cAAc,SAAS,OAAO;GAC1D,UAAU;IACR,~~KAAKG~~,gBAAgB;GACvB;EACF,CAAC;CACH;CAEA,kBAAkB;EAChB,KAAK,MAAM,YAAY,~~KAAKD~~,kBAC1B,IAAI;GACF,SAAS;EACX,SAAS,GAAG;GAIV,KAAKH,IAAI,QACP,UACA,qDACA,CACF;EACF;CAEJ;AACF"}
1	+ {"version":3,"file":"context.js","names":["#lc","#mainSources","#batchViewUpdates","#commitListeners","#pendingPipelines","#pipelinesReady","#endTransaction"],"sources":["../../../../../zero-client/src/client/context.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NoIndexDiff} from '../../../replicache/src/btree/node.ts';\nimport type {Hash} from '../../../replicache/src/hash.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport {ErrorKind} from '../../../zero-protocol/src/error-kind.ts';\nimport type {DebugDelegate} from '../../../zql/src/builder/debug-delegate.ts';\nimport type {Input} from '../../../zql/src/ivm/operator.ts';\nimport type {Source, SourceInput} from '../../../zql/src/ivm/source.ts';\nimport {MeasurePushOperator} from '../../../zql/src/query/measure-push-operator.ts';\nimport type {MetricsDelegate} from '../../../zql/src/query/metrics-delegate.ts';\nimport {QueryDelegateBase} from '../../../zql/src/query/query-delegate-base.ts';\nimport type {CommitListener} from '../../../zql/src/query/query-delegate.ts';\nimport type {RunOptions} from '../../../zql/src/query/query.ts';\nimport {type IVMSourceBranch} from './ivm-branch.ts';\nimport type {QueryManager} from './query-manager.ts';\n\nexport type AddQuery = QueryManager['addLegacy'];\nexport type AddCustomQuery = QueryManager['addCustom'];\n\nexport type UpdateQuery = QueryManager['updateLegacy'];\nexport type UpdateCustomQuery = QueryManager['updateCustom'];\nexport type FlushQueryChanges = QueryManager['flushBatch'];\n\n/*\n ZeroContext glues together zql and Replicache. It listens to changes in\n * Replicache data and pushes them into IVM and on tells the server about new\n * queries.\n /\nexport class ZeroContext extends QueryDelegateBase {\n readonly #lc: LogContext;\n\n // It is a bummer to have to maintain separate MemorySources here and copy the\n // data in from the Replicache db. But we want the data to be accessible via\n // pipelines synchronously* and the core Replicache infra is all async. So\n // that needs to be fixed.\n readonly #mainSources: IVMSourceBranch;\n\n readonly addServerQuery: AddQuery;\n readonly addCustomQuery: AddCustomQuery;\n readonly updateServerQuery: UpdateQuery;\n readonly updateCustomQuery: UpdateCustomQuery;\n readonly flushQueryChanges: () => void;\n readonly #batchViewUpdates: (applyViewUpdates: () => void) => void;\n readonly #commitListeners: Set<CommitListener> = new Set();\n\n // Pipeline construction is deferred between `deferPipelines()` and\n // `markPipelinesReady()`. Zero calls the former at construction and the\n // latter once the replica has been loaded into the IVM sources, so cold\n // boot loads the sources once instead of pushing every row through every\n // already-materialized pipeline.\n #pipelinesReady = true;\n readonly #pendingPipelines: Set<() => void> = new Set();\n\n readonly assertValidRunOptions: (options?: RunOptions) => void;\n\n /*\n Client-side queries start out as \"unknown\" and are then updated to\n * \"complete\" once the server has sent back the query result.\n /\n readonly defaultQueryComplete = false;\n\n readonly addMetric: MetricsDelegate['addMetric'];\n\n constructor(\n lc: LogContext,\n mainSources: IVMSourceBranch,\n addQuery: AddQuery,\n addCustomQuery: AddCustomQuery,\n updateQuery: UpdateQuery,\n updateCustomQuery: UpdateCustomQuery,\n flushQueryChanges: () => void,\n batchViewUpdates: (applyViewUpdates: () => void) => void,\n addMetric: MetricsDelegate['addMetric'],\n assertValidRunOptions: (options?: RunOptions) => void,\n ) {\n super();\n this.#lc = lc;\n this.#mainSources = mainSources;\n this.addServerQuery = addQuery;\n this.updateServerQuery = updateQuery;\n this.updateCustomQuery = updateCustomQuery;\n this.#batchViewUpdates = batchViewUpdates;\n this.assertValidRunOptions = assertValidRunOptions;\n this.addCustomQuery = addCustomQuery;\n this.flushQueryChanges = flushQueryChanges;\n this.addMetric = addMetric;\n }\n\n applyFiltersAnyway?: boolean \| undefined;\n\n debug?: DebugDelegate \| undefined;\n\n getSource(name: string): Source \| undefined {\n return this.#mainSources.getSource(name);\n }\n\n override get pipelinesReady(): boolean {\n return this.#pipelinesReady;\n }\n\n override onPipelinesReady(cb: () => void): () => void {\n assert(\n !this.#pipelinesReady,\n 'onPipelinesReady called while pipelines are ready',\n );\n this.#pendingPipelines.add(cb);\n return () => {\n this.#pendingPipelines.delete(cb);\n };\n }\n\n /\n Stop building pipelines for materialized queries until\n * {@link markPipelinesReady} is called. Views materialized in the meantime\n * are empty and `unknown`, exactly as they would be over empty sources.\n /\n deferPipelines(): void {\n this.#pipelinesReady = false;\n }\n\n /\n Build and hydrate every pipeline deferred since {@link deferPipelines},\n * in materialization order, as a single view-update batch. A pipeline that\n * fails to build is logged and its view reports an error; the rest are\n * still built.\n */\n markPipelinesReady(): void {\n if (this.#pipelinesReady) {\n return;\n }\n this.#pipelinesReady = true;\n if (this.#pendingPipelines.size === 0) {\n return;\n }\n const pending = [...this.#pendingPipelines];\n this.#pendingPipelines.clear();\n this.batchViewUpdates(() => {\n try {\n for (const attach of pending) {\n try {\n attach();\n } catch (e) {\n // The failing view has already been marked as errored; keep\n // building the others rather than aborting startup.\n this.#lc.error?.(\n ErrorKind.Internal,\n 'Failed to build a deferred query pipeline',\n e,\n );\n }\n }\n } finally {\n this.#endTransaction();\n }\n });\n }\n\n mapAst(ast: AST): AST {\n return ast;\n }\n\n override decorateSourceInput(input: SourceInput, queryID: string): Input {\n return new MeasurePushOperator(input, queryID, this, 'query-update-client');\n }\n\n onTransactionCommit(cb: CommitListener): () => void {\n this.#commitListeners.add(cb);\n return () => {\n this.#commitListeners.delete(cb);\n };\n }\n\n override batchViewUpdates<T>(applyViewUpdates: () => T) {\n let result: T \| undefined;\n let viewChangesPerformed = false;\n this.#batchViewUpdates(() => {\n result = applyViewUpdates();\n viewChangesPerformed = true;\n });\n assert(\n viewChangesPerformed,\n 'batchViewUpdates must call applyViewUpdates synchronously.',\n );\n return result as T;\n }\n\n processChanges(\n expectedHead: Hash \| undefined,\n newHead: Hash,\n changes: NoIndexDiff,\n ) {\n this.batchViewUpdates(() => {\n try {\n this.#mainSources.advance(expectedHead, newHead, changes);\n } finally {\n this.#endTransaction();\n }\n });\n }\n\n #endTransaction() {\n for (const listener of this.#commitListeners) {\n try {\n listener();\n } catch (e) {\n // We should not fatal the inner-workings of Zero due to the user's application\n // code throwing an error.\n // Hence we wrap notifications in a try-catch block.\n this.#lc.error?.(\n ErrorKind.Internal,\n 'Failed notifying a commit listener of IVM updates',\n e,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;AA6BA,IAAa,cAAb,cAAiC,kBAAkB;CACjD;CAMA;CAEA;CACA;CACA;CACA;CACA;CACA;CACA,mCAAiD,IAAI,IAAI;CAOzD,kBAAkB;CAClB,oCAA8C,IAAI,IAAI;CAEtD;;;;;CAMA,uBAAgC;CAEhC;CAEA,YACE,IACA,aACA,UACA,gBACA,aACA,mBACA,mBACA,kBACA,WACA,uBACA;EACA,MAAM;EACN,KAAKA,MAAM;EACX,KAAKC,eAAe;EACpB,KAAK,iBAAiB;EACtB,KAAK,oBAAoB;EACzB,KAAK,oBAAoB;EACzB,KAAKC,oBAAoB;EACzB,KAAK,wBAAwB;EAC7B,KAAK,iBAAiB;EACtB,KAAK,oBAAoB;EACzB,KAAK,YAAY;CACnB;CAEA;CAEA;CAEA,UAAU,MAAkC;EAC1C,OAAO,KAAKD,aAAa,UAAU,IAAI;CACzC;CAEA,IAAa,iBAA0B;EACrC,OAAO,KAAKI;CACd;CAEA,iBAA0B,IAA4B;EACpD,OACE,CAAC,KAAKA,iBACN,mDACF;EACA,KAAKD,kBAAkB,IAAI,EAAE;EAC7B,aAAa;GACX,KAAKA,kBAAkB,OAAO,EAAE;EAClC;CACF;;;;;;CAOA,iBAAuB;EACrB,KAAKC,kBAAkB;CACzB;;;;;;;CAQA,qBAA2B;EACzB,IAAI,KAAKA,iBACP;EAEF,KAAKA,kBAAkB;EACvB,IAAI,KAAKD,kBAAkB,SAAS,GAClC;EAEF,MAAM,UAAU,CAAC,GAAG,KAAKA,iBAAiB;EAC1C,KAAKA,kBAAkB,MAAM;EAC7B,KAAK,uBAAuB;GAC1B,IAAI;IACF,KAAK,MAAM,UAAU,SACnB,IAAI;KACF,OAAO;IACT,SAAS,GAAG;KAGV,KAAKJ,IAAI,QACP,UACA,6CACA,CACF;IACF;GAEJ,UAAU;IACR,KAAKM,gBAAgB;GACvB;EACF,CAAC;CACH;CAEA,OAAO,KAAe;EACpB,OAAO;CACT;CAEA,oBAA6B,OAAoB,SAAwB;EACvE,OAAO,IAAI,oBAAoB,OAAO,SAAS,MAAM,qBAAqB;CAC5E;CAEA,oBAAoB,IAAgC;EAClD,KAAKH,iBAAiB,IAAI,EAAE;EAC5B,aAAa;GACX,KAAKA,iBAAiB,OAAO,EAAE;EACjC;CACF;CAEA,iBAA6B,kBAA2B;EACtD,IAAI;EACJ,IAAI,uBAAuB;EAC3B,KAAKD,wBAAwB;GAC3B,SAAS,iBAAiB;GAC1B,uBAAuB;EACzB,CAAC;EACD,OACE,sBACA,4DACF;EACA,OAAO;CACT;CAEA,eACE,cACA,SACA,SACA;EACA,KAAK,uBAAuB;GAC1B,IAAI;IACF,KAAKD,aAAa,QAAQ,cAAc,SAAS,OAAO;GAC1D,UAAU;IACR,KAAKK,gBAAgB;GACvB;EACF,CAAC;CACH;CAEA,kBAAkB;EAChB,KAAK,MAAM,YAAY,KAAKH,kBAC1B,IAAI;GACF,SAAS;EACX,SAAS,GAAG;GAIV,KAAKH,IAAI,QACP,UACA,qDACA,CACF;EACF;CAEJ;AACF"}

package/out/zero-client/src/client/inspector/lazy-inspector.js CHANGED Viewed

@@ -161,7 +161,7 @@ function clientMap(delegate, clientID) {
 	return withDagRead(delegate, async (dagRead) => {
 		const tree = await getBTree(dagRead, clientID);
 		const map = /* @__PURE__ */ new Map();
-		for await (const [key, value] of tree.scan("")) map.set(key, value);
+		for await (const [key, value] of tree.scan("", true)) map.set(key, value);
 		return map;
 	});
 }
@@ -170,7 +170,7 @@ function clientRows(delegate, clientID, tableName) {
 		const prefix = "e/" + tableName + "/";
 		const tree = await getBTree(dagRead, clientID);
 		const rows = [];
-		for await (const [key, value] of tree.scan(prefix)) {
+		for await (const [key, value] of tree.scan(prefix, false)) {
 			if (!key.startsWith(prefix)) break;
 			rows.push(value);
 		}