@rocicorp/zero 0.26.1 → 0.26.2-canary.4

package/out/zero-cache/src/services/replicator/write-worker.js

@@ -0,0 +1,68 @@
+import { must } from "../../../../shared/src/must.js";
+import { Database } from "../../../../zqlite/src/db.js";
+import { createLogContext } from "../../server/logging.js";
+import { StatementRunner } from "../../db/statements.js";
+import { getSubscriptionState } from "./schema/replication-state.js";
+import { ChangeProcessor } from "./change-processor.js";
+import { applyPragmas } from "./write-worker-client.js";
+import { parentPort } from "node:worker_threads";
+//#region ../zero-cache/src/services/replicator/write-worker.ts
+if (!parentPort) throw new Error("write-worker must be run as a worker thread");
+var port = parentPort;
+function createAPI() {
+	let db;
+	let runner;
+	let processor;
+	let mode;
+	let lc;
+	function createProcessor() {
+		processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {
+			port.postMessage({ writeError: err instanceof Error ? err : new Error(String(err)) });
+		});
+	}
+	return {
+		init(dbPath, cpMode, pragmas, logConfig) {
+			lc = createLogContext({ log: logConfig }, { worker: "write-worker" });
+			db = new Database(lc, dbPath);
+			applyPragmas(db, pragmas);
+			runner = new StatementRunner(db);
+			mode = cpMode;
+			createProcessor();
+		},
+		getSubscriptionState() {
+			return getSubscriptionState(must(runner));
+		},
+		processMessage(downstream) {
+			return must(processor).processMessage(must(lc), downstream);
+		},
+		abort() {
+			must(processor).abort(must(lc));
+			createProcessor();
+		},
+		stop() {
+			db?.close();
+			db = void 0;
+			runner = void 0;
+			processor = void 0;
+		}
+	};
+}
+var api = createAPI();
+port.on("message", (msg) => {
+	try {
+		const result = api[msg.method](...msg.args);
+		if (msg.method !== "abort") port.postMessage({
+			method: msg.method,
+			result
+		});
+	} catch (e) {
+		if (msg.method !== "abort") port.postMessage({
+			method: msg.method,
+			error: e
+		});
+	}
+});
+//#endregion
+export {};
+
+//# sourceMappingURL=write-worker.js.map

package/out/zero-cache/src/services/replicator/write-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"write-worker.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/write-worker.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {parentPort} from 'node:worker_threads';\nimport type {LogConfig} from '../../../../shared/src/logging.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {createLogContext} from '../../server/logging.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {ChangeProcessor, type ChangeProcessorMode} from './change-processor.ts';\nimport {getSubscriptionState} from './schema/replication-state.ts';\nimport {\n  applyPragmas,\n  type ArgsMap,\n  type Method,\n  type PragmaConfig,\n  type Request,\n  type Response,\n  type ResultMap,\n  type WriteError,\n} from './write-worker-client.ts';\n\nif (!parentPort) {\n  throw new Error('write-worker must be run as a worker thread');\n}\n\nconst port = parentPort;\n\ntype API = {[M in Method]: (...args: ArgsMap[M]) => ResultMap[M]};\n\nfunction createAPI(): API {\n  let db: Database | undefined;\n  let runner: StatementRunner | undefined;\n  let processor: ChangeProcessor | undefined;\n  let mode: ChangeProcessorMode | undefined;\n  let lc: LogContext | undefined;\n\n  function createProcessor() {\n    processor = new ChangeProcessor(must(runner), must(mode), (_lc, err) => {\n      port.postMessage({\n        writeError: err instanceof Error ? err : new Error(String(err)),\n      } satisfies WriteError);\n    });\n  }\n\n  return {\n    init(\n      dbPath: string,\n      cpMode: ChangeProcessorMode,\n      pragmas: PragmaConfig,\n      logConfig: LogConfig,\n    ) {\n      lc = createLogContext({log: logConfig}, {worker: 'write-worker'});\n      db = new Database(lc, dbPath);\n      applyPragmas(db, pragmas);\n      runner = new StatementRunner(db);\n      mode = cpMode;\n      createProcessor();\n    },\n\n    getSubscriptionState() {\n      return getSubscriptionState(must(runner));\n    },\n\n    processMessage(downstream: ChangeStreamData) {\n      return must(processor).processMessage(must(lc), downstream);\n    },\n\n    abort() {\n      must(processor).abort(must(lc));\n      createProcessor();\n    },\n\n    stop() {\n      db?.close();\n      db = undefined;\n      runner = undefined;\n      processor = undefined;\n    },\n  };\n}\n\nconst api = createAPI();\n\nport.on('message', (msg: Request) => {\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any -- TS can't narrow msg.method + msg.args together\n    const result = (api[msg.method] as (...args: any[]) => unknown)(\n      ...msg.args,\n    );\n    // abort is fire-and-forget — no pending slot on the client side.\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, result} as Response);\n    }\n  } catch (e) {\n    if (msg.method !== 'abort') {\n      port.postMessage({method: msg.method, error: e} as Response);\n    }\n  }\n});\n"],"mappings":";;;;;;;;;AAqBA,IAAI,CAAC,WACH,OAAM,IAAI,MAAM,8CAA8C;AAGhE,IAAM,OAAO;AAIb,SAAS,YAAiB;CACxB,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,IAAI;CAEJ,SAAS,kBAAkB;AACzB,cAAY,IAAI,gBAAgB,KAAK,OAAO,EAAE,KAAK,KAAK,GAAG,KAAK,QAAQ;AACtE,QAAK,YAAY,EACf,YAAY,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,IAAI,CAAC,EAChE,CAAsB;IACvB;;AAGJ,QAAO;EACL,KACE,QACA,QACA,SACA,WACA;AACA,QAAK,iBAAiB,EAAC,KAAK,WAAU,EAAE,EAAC,QAAQ,gBAAe,CAAC;AACjE,QAAK,IAAI,SAAS,IAAI,OAAO;AAC7B,gBAAa,IAAI,QAAQ;AACzB,YAAS,IAAI,gBAAgB,GAAG;AAChC,UAAO;AACP,oBAAiB;;EAGnB,uBAAuB;AACrB,UAAO,qBAAqB,KAAK,OAAO,CAAC;;EAG3C,eAAe,YAA8B;AAC3C,UAAO,KAAK,UAAU,CAAC,eAAe,KAAK,GAAG,EAAE,WAAW;;EAG7D,QAAQ;AACN,QAAK,UAAU,CAAC,MAAM,KAAK,GAAG,CAAC;AAC/B,oBAAiB;;EAGnB,OAAO;AACL,OAAI,OAAO;AACX,QAAK,KAAA;AACL,YAAS,KAAA;AACT,eAAY,KAAA;;EAEf;;AAGH,IAAM,MAAM,WAAW;AAEvB,KAAK,GAAG,YAAY,QAAiB;AACnC,KAAI;EAEF,MAAM,SAAU,IAAI,IAAI,QACtB,GAAG,IAAI,KACR;AAED,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ;GAAO,CAAa;UAErD,GAAG;AACV,MAAI,IAAI,WAAW,QACjB,MAAK,YAAY;GAAC,QAAQ,IAAI;GAAQ,OAAO;GAAE,CAAa;;EAGhE"}

package/out/zero-cache/src/services/run-ast.js

@@ -1,128 +1,87 @@
-import { astToZQL } from "../../../ast-to-zql/src/ast-to-zql.js";
-import { formatOutput } from "../../../ast-to-zql/src/format.js";
 import { assert } from "../../../shared/src/asserts.js";
-import { must } from "../../../shared/src/must.js";
 import { sleep } from "../../../shared/src/sleep.js";
-import "../../../zero-protocol/src/ast.js";
+import { must } from "../../../shared/src/must.js";
+import { mapAST } from "../../../zero-protocol/src/ast.js";
 import { hashOfAST } from "../../../zero-protocol/src/query-hash.js";
-import { buildPipeline } from "../../../zql/src/builder/builder.js";
 import { skipYields } from "../../../zql/src/ivm/operator.js";
-import { resolveSimpleScalarSubqueries } from "../../../zqlite/src/resolve-scalar-subqueries.js";
-import { transformAndHashQuery } from "../auth/read-authorizer.js";
+import { buildPipeline } from "../../../zql/src/builder/builder.js";
 import { computeZqlSpecs } from "../db/lite-tables.js";
+import { astToZQL } from "../../../ast-to-zql/src/ast-to-zql.js";
+import { formatOutput } from "../../../ast-to-zql/src/format.js";
+import { transformAndHashQuery } from "../auth/read-authorizer.js";
+import { resolveSimpleScalarSubqueries } from "../../../zqlite/src/resolve-scalar-subqueries.js";
 import { hydrate } from "./view-syncer/pipeline-driver.js";
+//#region ../zero-cache/src/services/run-ast.ts
 async function runAst(lc, clientSchema, ast, isTransformed, options, yieldProcess) {
-  const { permissions, host, db } = options;
-  const result = {
-    warnings: [],
-    syncedRows: void 0,
-    syncedRowCount: 0,
-    start: 0,
-    end: 0,
-    elapsed: 0,
-    afterPermissions: void 0,
-    readRows: void 0,
-    readRowCountsByQuery: {},
-    readRowCount: void 0
-  };
-  if (options.applyPermissions) {
-    const auth = options.auth;
-    if (!auth) {
-      result.warnings.push(
-        "No auth data provided. Permission rules will compare to `NULL` wherever an auth data field is referenced."
-      );
-    }
-    ast = transformAndHashQuery(
-      lc,
-      "clientGroupIDForAnalyze",
-      ast,
-      must(permissions),
-      auth,
-      false
-    ).transformedAst;
-    result.afterPermissions = await formatOutput(ast.table + astToZQL(ast));
-  }
-  const executor = (subqueryAST, childField) => {
-    const input = buildPipeline(subqueryAST, host, "scalar-subquery");
-    let node;
-    for (const n of skipYields(input.fetch({}))) {
-      node ??= n;
-    }
-    input.destroy();
-    return node ? node.row[childField] ?? null : void 0;
-  };
-  const { ast: resolvedAst } = resolveSimpleScalarSubqueries(
-    ast,
-    options.tableSpecs,
-    executor
-  );
-  const pipeline = buildPipeline(
-    resolvedAst,
-    host,
-    "query-id",
-    options.costModel,
-    lc,
-    options.planDebugger
-  );
-  const start = performance.now();
-  let syncedRowCount = 0;
-  const rowsByTable = {};
-  const seenByTable = /* @__PURE__ */ new Set();
-  for (const rowChange of hydrate(
-    pipeline,
-    hashOfAST(resolvedAst),
-    clientSchema,
-    computeZqlSpecs(lc, db, { includeBackfillingColumns: false })
-  )) {
-    if (rowChange === "yield") {
-      await yieldProcess();
-      continue;
-    }
-    assert(rowChange.type === "add", "Hydration only handles add row changes");
-    if (syncedRowCount % 10 === 0) {
-      await Promise.resolve();
-    }
-    if (syncedRowCount % 100 === 0) {
-      await sleep(1);
-    }
-    let rows = rowsByTable[rowChange.table];
-    const s = rowChange.table + "." + JSON.stringify(rowChange.row);
-    if (seenByTable.has(s)) {
-      continue;
-    }
-    syncedRowCount++;
-    seenByTable.add(s);
-    if (options.syncedRows) {
-      if (!rows) {
-        rows = [];
-        rowsByTable[rowChange.table] = rows;
-      }
-      rows.push(rowChange.row);
-    }
-  }
-  const end = performance.now();
-  if (options.syncedRows) {
-    result.syncedRows = rowsByTable;
-  }
-  result.start = start;
-  result.end = end;
-  result.elapsed = end - start;
-  result.syncedRowCount = syncedRowCount;
-  result.readRowCountsByQuery = host.debug?.getVendedRowCounts() ?? {};
-  let readRowCount = 0;
-  for (const c of Object.values(result.readRowCountsByQuery)) {
-    for (const v of Object.values(c)) {
-      readRowCount += v;
-    }
-  }
-  result.readRowCount = readRowCount;
-  result.dbScansByQuery = host.debug?.getNVisitCounts() ?? {};
-  if (options.vendedRows) {
-    result.readRows = host.debug?.getVendedRows();
-  }
-  return result;
+	const { clientToServerMapper, permissions, host, db } = options;
+	const result = {
+		warnings: [],
+		syncedRows: void 0,
+		syncedRowCount: 0,
+		start: 0,
+		end: 0,
+		elapsed: 0,
+		afterPermissions: void 0,
+		readRows: void 0,
+		readRowCountsByQuery: {},
+		readRowCount: void 0
+	};
+	if (!isTransformed) ast = mapAST(ast, must(clientToServerMapper));
+	if (options.applyPermissions) {
+		const auth = options.auth;
+		if (!auth) result.warnings.push("No auth data provided. Permission rules will compare to `NULL` wherever an auth data field is referenced.");
+		ast = transformAndHashQuery(lc, "clientGroupIDForAnalyze", ast, must(permissions), auth, false).transformedAst;
+		result.afterPermissions = await formatOutput(ast.table + astToZQL(ast));
+	}
+	const executor = (subqueryAST, childField) => {
+		const input = buildPipeline(subqueryAST, host, "scalar-subquery");
+		let node;
+		for (const n of skipYields(input.fetch({}))) node ??= n;
+		input.destroy();
+		return node ? node.row[childField] ?? null : void 0;
+	};
+	const { ast: resolvedAst } = resolveSimpleScalarSubqueries(ast, options.tableSpecs, executor);
+	const pipeline = buildPipeline(resolvedAst, host, "query-id", options.costModel, lc, options.planDebugger);
+	const start = performance.now();
+	let syncedRowCount = 0;
+	const rowsByTable = {};
+	const seenByTable = /* @__PURE__ */ new Set();
+	for (const rowChange of hydrate(pipeline, hashOfAST(resolvedAst), clientSchema, computeZqlSpecs(lc, db, { includeBackfillingColumns: false }))) {
+		if (rowChange === "yield") {
+			await yieldProcess();
+			continue;
+		}
+		assert(rowChange.type === "add", "Hydration only handles add row changes");
+		if (syncedRowCount % 10 === 0) await Promise.resolve();
+		if (syncedRowCount % 100 === 0) await sleep(1);
+		let rows = rowsByTable[rowChange.table];
+		const s = rowChange.table + "." + JSON.stringify(rowChange.row);
+		if (seenByTable.has(s)) continue;
+		syncedRowCount++;
+		seenByTable.add(s);
+		if (options.syncedRows) {
+			if (!rows) {
+				rows = [];
+				rowsByTable[rowChange.table] = rows;
+			}
+			rows.push(rowChange.row);
+		}
+	}
+	const end = performance.now();
+	if (options.syncedRows) result.syncedRows = rowsByTable;
+	result.start = start;
+	result.end = end;
+	result.elapsed = end - start;
+	result.syncedRowCount = syncedRowCount;
+	result.readRowCountsByQuery = host.debug?.getVendedRowCounts() ?? {};
+	let readRowCount = 0;
+	for (const c of Object.values(result.readRowCountsByQuery)) for (const v of Object.values(c)) readRowCount += v;
+	result.readRowCount = readRowCount;
+	result.dbScansByQuery = host.debug?.getNVisitCounts() ?? {};
+	if (options.vendedRows) result.readRows = host.debug?.getVendedRows();
+	return result;
 }
-export {
-  runAst
-};
-//# sourceMappingURL=run-ast.js.map
+//#endregion
+export { runAst };
+
+//# sourceMappingURL=run-ast.js.map

package/out/zero-cache/src/services/run-ast.js.map

@@ -1 +1 @@
-{"version":3,"file":"run-ast.js","sources":["../../../../../zero-cache/src/services/run-ast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\n// @circular-dep-ignore\nimport {astToZQL} from '../../../ast-to-zql/src/ast-to-zql.ts';\n// @circular-dep-ignore\nimport {formatOutput} from '../../../ast-to-zql/src/format.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST, LiteralValue} from '../../../zero-protocol/src/ast.ts';\nimport {mapAST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {Row} from '../../../zero-protocol/src/data.ts';\nimport {hashOfAST} from '../../../zero-protocol/src/query-hash.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport type {NameMapper} from '../../../zero-schema/src/name-mapper.ts';\nimport {\n  buildPipeline,\n  type BuilderDelegate,\n} from '../../../zql/src/builder/builder.ts';\nimport type {Node} from '../../../zql/src/ivm/data.ts';\nimport {skipYields} from '../../../zql/src/ivm/operator.ts';\nimport type {ConnectionCostModel} from '../../../zql/src/planner/planner-connection.ts';\nimport type {PlanDebugger} from '../../../zql/src/planner/planner-debug.ts';\nimport type {Database} from '../../../zqlite/src/db.ts';\nimport {resolveSimpleScalarSubqueries} from '../../../zqlite/src/resolve-scalar-subqueries.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport {transformAndHashQuery} from '../auth/read-authorizer.ts';\nimport {computeZqlSpecs} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec} from '../db/specs.ts';\nimport {hydrate} from './view-syncer/pipeline-driver.ts';\n\nexport type RunAstOptions = {\n  applyPermissions?: boolean | undefined;\n  auth?: JWTAuth | undefined;\n  clientToServerMapper?: NameMapper | undefined;\n  costModel?: ConnectionCostModel | undefined;\n  db: Database;\n  host: BuilderDelegate;\n  permissions?: PermissionsConfig | undefined;\n  planDebugger?: PlanDebugger | undefined;\n  syncedRows?: boolean | undefined;\n  tableSpecs: Map<string, LiteAndZqlSpec>;\n  vendedRows?: boolean | undefined;\n};\n\nexport async function runAst(\n  lc: LogContext,\n  clientSchema: ClientSchema,\n  ast: AST,\n  isTransformed: boolean,\n  options: RunAstOptions,\n  yieldProcess: () => Promise<void>,\n): Promise<AnalyzeQueryResult> {\n  const {clientToServerMapper, permissions, host, db} = options;\n  const result: AnalyzeQueryResult = {\n    warnings: [],\n    syncedRows: undefined,\n    syncedRowCount: 0,\n    start: 0,\n    end: 0,\n    elapsed: 0,\n    afterPermissions: undefined,\n    readRows: undefined,\n    readRowCountsByQuery: {},\n    readRowCount: undefined,\n  };\n\n  if (!isTransformed) {\n    // map the AST to server names if not already transformed\n    ast = mapAST(ast, must(clientToServerMapper));\n  }\n  if (options.applyPermissions) {\n    const auth = options.auth;\n    if (!auth) {\n      result.warnings.push(\n        'No auth data provided. Permission rules will compare to `NULL` wherever an auth data field is referenced.',\n      );\n    }\n    ast = transformAndHashQuery(\n      lc,\n      'clientGroupIDForAnalyze',\n      ast,\n      must(permissions),\n      auth,\n      false,\n    ).transformedAst;\n    result.afterPermissions = await formatOutput(ast.table + astToZQL(ast));\n  }\n\n  // Resolve scalar subqueries (e.g. whereExists with {scalar: true}) to\n  // literal equality conditions so that SQLite can use indexes effectively.\n  // Without this, correlated subqueries get stripped from SQL filters and\n  // queries on large tables fall back to full table scans.\n  const executor = (\n    subqueryAST: AST,\n    childField: string,\n  ): LiteralValue | null | undefined => {\n    const input = buildPipeline(subqueryAST, host, 'scalar-subquery');\n    // Consume the full stream rather than using first() to avoid\n    // triggering early return on Take's #initialFetch assertion.\n    // The subquery AST already has limit: 1, so at most one row is produced.\n    let node: Node | undefined;\n    for (const n of skipYields(input.fetch({}))) {\n      node ??= n;\n    }\n    input.destroy();\n    return node ? ((node.row[childField] as LiteralValue) ?? null) : undefined;\n  };\n\n  const {ast: resolvedAst} = resolveSimpleScalarSubqueries(\n    ast,\n    options.tableSpecs,\n    executor,\n  );\n\n  const pipeline = buildPipeline(\n    resolvedAst,\n    host,\n    'query-id',\n    options.costModel,\n    lc,\n    options.planDebugger,\n  );\n\n  const start = performance.now();\n\n  let syncedRowCount = 0;\n  const rowsByTable: Record<string, Row[]> = {};\n  const seenByTable: Set<string> = new Set();\n  for (const rowChange of hydrate(\n    pipeline,\n    hashOfAST(resolvedAst),\n    clientSchema,\n    computeZqlSpecs(lc, db, {includeBackfillingColumns: false}),\n  )) {\n    if (rowChange === 'yield') {\n      await yieldProcess();\n      continue;\n    }\n    assert(rowChange.type === 'add', 'Hydration only handles add row changes');\n\n    // yield to other tasks to avoid blocking for too long\n    if (syncedRowCount % 10 === 0) {\n      await Promise.resolve();\n    }\n    if (syncedRowCount % 100 === 0) {\n      await sleep(1);\n    }\n\n    let rows: Row[] = rowsByTable[rowChange.table];\n    const s = rowChange.table + '.' + JSON.stringify(rowChange.row);\n    if (seenByTable.has(s)) {\n      continue; // skip duplicates\n    }\n    syncedRowCount++;\n    seenByTable.add(s);\n    if (options.syncedRows) {\n      if (!rows) {\n        rows = [];\n        rowsByTable[rowChange.table] = rows;\n      }\n      rows.push(rowChange.row);\n    }\n  }\n\n  const end = performance.now();\n  if (options.syncedRows) {\n    result.syncedRows = rowsByTable;\n  }\n  result.start = start;\n  result.end = end;\n  result.elapsed = end - start;\n\n  // Always include the count of synced and vended rows.\n  result.syncedRowCount = syncedRowCount;\n  result.readRowCountsByQuery = host.debug?.getVendedRowCounts() ?? {};\n  let readRowCount = 0;\n  for (const c of Object.values(result.readRowCountsByQuery)) {\n    for (const v of Object.values(c)) {\n      readRowCount += v;\n    }\n  }\n  result.readRowCount = readRowCount;\n  result.dbScansByQuery = host.debug?.getNVisitCounts() ?? {};\n\n  if (options.vendedRows) {\n    result.readRows = host.debug?.getVendedRows();\n  }\n  return result;\n}\n"],"names":[],"mappings":";;;;;;;;;;;;;AA8CA,eAAsB,OACpB,IACA,cACA,KACA,eACA,SACA,cAC6B;AAC7B,QAAM,EAAuB,aAAa,MAAM,OAAM;AACtD,QAAM,SAA6B;AAAA,IACjC,UAAU,CAAA;AAAA,IACV,YAAY;AAAA,IACZ,gBAAgB;AAAA,IAChB,OAAO;AAAA,IACP,KAAK;AAAA,IACL,SAAS;AAAA,IACT,kBAAkB;AAAA,IAClB,UAAU;AAAA,IACV,sBAAsB,CAAA;AAAA,IACtB,cAAc;AAAA,EAAA;AAOhB,MAAI,QAAQ,kBAAkB;AAC5B,UAAM,OAAO,QAAQ;AACrB,QAAI,CAAC,MAAM;AACT,aAAO,SAAS;AAAA,QACd;AAAA,MAAA;AAAA,IAEJ;AACA,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA,KAAK,WAAW;AAAA,MAChB;AAAA,MACA;AAAA,IAAA,EACA;AACF,WAAO,mBAAmB,MAAM,aAAa,IAAI,QAAQ,SAAS,GAAG,CAAC;AAAA,EACxE;AAMA,QAAM,WAAW,CACf,aACA,eACoC;AACpC,UAAM,QAAQ,cAAc,aAAa,MAAM,iBAAiB;AAIhE,QAAI;AACJ,eAAW,KAAK,WAAW,MAAM,MAAM,CAAA,CAAE,CAAC,GAAG;AAC3C,eAAS;AAAA,IACX;AACA,UAAM,QAAA;AACN,WAAO,OAAS,KAAK,IAAI,UAAU,KAAsB,OAAQ;AAAA,EACnE;AAEA,QAAM,EAAC,KAAK,YAAA,IAAe;AAAA,IACzB;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,EAAA;AAGF,QAAM,WAAW;AAAA,IACf;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,IACA,QAAQ;AAAA,EAAA;AAGV,QAAM,QAAQ,YAAY,IAAA;AAE1B,MAAI,iBAAiB;AACrB,QAAM,cAAqC,CAAA;AAC3C,QAAM,kCAA+B,IAAA;AACrC,aAAW,aAAa;AAAA,IACtB;AAAA,IACA,UAAU,WAAW;AAAA,IACrB;AAAA,IACA,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,OAAM;AAAA,EAAA,GACzD;AACD,QAAI,cAAc,SAAS;AACzB,YAAM,aAAA;AACN;AAAA,IACF;AACA,WAAO,UAAU,SAAS,OAAO,wCAAwC;AAGzE,QAAI,iBAAiB,OAAO,GAAG;AAC7B,YAAM,QAAQ,QAAA;AAAA,IAChB;AACA,QAAI,iBAAiB,QAAQ,GAAG;AAC9B,YAAM,MAAM,CAAC;AAAA,IACf;AAEA,QAAI,OAAc,YAAY,UAAU,KAAK;AAC7C,UAAM,IAAI,UAAU,QAAQ,MAAM,KAAK,UAAU,UAAU,GAAG;AAC9D,QAAI,YAAY,IAAI,CAAC,GAAG;AACtB;AAAA,IACF;AACA;AACA,gBAAY,IAAI,CAAC;AACjB,QAAI,QAAQ,YAAY;AACtB,UAAI,CAAC,MAAM;AACT,eAAO,CAAA;AACP,oBAAY,UAAU,KAAK,IAAI;AAAA,MACjC;AACA,WAAK,KAAK,UAAU,GAAG;AAAA,IACzB;AAAA,EACF;AAEA,QAAM,MAAM,YAAY,IAAA;AACxB,MAAI,QAAQ,YAAY;AACtB,WAAO,aAAa;AAAA,EACtB;AACA,SAAO,QAAQ;AACf,SAAO,MAAM;AACb,SAAO,UAAU,MAAM;AAGvB,SAAO,iBAAiB;AACxB,SAAO,uBAAuB,KAAK,OAAO,mBAAA,KAAwB,CAAA;AAClE,MAAI,eAAe;AACnB,aAAW,KAAK,OAAO,OAAO,OAAO,oBAAoB,GAAG;AAC1D,eAAW,KAAK,OAAO,OAAO,CAAC,GAAG;AAChC,sBAAgB;AAAA,IAClB;AAAA,EACF;AACA,SAAO,eAAe;AACtB,SAAO,iBAAiB,KAAK,OAAO,gBAAA,KAAqB,CAAA;AAEzD,MAAI,QAAQ,YAAY;AACtB,WAAO,WAAW,KAAK,OAAO,cAAA;AAAA,EAChC;AACA,SAAO;AACT;"}
+{"version":3,"file":"run-ast.js","names":[],"sources":["../../../../../zero-cache/src/services/run-ast.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\n// @circular-dep-ignore\nimport {astToZQL} from '../../../ast-to-zql/src/ast-to-zql.ts';\n// @circular-dep-ignore\nimport {formatOutput} from '../../../ast-to-zql/src/format.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST, LiteralValue} from '../../../zero-protocol/src/ast.ts';\nimport {mapAST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {Row} from '../../../zero-protocol/src/data.ts';\nimport {hashOfAST} from '../../../zero-protocol/src/query-hash.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport type {NameMapper} from '../../../zero-schema/src/name-mapper.ts';\nimport {\n  buildPipeline,\n  type BuilderDelegate,\n} from '../../../zql/src/builder/builder.ts';\nimport type {Node} from '../../../zql/src/ivm/data.ts';\nimport {skipYields} from '../../../zql/src/ivm/operator.ts';\nimport type {ConnectionCostModel} from '../../../zql/src/planner/planner-connection.ts';\nimport type {PlanDebugger} from '../../../zql/src/planner/planner-debug.ts';\nimport type {Database} from '../../../zqlite/src/db.ts';\nimport {resolveSimpleScalarSubqueries} from '../../../zqlite/src/resolve-scalar-subqueries.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport {transformAndHashQuery} from '../auth/read-authorizer.ts';\nimport {computeZqlSpecs} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec} from '../db/specs.ts';\nimport {hydrate} from './view-syncer/pipeline-driver.ts';\n\nexport type RunAstOptions = {\n  applyPermissions?: boolean | undefined;\n  auth?: JWTAuth | undefined;\n  clientToServerMapper?: NameMapper | undefined;\n  costModel?: ConnectionCostModel | undefined;\n  db: Database;\n  host: BuilderDelegate;\n  permissions?: PermissionsConfig | undefined;\n  planDebugger?: PlanDebugger | undefined;\n  syncedRows?: boolean | undefined;\n  tableSpecs: Map<string, LiteAndZqlSpec>;\n  vendedRows?: boolean | undefined;\n};\n\nexport async function runAst(\n  lc: LogContext,\n  clientSchema: ClientSchema,\n  ast: AST,\n  isTransformed: boolean,\n  options: RunAstOptions,\n  yieldProcess: () => Promise<void>,\n): Promise<AnalyzeQueryResult> {\n  const {clientToServerMapper, permissions, host, db} = options;\n  const result: AnalyzeQueryResult = {\n    warnings: [],\n    syncedRows: undefined,\n    syncedRowCount: 0,\n    start: 0,\n    end: 0,\n    elapsed: 0,\n    afterPermissions: undefined,\n    readRows: undefined,\n    readRowCountsByQuery: {},\n    readRowCount: undefined,\n  };\n\n  if (!isTransformed) {\n    // map the AST to server names if not already transformed\n    ast = mapAST(ast, must(clientToServerMapper));\n  }\n  if (options.applyPermissions) {\n    const auth = options.auth;\n    if (!auth) {\n      result.warnings.push(\n        'No auth data provided. Permission rules will compare to `NULL` wherever an auth data field is referenced.',\n      );\n    }\n    ast = transformAndHashQuery(\n      lc,\n      'clientGroupIDForAnalyze',\n      ast,\n      must(permissions),\n      auth,\n      false,\n    ).transformedAst;\n    result.afterPermissions = await formatOutput(ast.table + astToZQL(ast));\n  }\n\n  // Resolve scalar subqueries (e.g. whereExists with {scalar: true}) to\n  // literal equality conditions so that SQLite can use indexes effectively.\n  // Without this, correlated subqueries get stripped from SQL filters and\n  // queries on large tables fall back to full table scans.\n  const executor = (\n    subqueryAST: AST,\n    childField: string,\n  ): LiteralValue | null | undefined => {\n    const input = buildPipeline(subqueryAST, host, 'scalar-subquery');\n    // Consume the full stream rather than using first() to avoid\n    // triggering early return on Take's #initialFetch assertion.\n    // The subquery AST already has limit: 1, so at most one row is produced.\n    let node: Node | undefined;\n    for (const n of skipYields(input.fetch({}))) {\n      node ??= n;\n    }\n    input.destroy();\n    return node ? ((node.row[childField] as LiteralValue) ?? null) : undefined;\n  };\n\n  const {ast: resolvedAst} = resolveSimpleScalarSubqueries(\n    ast,\n    options.tableSpecs,\n    executor,\n  );\n\n  const pipeline = buildPipeline(\n    resolvedAst,\n    host,\n    'query-id',\n    options.costModel,\n    lc,\n    options.planDebugger,\n  );\n\n  const start = performance.now();\n\n  let syncedRowCount = 0;\n  const rowsByTable: Record<string, Row[]> = {};\n  const seenByTable: Set<string> = new Set();\n  for (const rowChange of hydrate(\n    pipeline,\n    hashOfAST(resolvedAst),\n    clientSchema,\n    computeZqlSpecs(lc, db, {includeBackfillingColumns: false}),\n  )) {\n    if (rowChange === 'yield') {\n      await yieldProcess();\n      continue;\n    }\n    assert(rowChange.type === 'add', 'Hydration only handles add row changes');\n\n    // yield to other tasks to avoid blocking for too long\n    if (syncedRowCount % 10 === 0) {\n      await Promise.resolve();\n    }\n    if (syncedRowCount % 100 === 0) {\n      await sleep(1);\n    }\n\n    let rows: Row[] = rowsByTable[rowChange.table];\n    const s = rowChange.table + '.' + JSON.stringify(rowChange.row);\n    if (seenByTable.has(s)) {\n      continue; // skip duplicates\n    }\n    syncedRowCount++;\n    seenByTable.add(s);\n    if (options.syncedRows) {\n      if (!rows) {\n        rows = [];\n        rowsByTable[rowChange.table] = rows;\n      }\n      rows.push(rowChange.row);\n    }\n  }\n\n  const end = performance.now();\n  if (options.syncedRows) {\n    result.syncedRows = rowsByTable;\n  }\n  result.start = start;\n  result.end = end;\n  result.elapsed = end - start;\n\n  // Always include the count of synced and vended rows.\n  result.syncedRowCount = syncedRowCount;\n  result.readRowCountsByQuery = host.debug?.getVendedRowCounts() ?? {};\n  let readRowCount = 0;\n  for (const c of Object.values(result.readRowCountsByQuery)) {\n    for (const v of Object.values(c)) {\n      readRowCount += v;\n    }\n  }\n  result.readRowCount = readRowCount;\n  result.dbScansByQuery = host.debug?.getNVisitCounts() ?? {};\n\n  if (options.vendedRows) {\n    result.readRows = host.debug?.getVendedRows();\n  }\n  return result;\n}\n"],"mappings":";;;;;;;;;;;;;;AA8CA,eAAsB,OACpB,IACA,cACA,KACA,eACA,SACA,cAC6B;CAC7B,MAAM,EAAC,sBAAsB,aAAa,MAAM,OAAM;CACtD,MAAM,SAA6B;EACjC,UAAU,EAAE;EACZ,YAAY,KAAA;EACZ,gBAAgB;EAChB,OAAO;EACP,KAAK;EACL,SAAS;EACT,kBAAkB,KAAA;EAClB,UAAU,KAAA;EACV,sBAAsB,EAAE;EACxB,cAAc,KAAA;EACf;AAED,KAAI,CAAC,cAEH,OAAM,OAAO,KAAK,KAAK,qBAAqB,CAAC;AAE/C,KAAI,QAAQ,kBAAkB;EAC5B,MAAM,OAAO,QAAQ;AACrB,MAAI,CAAC,KACH,QAAO,SAAS,KACd,4GACD;AAEH,QAAM,sBACJ,IACA,2BACA,KACA,KAAK,YAAY,EACjB,MACA,MACD,CAAC;AACF,SAAO,mBAAmB,MAAM,aAAa,IAAI,QAAQ,SAAS,IAAI,CAAC;;CAOzE,MAAM,YACJ,aACA,eACoC;EACpC,MAAM,QAAQ,cAAc,aAAa,MAAM,kBAAkB;EAIjE,IAAI;AACJ,OAAK,MAAM,KAAK,WAAW,MAAM,MAAM,EAAE,CAAC,CAAC,CACzC,UAAS;AAEX,QAAM,SAAS;AACf,SAAO,OAAS,KAAK,IAAI,eAAgC,OAAQ,KAAA;;CAGnE,MAAM,EAAC,KAAK,gBAAe,8BACzB,KACA,QAAQ,YACR,SACD;CAED,MAAM,WAAW,cACf,aACA,MACA,YACA,QAAQ,WACR,IACA,QAAQ,aACT;CAED,MAAM,QAAQ,YAAY,KAAK;CAE/B,IAAI,iBAAiB;CACrB,MAAM,cAAqC,EAAE;CAC7C,MAAM,8BAA2B,IAAI,KAAK;AAC1C,MAAK,MAAM,aAAa,QACtB,UACA,UAAU,YAAY,EACtB,cACA,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,OAAM,CAAC,CAC5D,EAAE;AACD,MAAI,cAAc,SAAS;AACzB,SAAM,cAAc;AACpB;;AAEF,SAAO,UAAU,SAAS,OAAO,yCAAyC;AAG1E,MAAI,iBAAiB,OAAO,EAC1B,OAAM,QAAQ,SAAS;AAEzB,MAAI,iBAAiB,QAAQ,EAC3B,OAAM,MAAM,EAAE;EAGhB,IAAI,OAAc,YAAY,UAAU;EACxC,MAAM,IAAI,UAAU,QAAQ,MAAM,KAAK,UAAU,UAAU,IAAI;AAC/D,MAAI,YAAY,IAAI,EAAE,CACpB;AAEF;AACA,cAAY,IAAI,EAAE;AAClB,MAAI,QAAQ,YAAY;AACtB,OAAI,CAAC,MAAM;AACT,WAAO,EAAE;AACT,gBAAY,UAAU,SAAS;;AAEjC,QAAK,KAAK,UAAU,IAAI;;;CAI5B,MAAM,MAAM,YAAY,KAAK;AAC7B,KAAI,QAAQ,WACV,QAAO,aAAa;AAEtB,QAAO,QAAQ;AACf,QAAO,MAAM;AACb,QAAO,UAAU,MAAM;AAGvB,QAAO,iBAAiB;AACxB,QAAO,uBAAuB,KAAK,OAAO,oBAAoB,IAAI,EAAE;CACpE,IAAI,eAAe;AACnB,MAAK,MAAM,KAAK,OAAO,OAAO,OAAO,qBAAqB,CACxD,MAAK,MAAM,KAAK,OAAO,OAAO,EAAE,CAC9B,iBAAgB;AAGpB,QAAO,eAAe;AACtB,QAAO,iBAAiB,KAAK,OAAO,iBAAiB,IAAI,EAAE;AAE3D,KAAI,QAAQ,WACV,QAAO,WAAW,KAAK,OAAO,eAAe;AAE/C,QAAO"}

package/out/zero-cache/src/services/runner.js

@@ -1,42 +1,40 @@
-class ServiceRunner {
-  #lc;
-  #instances = /* @__PURE__ */ new Map();
-  #create;
-  #isValid;
-  constructor(lc, factory, isValid = () => true) {
-    this.#lc = lc;
-    this.#create = factory;
-    this.#isValid = isValid;
-  }
-  /**
-   * Creates and runs the Service with the given `id`, returning
-   * an existing one if it is still running a valid.
-   */
-  getService(id) {
-    const existing = this.#instances.get(id);
-    if (existing && this.#isValid(existing)) {
-      return existing;
-    }
-    const service = this.#create(id);
-    this.#instances.set(id, service);
-    void service.run().catch(
-      (e) => this.#lc.error?.(
-        `Error running ${service.constructor?.name} ${service.id}`,
-        e
-      )
-    ).finally(() => {
-      this.#instances.delete(id);
-    });
-    return service;
-  }
-  get size() {
-    return this.#instances.size;
-  }
-  getServices() {
-    return this.#instances.values();
-  }
-}
-export {
-  ServiceRunner
+//#region ../zero-cache/src/services/runner.ts
+/**
+* Manages the creation and lifecycle of objects that implement
+* {@link Service}.
+*/
+var ServiceRunner = class {
+	#lc;
+	#instances = /* @__PURE__ */ new Map();
+	#create;
+	#isValid;
+	constructor(lc, factory, isValid = () => true) {
+		this.#lc = lc;
+		this.#create = factory;
+		this.#isValid = isValid;
+	}
+	/**
+	* Creates and runs the Service with the given `id`, returning
+	* an existing one if it is still running a valid.
+	*/
+	getService(id) {
+		const existing = this.#instances.get(id);
+		if (existing && this.#isValid(existing)) return existing;
+		const service = this.#create(id);
+		this.#instances.set(id, service);
+		service.run().catch((e) => this.#lc.error?.(`Error running ${service.constructor?.name} ${service.id}`, e)).finally(() => {
+			this.#instances.delete(id);
+		});
+		return service;
+	}
+	get size() {
+		return this.#instances.size;
+	}
+	getServices() {
+		return this.#instances.values();
+	}
 };
-//# sourceMappingURL=runner.js.map
+//#endregion
+export { ServiceRunner };
+
+//# sourceMappingURL=runner.js.map

package/out/zero-cache/src/services/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sources":["../../../../../zero-cache/src/services/runner.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Service} from './service.ts';\n\n/**\n * Manages the creation and lifecycle of objects that implement\n * {@link Service}.\n */\nexport class ServiceRunner<S extends Service> {\n  readonly #lc: LogContext;\n  readonly #instances = new Map<string, S>();\n  readonly #create: (id: string) => S;\n  readonly #isValid: (existing: S) => boolean;\n\n  constructor(\n    lc: LogContext,\n    factory: (id: string) => S,\n    isValid: (existing: S) => boolean = () => true,\n  ) {\n    this.#lc = lc;\n    this.#create = factory;\n    this.#isValid = isValid;\n  }\n\n  /**\n   * Creates and runs the Service with the given `id`, returning\n   * an existing one if it is still running a valid.\n   */\n  getService(id: string): S {\n    const existing = this.#instances.get(id);\n    if (existing && this.#isValid(existing)) {\n      return existing;\n    }\n    const service = this.#create(id);\n    this.#instances.set(id, service);\n    void service\n      .run()\n      .catch(e =>\n        this.#lc.error?.(\n          `Error running ${service.constructor?.name} ${service.id}`,\n          e,\n        ),\n      )\n      .finally(() => {\n        this.#instances.delete(id);\n      });\n    return service;\n  }\n\n  get size() {\n    return this.#instances.size;\n  }\n\n  getServices(): Iterable<S> {\n    return this.#instances.values();\n  }\n}\n"],"names":[],"mappings":"AAOO,MAAM,cAAiC;AAAA,EACnC;AAAA,EACA,iCAAiB,IAAA;AAAA,EACjB;AAAA,EACA;AAAA,EAET,YACE,IACA,SACA,UAAoC,MAAM,MAC1C;AACA,SAAK,MAAM;AACX,SAAK,UAAU;AACf,SAAK,WAAW;AAAA,EAClB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,WAAW,IAAe;AACxB,UAAM,WAAW,KAAK,WAAW,IAAI,EAAE;AACvC,QAAI,YAAY,KAAK,SAAS,QAAQ,GAAG;AACvC,aAAO;AAAA,IACT;AACA,UAAM,UAAU,KAAK,QAAQ,EAAE;AAC/B,SAAK,WAAW,IAAI,IAAI,OAAO;AAC/B,SAAK,QACF,MACA;AAAA,MAAM,CAAA,MACL,KAAK,IAAI;AAAA,QACP,iBAAiB,QAAQ,aAAa,IAAI,IAAI,QAAQ,EAAE;AAAA,QACxD;AAAA,MAAA;AAAA,IACF,EAED,QAAQ,MAAM;AACb,WAAK,WAAW,OAAO,EAAE;AAAA,IAC3B,CAAC;AACH,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,OAAO;AACT,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,cAA2B;AACzB,WAAO,KAAK,WAAW,OAAA;AAAA,EACzB;AACF;"}
+{"version":3,"file":"runner.js","names":["#lc","#instances","#create","#isValid"],"sources":["../../../../../zero-cache/src/services/runner.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {Service} from './service.ts';\n\n/**\n * Manages the creation and lifecycle of objects that implement\n * {@link Service}.\n */\nexport class ServiceRunner<S extends Service> {\n  readonly #lc: LogContext;\n  readonly #instances = new Map<string, S>();\n  readonly #create: (id: string) => S;\n  readonly #isValid: (existing: S) => boolean;\n\n  constructor(\n    lc: LogContext,\n    factory: (id: string) => S,\n    isValid: (existing: S) => boolean = () => true,\n  ) {\n    this.#lc = lc;\n    this.#create = factory;\n    this.#isValid = isValid;\n  }\n\n  /**\n   * Creates and runs the Service with the given `id`, returning\n   * an existing one if it is still running a valid.\n   */\n  getService(id: string): S {\n    const existing = this.#instances.get(id);\n    if (existing && this.#isValid(existing)) {\n      return existing;\n    }\n    const service = this.#create(id);\n    this.#instances.set(id, service);\n    void service\n      .run()\n      .catch(e =>\n        this.#lc.error?.(\n          `Error running ${service.constructor?.name} ${service.id}`,\n          e,\n        ),\n      )\n      .finally(() => {\n        this.#instances.delete(id);\n      });\n    return service;\n  }\n\n  get size() {\n    return this.#instances.size;\n  }\n\n  getServices(): Iterable<S> {\n    return this.#instances.values();\n  }\n}\n"],"mappings":";;;;;AAOA,IAAa,gBAAb,MAA8C;CAC5C;CACA,6BAAsB,IAAI,KAAgB;CAC1C;CACA;CAEA,YACE,IACA,SACA,gBAA0C,MAC1C;AACA,QAAA,KAAW;AACX,QAAA,SAAe;AACf,QAAA,UAAgB;;;;;;CAOlB,WAAW,IAAe;EACxB,MAAM,WAAW,MAAA,UAAgB,IAAI,GAAG;AACxC,MAAI,YAAY,MAAA,QAAc,SAAS,CACrC,QAAO;EAET,MAAM,UAAU,MAAA,OAAa,GAAG;AAChC,QAAA,UAAgB,IAAI,IAAI,QAAQ;AAC3B,UACF,KAAK,CACL,OAAM,MACL,MAAA,GAAS,QACP,iBAAiB,QAAQ,aAAa,KAAK,GAAG,QAAQ,MACtD,EACD,CACF,CACA,cAAc;AACb,SAAA,UAAgB,OAAO,GAAG;IAC1B;AACJ,SAAO;;CAGT,IAAI,OAAO;AACT,SAAO,MAAA,UAAgB;;CAGzB,cAA2B;AACzB,SAAO,MAAA,UAAgB,QAAQ"}