npm - @rocicorp/zero - Versions diffs - 1.10.0-canary.11 → 1.10.0-canary.13 - Mend

@rocicorp/zero 1.10.0-canary.11 → 1.10.0-canary.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (53) hide show

package/out/zero/package.js CHANGED Viewed

@@ -1,6 +1,6 @@
 var package_default = {
 	name: "@rocicorp/zero",
-	version: "1.10.0-canary.11",
+	version: "1.10.0-canary.13",
 	description: "Zero is a web framework for serverless web development.",
 	homepage: "https://zero.rocicorp.dev",
 	bugs: { "url": "https://bugs.rocicorp.dev" },

package/out/zero/package.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.10.0-canary.11\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"license\": \"Apache-2.0\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"bin\": {\n \"analyze-query\": \"./out/zero/src/analyze-query.js\",\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\",\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\"\n },\n \"files\": [\n \"out\",\n \"!.tsbuildinfo\"\n ],\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./analyze\": {\n \"types\": \"./out/zero/src/analyze.d.ts\",\n \"default\": \"./out/zero/src/analyze.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/kysely\": {\n \"types\": \"./out/zero/src/adapters/kysely.d.ts\",\n \"default\": \"./out/zero/src/adapters/kysely.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"oxfmt .\",\n \"check-format\": \"oxfmt --check .\",\n \"lint\": \"oxlint --quiet --config ../../oxlint.config.ts src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\",\n \"fmt\": \"oxfmt .\",\n \"check-fmt\": \"oxfmt --check .\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.3.11\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.39.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.0.0\",\n \"@fastify/websocket\": \"^11.0.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/api-logs\": \"^0.218.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.76.0\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"^0.218.0\",\n \"@opentelemetry/resources\": \"^2.7.1\",\n \"@opentelemetry/sdk-metrics\": \"^2.7.1\",\n \"@opentelemetry/sdk-node\": \"^0.218.0\",\n \"@opentelemetry/sdk-trace-node\": \"^2.7.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.41.1\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^1.0.4\",\n \"@rocicorp/logger\": \"^6.1.0\",\n \"@rocicorp/resolver\": \"^1.0.2\",\n \"@rocicorp/zero-sqlite3\": \"^1.1.4\",\n \"@standard-schema/spec\": \"^1.0.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.5.12\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk-template\": \"^1.1.0\",\n \"chokidar\": \"^4.0.1\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.1\",\n \"command-line-usage\": \"^7.0.3\",\n \"compare-utf8\": \"^0.2.0\",\n \"defu\": \"^6.1.4\",\n \"eventemitter3\": \"^5.0.1\",\n \"fastify\": \"^5.0.0\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.9.3\",\n \"js-xxhash\": \"^4.0.0\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.0\",\n \"nanoid\": \"^5.1.2\",\n \"oxfmt\": \"^0.45.0\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"semver\": \"^7.5.4\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.18.1\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"@vitest/runner\": \"^4.1.6\",\n \"analyze-query\": \"workspace:\",\n \"ast-to-zql\": \"workspace:\",\n \"expo-sqlite\": \">=15\",\n \"oxc-parser\": \"^0.129.0\",\n \"replicache\": \"workspace:\",\n \"shared\": \"workspace:\",\n \"syncpack\": \"^14.3.0\",\n \"typedoc\": \"^0.28.17\",\n \"typedoc-plugin-markdown\": \"^4.10.0\",\n \"typescript\": \"~6.0.2\",\n \"vite\": \"^8.0.13\",\n \"vitest\": \"^4.1.6\",\n \"zero-cache\": \"workspace:\",\n \"zero-client\": \"workspace:\",\n \"zero-pg\": \"workspace:\",\n \"zero-react\": \"workspace:\",\n \"zero-server\": \"workspace:\",\n \"zero-solid\": \"workspace:\",\n \"zqlite\": \"workspace:\"\n },\n \"engines\": {\n \"node\": \">=22\"\n },\n \"packageManager\": \"pnpm@11.11.0\"\n}\n"],"mappings":""}
1	+ {"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.10.0-canary.13\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"license\": \"Apache-2.0\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"bin\": {\n \"analyze-query\": \"./out/zero/src/analyze-query.js\",\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\",\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\"\n },\n \"files\": [\n \"out\",\n \"!.tsbuildinfo\"\n ],\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./analyze\": {\n \"types\": \"./out/zero/src/analyze.d.ts\",\n \"default\": \"./out/zero/src/analyze.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/kysely\": {\n \"types\": \"./out/zero/src/adapters/kysely.d.ts\",\n \"default\": \"./out/zero/src/adapters/kysely.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"oxfmt .\",\n \"check-format\": \"oxfmt --check .\",\n \"lint\": \"oxlint --quiet --config ../../oxlint.config.ts src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\",\n \"fmt\": \"oxfmt .\",\n \"check-fmt\": \"oxfmt --check .\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.3.11\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.39.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.0.0\",\n \"@fastify/websocket\": \"^11.0.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/api-logs\": \"^0.218.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.76.0\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"^0.218.0\",\n \"@opentelemetry/resources\": \"^2.7.1\",\n \"@opentelemetry/sdk-metrics\": \"^2.7.1\",\n \"@opentelemetry/sdk-node\": \"^0.218.0\",\n \"@opentelemetry/sdk-trace-node\": \"^2.7.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.41.1\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^1.0.4\",\n \"@rocicorp/logger\": \"^6.1.0\",\n \"@rocicorp/resolver\": \"^1.0.2\",\n \"@rocicorp/zero-sqlite3\": \"^1.1.4\",\n \"@standard-schema/spec\": \"^1.0.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.5.12\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk-template\": \"^1.1.0\",\n \"chokidar\": \"^4.0.1\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.1\",\n \"command-line-usage\": \"^7.0.3\",\n \"compare-utf8\": \"^0.2.0\",\n \"defu\": \"^6.1.4\",\n \"eventemitter3\": \"^5.0.1\",\n \"fastify\": \"^5.0.0\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.9.3\",\n \"js-xxhash\": \"^4.0.0\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.0\",\n \"nanoid\": \"^5.1.2\",\n \"oxfmt\": \"^0.45.0\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"semver\": \"^7.5.4\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.18.1\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"@vitest/runner\": \"^4.1.6\",\n \"analyze-query\": \"workspace:\",\n \"ast-to-zql\": \"workspace:\",\n \"expo-sqlite\": \">=15\",\n \"oxc-parser\": \"^0.129.0\",\n \"replicache\": \"workspace:\",\n \"shared\": \"workspace:\",\n \"syncpack\": \"^14.3.0\",\n \"typedoc\": \"^0.28.17\",\n \"typedoc-plugin-markdown\": \"^4.10.0\",\n \"typescript\": \"~6.0.2\",\n \"vite\": \"^8.0.13\",\n \"vitest\": \"^4.1.6\",\n \"zero-cache\": \"workspace:\",\n \"zero-client\": \"workspace:\",\n \"zero-pg\": \"workspace:\",\n \"zero-react\": \"workspace:\",\n \"zero-server\": \"workspace:\",\n \"zero-solid\": \"workspace:\",\n \"zqlite\": \"workspace:\"\n },\n \"engines\": {\n \"node\": \">=22\"\n },\n \"packageManager\": \"pnpm@11.11.0\"\n}\n"],"mappings":""}

package/out/zero-cache/src/server/replicator.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAmCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,~~CAqGf~~"}
1	+ {"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAmCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAgB/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAoGf"}

package/out/zero-cache/src/server/replicator.js CHANGED Viewed

@@ -57,7 +57,7 @@ async function runWorker(parent, env, ...args) {
 	const shard = getShardConfig(config);
 	const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
 	const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
-	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, dbPath, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
+	const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
 	setUpMessageHandlers(lc, replicator, parent);
 	const running = runUntilKilled(lc, parent, replicator);
 	for await (const _ of replicator.subscribe()) {

package/out/zero-cache/src/server/replicator.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n LitestreamConfig,\n NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n tryRestore,\n type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n deleteStaleChangeLog,\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const workerName = `${mode}-replicator`;\n startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n lc = createLogContext(config, workerName);\n const unregisterInitialCorruptionDiagnosticTarget =\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Remote view-syncers restore from the replication-manager, which supplies\n // the backup URL. A local change-streamer owns the canonical replica and\n // must not infer backup configuration from bundled executable paths.\n if (\n fileMode === 'serving' &&\n !runningLocalChangeStreamer &&\n (config.litestream.executable \|\| config.litestream.executableV5)\n ) {\n await restoreReplica(lc, config);\n }\n\n const {file: dbPath, walMode} = await setupReplica(\n lc,\n fileMode,\n config.replica,\n );\n unregisterInitialCorruptionDiagnosticTarget();\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath,\n },\n config.sqliteCorruptionChecks,\n );\n\n // A file left behind by a run with the writer enabled would otherwise hold\n // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n // this is keyed on the config flag and nothing else.\n deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n setupMetrics(lc, dbPath, walMode);\n\n // Create the write worker for async SQLite writes.\n const pragmas = getPragmaConfig(fileMode);\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log);\n\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n dbPath,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n const running = runUntilKilled(lc, parent, replicator);\n\n // Signal readiness once the first ReplicaVersionReady notification is received.\n for await (const _ of replicator.subscribe()) {\n parent.send(['ready', {ready: true}]);\n break;\n }\n\n return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n getOrCreateGauge('replica', 'db_size', {\n description:\n `The size of the replica's main db file, ` +\n `which does not include the wal file(s)`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, file));\n\n getOrCreateGauge('replica', 'wal_size', {\n description: `The size of the replica's wal file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n if (walMode === 'wal2') {\n getOrCreateGauge('replica', 'wal2_size', {\n description: `The size of the replica's wal2 file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal2`));\n }\n\n // A whole-database total rather than a per-file one. Its counterpart,\n // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n // the process that writes the log.\n getOrCreateGauge('replica', 'file_bytes', {\n description:\n `The replica's total on-disk footprint: its main db file plus its ` +\n `wal sidecars.`,\n unit: 'By',\n }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n return async o => {\n try {\n const stats = await stat(file);\n o.observe(stats.size);\n } catch (e) {\n lc.warn?.(`unable to stat ${file} for size metrics`, e);\n }\n };\n}\n\nfunction observeSQLiteFileBytes(\n lc: LogContext,\n file: string,\n): ObservableCallback {\n return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n const start = performance.now();\n let backupURL: string \| undefined;\n let result: RestoreResult \| undefined;\n try {\n for (;;) {\n const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n // The backupURL comes from the replication-manager's snapshot response.\n ({backupURL} = snapshotStatus);\n const litestream: LitestreamConfig = {...config.litestream, backupURL};\n const attempt = await tryRestore(\n lc,\n litestream,\n config.replica.file,\n snapshotStatus,\n 'view_syncer',\n );\n if (attempt.restored) {\n result = attempt.result;\n return;\n }\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } finally {\n const attrs = litestreamRestoreMetricAttrs(\n config.litestream,\n 'view_syncer',\n backupURL,\n );\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EAAC,MAAM,QAAQ,YAAW,MAAM,aACpC,IACA,UACA,OAAO,OACT;CACA,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAGhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,GAAG;CAEzD,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,QACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
1	+ {"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {\n LitestreamConfig,\n NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n tryRestore,\n type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n deleteStaleChangeLog,\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const workerName = `${mode}-replicator`;\n startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n lc = createLogContext(config, workerName);\n const unregisterInitialCorruptionDiagnosticTarget =\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n // Remote view-syncers restore from the replication-manager, which supplies\n // the backup URL. A local change-streamer owns the canonical replica and\n // must not infer backup configuration from bundled executable paths.\n if (\n fileMode === 'serving' &&\n !runningLocalChangeStreamer &&\n (config.litestream.executable \|\| config.litestream.executableV5)\n ) {\n await restoreReplica(lc, config);\n }\n\n const {file: dbPath, walMode} = await setupReplica(\n lc,\n fileMode,\n config.replica,\n );\n unregisterInitialCorruptionDiagnosticTarget();\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath,\n },\n config.sqliteCorruptionChecks,\n );\n\n // A file left behind by a run with the writer enabled would otherwise hold\n // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n // this is keyed on the config flag and nothing else.\n deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n setupMetrics(lc, dbPath, walMode);\n\n // Create the write worker for async SQLite writes.\n const pragmas = getPragmaConfig(fileMode);\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log);\n\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n const running = runUntilKilled(lc, parent, replicator);\n\n // Signal readiness once the first ReplicaVersionReady notification is received.\n for await (const _ of replicator.subscribe()) {\n parent.send(['ready', {ready: true}]);\n break;\n }\n\n return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n getOrCreateGauge('replica', 'db_size', {\n description:\n `The size of the replica's main db file, ` +\n `which does not include the wal file(s)`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, file));\n\n getOrCreateGauge('replica', 'wal_size', {\n description: `The size of the replica's wal file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n if (walMode === 'wal2') {\n getOrCreateGauge('replica', 'wal2_size', {\n description: `The size of the replica's wal2 file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal2`));\n }\n\n // A whole-database total rather than a per-file one. Its counterpart,\n // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n // the process that writes the log.\n getOrCreateGauge('replica', 'file_bytes', {\n description:\n `The replica's total on-disk footprint: its main db file plus its ` +\n `wal sidecars.`,\n unit: 'By',\n }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n return async o => {\n try {\n const stats = await stat(file);\n o.observe(stats.size);\n } catch (e) {\n lc.warn?.(`unable to stat ${file} for size metrics`, e);\n }\n };\n}\n\nfunction observeSQLiteFileBytes(\n lc: LogContext,\n file: string,\n): ObservableCallback {\n return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// View-syncers (no replicaConstraints) wait indefinitely for the\n// replication-manager to publish a restorable backup. On a fresh stack the\n// first backup is not durable until the initial sync completes and litestream\n// uploads the initial snapshot, which can take many minutes for a large\n// replica. The platform's startup probe budget (which scales with replica\n// size) is the backstop, so restoreReplica must not impose its own shorter\n// cap and self-terminate while the backup is still being produced.\nasync function restoreReplica(lc: LogContext, config: NormalizedZeroConfig) {\n const start = performance.now();\n let backupURL: string \| undefined;\n let result: RestoreResult \| undefined;\n try {\n for (;;) {\n const snapshotStatus = await reserveAndGetSnapshotStatus(lc, config);\n // The backupURL comes from the replication-manager's snapshot response.\n ({backupURL} = snapshotStatus);\n const litestream: LitestreamConfig = {...config.litestream, backupURL};\n const attempt = await tryRestore(\n lc,\n litestream,\n config.replica.file,\n snapshotStatus,\n 'view_syncer',\n );\n if (attempt.restored) {\n result = attempt.result;\n return;\n }\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } finally {\n const attrs = litestreamRestoreMetricAttrs(\n config.litestream,\n 'view_syncer',\n backupURL,\n );\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAW,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAKxB,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,MAAM,eAAe,IAAI,MAAM;CAGjC,MAAM,EAAC,MAAM,QAAQ,YAAW,MAAM,aACpC,IACA,UACA,OAAO,OACT;CACA,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,OAAO;CAGhC,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,GAAG;CAEzD,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAEA,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,IACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,MAAM,UAAU,eAAe,IAAI,QAAQ,UAAU;CAGrD,WAAW,MAAM,KAAK,WAAW,UAAU,GAAG;EAC5C,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;EACpC;CACF;CAEA,OAAO;AACT;AAEA,SAAS,aAAa,IAAgB,MAAc,SAAkB;CACpE,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,EAAE,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAMpD,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,EAAE,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAS1B,eAAe,eAAe,IAAgB,QAA8B;CAC1E,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,IAAI;EACF,SAAS;GACP,MAAM,iBAAiB,MAAM,4BAA4B,IAAI,MAAM;GAEnE,CAAC,CAAC,aAAa;GAEf,MAAM,UAAU,MAAM,WACpB,IACA;IAHoC,GAAG,OAAO;IAAY;GAG1D,GACA,OAAO,QAAQ,MACf,gBACA,aACF;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB;GACF;GACA,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EAMR,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,EAAE,IAAI,GAAG,MAAM;EACrC,0BAA0B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}

package/out/zero-cache/src/services/analyze.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAC9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AAWxF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAQjE,wBAAsB,YAAY,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,UAAU,UAAO,EACjB,UAAU,UAAQ,EAClB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,IAAI,CAAC,EAAE,OAAO,EACd,SAAS,UAAQ,GAChB,OAAO,CAAC,kBAAkB,CAAC,~~CA2F7B~~"}
1	+ {"version":3,"file":"analyze.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/services/analyze.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,oDAAoD,CAAC;AAC3F,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6CAA6C,CAAC;AAC9E,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,kDAAkD,CAAC;AAWxF,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AAC7C,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,wBAAwB,CAAC;AAQjE,wBAAsB,YAAY,CAChC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,oBAAoB,EAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,GAAG,EACR,UAAU,UAAO,EACjB,UAAU,UAAQ,EAClB,WAAW,CAAC,EAAE,iBAAiB,EAC/B,IAAI,CAAC,EAAE,OAAO,EACd,SAAS,UAAQ,GAChB,OAAO,CAAC,kBAAkB,CAAC,CA4F7B"}

package/out/zero-cache/src/services/analyze.js CHANGED Viewed

@@ -36,6 +36,7 @@ async function analyzeQuery(lc, config, clientSchema, ast, syncedRows = true, ve
 			planDebugger,
 			host: {
 				debug: new Debug(),
+				enableNotExists: true,
 				getSource(tableName) {
 					let source = tables.get(tableName);
 					if (source) return source;

package/out/zero-cache/src/services/analyze.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n AccumulatorDebugger,\n serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n lc: LogContext,\n config: NormalizedZeroConfig,\n clientSchema: ClientSchema,\n ast: AST,\n syncedRows = true,\n vendedRows = false,\n permissions?: PermissionsConfig,\n auth?: JWTAuth,\n joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n using db = new Database(lc, config.replica.file);\n const fullTables = new Map<string, LiteTableSpec>();\n const tableSpecs = new Map<string, LiteAndZqlSpec>();\n const tables = new Map<string, TableSource>();\n\n computeZqlSpecs(\n lc,\n db,\n {includeBackfillingColumns: false},\n tableSpecs,\n fullTables,\n );\n\n // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n // on the server, so the analysis reflects what actually executes. Diagnostic\n // event collection is orthogonal and opt-in via `joinPlans`.\n const costModel = config.enableQueryPlanner\n ? createSQLiteCostModel(db, tableSpecs)\n : undefined;\n const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n const timer = await new TimeSliceTimer(lc).start();\n const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n const yieldProcess = () => timer.yieldProcess();\n const result = await runAst(\n lc,\n clientSchema,\n ast,\n true,\n {\n applyPermissions: permissions !== undefined,\n syncedRows,\n vendedRows,\n auth,\n db,\n tableSpecs,\n permissions,\n costModel,\n planDebugger,\n host: {\n debug: new Debug(),\n getSource(tableName: string) {\n let source = tables.get(tableName);\n if (source) {\n return source;\n }\n\n const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n const {primaryKey} = tableSpec.tableSpec;\n\n source = new TableSource(\n lc,\n config.log,\n db,\n tableName,\n tableSpec.zqlSpec,\n primaryKey,\n shouldYield,\n );\n tables.set(tableName, source);\n return source;\n },\n createStorage() {\n return new MemoryStorage();\n },\n decorateSourceInput: input => input,\n decorateInput: input => input,\n addEdge() {},\n decorateFilterInput: input => input,\n },\n },\n yieldProcess,\n );\n\n // Fill in plans for any queries SQLite did not actually execute (and thus\n // did not populate scanStatus EXPLAIN for) using the substituted-binding\n // fallback. Plans captured at execution time use the real bindings and win.\n const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n const captured = result.sqlitePlans ?? {};\n for (const [query, plan] of Object.entries(fallback)) {\n if (!captured[query]) {\n captured[query] = plan;\n }\n }\n result.sqlitePlans = captured;\n\n if (planDebugger) {\n result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n }\n\n return result;\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,MAAM;IACJ,OAAO,IAAI,MAAM;IACjB,UAAU,WAAmB;KAC3B,IAAI,SAAS,OAAO,IAAI,SAAS;KACjC,IAAI,QACF,OAAO;KAGT,MAAM,YAAY,iBAAiB,YAAY,SAAS;KACxD,MAAM,EAAC,eAAc,UAAU;KAE/B,SAAS,IAAI,YACX,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACA,OAAO,IAAI,WAAW,MAAM;KAC5B,OAAO;IACT;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}
1	+ {"version":3,"file":"analyze.js","names":[],"sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n AccumulatorDebugger,\n serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {JWTAuth} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer} from './view-syncer/view-syncer.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n lc: LogContext,\n config: NormalizedZeroConfig,\n clientSchema: ClientSchema,\n ast: AST,\n syncedRows = true,\n vendedRows = false,\n permissions?: PermissionsConfig,\n auth?: JWTAuth,\n joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n using db = new Database(lc, config.replica.file);\n const fullTables = new Map<string, LiteTableSpec>();\n const tableSpecs = new Map<string, LiteAndZqlSpec>();\n const tables = new Map<string, TableSource>();\n\n computeZqlSpecs(\n lc,\n db,\n {includeBackfillingColumns: false},\n tableSpecs,\n fullTables,\n );\n\n // Mirror production: the planner runs iff ZERO_ENABLE_QUERY_PLANNER is set\n // on the server, so the analysis reflects what actually executes. Diagnostic\n // event collection is orthogonal and opt-in via `joinPlans`.\n const costModel = config.enableQueryPlanner\n ? createSQLiteCostModel(db, tableSpecs)\n : undefined;\n const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n const timer = await new TimeSliceTimer(lc).start();\n const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n const yieldProcess = () => timer.yieldProcess();\n const result = await runAst(\n lc,\n clientSchema,\n ast,\n true,\n {\n applyPermissions: permissions !== undefined,\n syncedRows,\n vendedRows,\n auth,\n db,\n tableSpecs,\n permissions,\n costModel,\n planDebugger,\n host: {\n debug: new Debug(),\n enableNotExists: true,\n getSource(tableName: string) {\n let source = tables.get(tableName);\n if (source) {\n return source;\n }\n\n const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n const {primaryKey} = tableSpec.tableSpec;\n\n source = new TableSource(\n lc,\n config.log,\n db,\n tableName,\n tableSpec.zqlSpec,\n primaryKey,\n shouldYield,\n );\n tables.set(tableName, source);\n return source;\n },\n createStorage() {\n return new MemoryStorage();\n },\n decorateSourceInput: input => input,\n decorateInput: input => input,\n addEdge() {},\n decorateFilterInput: input => input,\n },\n },\n yieldProcess,\n );\n\n // Fill in plans for any queries SQLite did not actually execute (and thus\n // did not populate scanStatus EXPLAIN for) using the substituted-binding\n // fallback. Plans captured at execution time use the real bindings and win.\n const fallback = explainQueries(result.readRowCountsByQuery ?? {}, db);\n const captured = result.sqlitePlans ?? {};\n for (const [query, plan] of Object.entries(fallback)) {\n if (!captured[query]) {\n captured[query] = plan;\n }\n }\n result.sqlitePlans = captured;\n\n if (planDebugger) {\n result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n }\n\n return result;\n}\n"],"mappings":";;;;;;;;;;;;AAsBA,IAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,MACA,YAAY,OACiB;;;EAC7B,MAAM,KAAA,YAAA,EAAK,IAAI,SAAS,IAAI,OAAO,QAAQ,IAAI,CAAA;EAC/C,MAAM,6BAAa,IAAI,IAA2B;EAClD,MAAM,6BAAa,IAAI,IAA4B;EACnD,MAAM,yBAAS,IAAI,IAAyB;EAE5C,gBACE,IACA,IACA,EAAC,2BAA2B,MAAK,GACjC,YACA,UACF;EAKA,MAAM,YAAY,OAAO,qBACrB,sBAAsB,IAAI,UAAU,IACpC,KAAA;EACJ,MAAM,eAAe,YAAY,IAAI,oBAAoB,IAAI,KAAA;EAC7D,MAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAM;EACjD,MAAM,oBAAoB,MAAM,WAAW,IAAI;EAC/C,MAAM,qBAAqB,MAAM,aAAa;EAC9C,MAAM,SAAS,MAAM,OACnB,IACA,cACA,KACA,MACA;GACE,kBAAkB,gBAAgB,KAAA;GAClC;GACA;GACA;GACA;GACA;GACA;GACA;GACA;GACA,MAAM;IACJ,OAAO,IAAI,MAAM;IACjB,iBAAiB;IACjB,UAAU,WAAmB;KAC3B,IAAI,SAAS,OAAO,IAAI,SAAS;KACjC,IAAI,QACF,OAAO;KAGT,MAAM,YAAY,iBAAiB,YAAY,SAAS;KACxD,MAAM,EAAC,eAAc,UAAU;KAE/B,SAAS,IAAI,YACX,IACA,OAAO,KACP,IACA,WACA,UAAU,SACV,YACA,WACF;KACA,OAAO,IAAI,WAAW,MAAM;KAC5B,OAAO;IACT;IACA,gBAAgB;KACd,OAAO,IAAI,cAAc;IAC3B;IACA,sBAAqB,UAAS;IAC9B,gBAAe,UAAS;IACxB,UAAU,CAAC;IACX,sBAAqB,UAAS;GAChC;EACF,GACA,YACF;EAKA,MAAM,WAAW,eAAe,OAAO,wBAAwB,CAAC,GAAG,EAAE;EACrE,MAAM,WAAW,OAAO,eAAe,CAAC;EACxC,KAAK,MAAM,CAAC,OAAO,SAAS,OAAO,QAAQ,QAAQ,GACjD,IAAI,CAAC,SAAS,QACZ,SAAS,SAAS;EAGtB,OAAO,cAAc;EAErB,IAAI,cACF,OAAO,YAAY,yBAAyB,aAAa,MAAM;EAGjE,OAAO;;;;;;AACT"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts CHANGED Viewed

@@ -2,7 +2,17 @@ import type { LogContext } from '@rocicorp/logger';
 import { type JSONFormat } from '../../../types/lite.ts';
 import type { BackfillCompleted, BackfillRequest, ChangeStreamMessage, MessageBackfill } from '../protocol/current.ts';
 import type { Cancelable, ChangeStreamMultiplexer, Listener } from './change-stream-multiplexer.ts';
-type BackfillStreamer = (req: BackfillRequest) => AsyncGenerator<MessageBackfill | BackfillCompleted>;
+/**
+ * A {@link MessageBackfill} or {@link BackfillCompleted} paired with the
+ * approximate number of upstream bytes it represents. The BackfillManager uses
+ * `byteSize` to bound the size of each replica transaction, committing and
+ * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.
+ */
+export type BackfillMessage = {
+    message: MessageBackfill | BackfillCompleted;
+    byteSize: number;
+};
+type BackfillStreamer = (req: BackfillRequest) => AsyncGenerator<BackfillMessage>;
 /**
  * The BackfillManager initiates backfills for BackfillRequests from the
  * change-streamer (i.e. unfinished backfills from previous sessions)
@@ -21,7 +31,7 @@ type BackfillStreamer = (req: BackfillRequest) => AsyncGenerator<MessageBackfill
  */
 export declare class BackfillManager implements Cancelable, Listener {
     #private;
-    constructor(lc: LogContext, changeStreamer: ChangeStreamMultiplexer, backfillStreamer: BackfillStreamer, jsonFormat?: JSONFormat, minBackoffMs?: number, maxBackoffMs?: number);
+    constructor(lc: LogContext, changeStreamer: ChangeStreamMultiplexer, backfillStreamer: BackfillStreamer, jsonFormat?: JSONFormat, minBackoffMs?: number, maxBackoffMs?: number, commitThresholdBytes?: number);
     run(lastWatermark: string, initialRequests: BackfillRequest[]): void;
     /**
      * Implements {@link Listener.onChange()}, invoked by the

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,~~GAAG,iBAAiB,~~CAAC,CAAC;~~AAgBzD~~;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;~~gBA+BxD~~,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B;~~IAWxC~~,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE;~~IAsP7D~~;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IA6M5C,MAAM,IAAI,IAAI;CAIf;AAED,uBAAe,mBAAoB,SAAQ,KAAK;gBAClC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAO9D;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;gBAEjC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAG9D"}
1	+ {"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,OAAO,EAAE,eAAe,GAAG,iBAAiB,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,CAAC,CAAC;AAuBrC;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;gBAiCxD,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B,EACtC,oBAAoB,SAAyB;IAY/C,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE;IA+P7D;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IA6M5C,MAAM,IAAI,IAAI;CAIf;AAED,uBAAe,mBAAoB,SAAQ,KAAK;gBAClC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAO9D;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;gBAEjC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAG9D"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js CHANGED Viewed

@@ -10,6 +10,12 @@ import { resolver } from "@rocicorp/resolver";
 function tableKey({ schema, name }) {
 	return `${schema}.${name}`;
 }
+/**
+* Commit (and reopen) the current backfill transaction once this many bytes of
+* backfill data have accumulated in it. Bounds the size of a single replica
+* COMMIT/checkpoint so it can't monopolize the replica.
+*/
+var COMMIT_THRESHOLD_BYTES = 8 * 1024 * 1024;
 var MIN_BACKOFF_INTERVAL_MS = 2e3;
 var MAX_BACKOFF_INTERVAL_MS = 6e4;
 /**
@@ -50,7 +56,8 @@ var BackfillManager = class {
 	#awaitingStatusWatermarks = [];
 	/** The watermark of the current transaction in the change stream. */
 	#currentTxWatermark = null;
-	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS) {
+	#commitThresholdBytes;
+	constructor(lc, changeStreamer, backfillStreamer, jsonFormat = "s", minBackoffMs = MIN_BACKOFF_INTERVAL_MS, maxBackoffMs = MAX_BACKOFF_INTERVAL_MS, commitThresholdBytes = COMMIT_THRESHOLD_BYTES) {
 		this.#lc = lc.withContext("component", "backfill-manager");
 		this.#changeStreamer = changeStreamer;
 		this.#backfillStreamer = backfillStreamer;
@@ -58,6 +65,7 @@ var BackfillManager = class {
 		this.#minBackoffMs = minBackoffMs;
 		this.#maxBackoffMs = maxBackoffMs;
 		this.#retryDelayMs = minBackoffMs;
+		this.#commitThresholdBytes = commitThresholdBytes;
 	}
 	run(lastWatermark, initialRequests) {
 		this.#lc.info?.(`starting backfill manager with ${initialRequests.length} initial requests`, { requests: initialRequests });
@@ -124,6 +132,7 @@ var BackfillManager = class {
 	async #runBackfill(lc, state) {
 		const changeStream = this.#changeStreamer;
 		let backfillTx = null;
+		let uncommittedBytes = 0;
 		/**
 		* @returns the new tx watermark, or null if backfill was cancelled
 		*/
@@ -165,10 +174,11 @@ var BackfillManager = class {
 				changeStream.release(backfillTx);
 			}
 			backfillTx = null;
+			uncommittedBytes = 0;
 		};
-		for await (const msg of this.#backfillStreamer(state.request)) {
+		for await (const { message: msg, byteSize } of this.#backfillStreamer(state.request)) {
 			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && (this.#changeStreamReached(lc, msg.watermark) || backfillTx !== null && backfillTx < msg.watermark);
-			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)) commitTx();
+			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush || uncommittedBytes >= this.#commitThresholdBytes)) commitTx();
 			mustWaitBeforeFlush && await mustWaitBeforeFlush;
 			if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);
 			if ((backfillTx ??= await beginTxFor(msg)) === null) {
@@ -177,6 +187,7 @@ var BackfillManager = class {
 				return;
 			}
 			await changeStream.push(["data", msg]);
+			uncommittedBytes += byteSize;
 		}
 		backfillTx && commitTx();
 		lc.debug?.(`backfill stream exited`, state.canceledReason ?? "");

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n stateVersionFromString,\n stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n ChangeStreamMessage,\n Identifier,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n Cancelable,\n ChangeStreamMultiplexer,\n Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n return `${schema}.${name}`;\n}\n\ntype BackfillStreamer = (\n req: BackfillRequest,\n) => AsyncGenerator<MessageBackfill \| BackfillCompleted>;\n\ntype RunningBackfillState = {\n request: BackfillRequest;\n canceledReason?: string \| undefined;\n minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n watermark: string;\n reached: () => void;\n};\n\n/*\n The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n \n The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n \n The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n /\nexport class BackfillManager implements Cancelable, Listener {\n readonly #lc: LogContext;\n\n /\n Tracks the metadata of required backfills based on schema changes\n * and initial backfill requests.\n /\n readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n tableKey,\n );\n readonly #changeStreamer: ChangeStreamMultiplexer;\n readonly #backfillStreamer: BackfillStreamer;\n readonly #jsonFormat: JSONFormat;\n\n /\n The current running backfill. The backfill request is always also in\n * `#requiredBackfills` (technically, it can be a subset of what's in\n * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n * upon completion.\n /\n #runningBackfill: RunningBackfillState \| null = null;\n\n /* The last seen watermark in the change stream. /\n #lastStatusWatermark: string \| null = null;\n\n readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n /* The watermark of the current transaction in the change stream. /\n #currentTxWatermark: string \| null = null;\n\n constructor(\n lc: LogContext,\n changeStreamer: ChangeStreamMultiplexer,\n backfillStreamer: BackfillStreamer,\n jsonFormat: JSONFormat = JSON_STRINGIFIED,\n minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n ) {\n this.#lc = lc.withContext('component', 'backfill-manager');\n this.#changeStreamer = changeStreamer;\n this.#backfillStreamer = backfillStreamer;\n this.#jsonFormat = jsonFormat;\n this.#minBackoffMs = minBackoffMs;\n this.#maxBackoffMs = maxBackoffMs;\n this.#retryDelayMs = minBackoffMs;\n }\n\n run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n this.#lc.info?.(\n `starting backfill manager with ${initialRequests.length} initial requests`,\n {requests: initialRequests},\n );\n this.#lastStatusWatermark = lastWatermark;\n initialRequests.forEach(req =>\n this.#setRequiredBackfill('initial-request', req),\n );\n this.#checkAndStartBackfill();\n }\n\n #setLastStatusWatermark({watermark}: {watermark: string}) {\n // Only allow the watermark to move forward. This prevents a backfill\n // transaction (whose watermark is unrelated to change-stream state)\n // from moving the watermark backwards.\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n this.#lastStatusWatermark = watermark;\n for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n const awaiting = this.#awaitingStatusWatermarks[i];\n if (watermark >= awaiting.watermark) {\n awaiting.reached();\n this.#awaitingStatusWatermarks.splice(i, 1);\n }\n }\n }\n }\n\n #changeStreamReached(\n lc: LogContext,\n watermark: string,\n ): Promise<void> \| null {\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n const {promise, resolve: reached} = resolver();\n this.#awaitingStatusWatermarks.push({watermark, reached});\n lc.info?.(\n `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n );\n return promise;\n }\n return null;\n }\n\n readonly #minBackoffMs: number;\n readonly #maxBackoffMs: number;\n #retryDelayMs: number;\n #backfillRetryTimer: NodeJS.Timeout \| undefined;\n\n #checkAndStartBackfill() {\n if (\n !this.#backfillRetryTimer &&\n !this.#runningBackfill &&\n this.#requiredBackfills.size\n ) {\n // Pick a random backfill to avoid head-of-line blocking by a\n // problematic backfill (e.g. awaiting a primary key). This is\n // simpler that adding logic to classify (and declassify)\n // problematic backfills.\n const candidates = [...this.#requiredBackfills.values()];\n const request = candidates[randInt(0, candidates.length - 1)];\n const state = {request, minWatermark: ''};\n const lc = this.#lc.withContext('table', request.table.name);\n\n this.#runningBackfill = state;\n void this.#runBackfill(lc, state)\n .then(() => {\n this.#stopRunningBackfill('backfill exited', state);\n this.#retryDelayMs = this.#minBackoffMs; // reset on success\n })\n // For unexpected errors (e.g. upstream replication slot\n // unavailability), retry with exponential backoff.\n .catch(e => {\n this.#stopRunningBackfill(String(e), state);\n this.#retryBackfillWithBackoff(e);\n });\n }\n }\n\n #retryBackfillWithBackoff(e: unknown) {\n const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n this.#lc[log]?.(\n `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n e,\n );\n this.#backfillRetryTimer = setTimeout(() => {\n this.#backfillRetryTimer = undefined;\n this.#checkAndStartBackfill();\n }, this.#retryDelayMs);\n\n this.#retryDelayMs = Math.min(this.#retryDelayMs 2, this.#maxBackoffMs);\n }\n\n async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n const changeStream = this.#changeStreamer; // Purely for readability\n\n // backfillTx is set if and only if a changeStreamer reservation has been\n // acquired and the backfill stream is inside a transaction.\n let backfillTx: string \| null = null;\n\n /*\n @returns the new tx watermark, or null if backfill was cancelled\n /\n const beginTxFor = async (\n msg: MessageBackfill \| BackfillCompleted,\n ): Promise<string \| null> => {\n assert(backfillTx === null, 'Expected no active backfill transaction');\n const lastWatermark = await changeStream.reserve('backfill');\n\n // After obtaining the changeStream reservation, check if the stream\n // had changes that resulted in invalidating / canceling this backfill.\n if (\n state.canceledReason \|\|\n (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n ) {\n if (state.canceledReason === undefined) {\n assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n this.#stopRunningBackfill(\n `row key change at ${state.minWatermark} ` +\n `postdates backfill watermark at ${msg.watermark}`,\n state,\n );\n }\n changeStream.release(lastWatermark);\n return null;\n }\n\n const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n let tx = stateVersionToString({\n major,\n minor: BigInt(minor) + 1n,\n });\n\n if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n // At this point it must be the case that the #changeStreamReached() the\n // backfill watermark. Given that guarantee, ensure that the version of the\n // transaction containing the backfill-completed message is at least up\n // to the backfill watermark, so that the final database state version is\n // never earlier than the version of any backfilled rows.\n tx = msg.watermark;\n }\n\n void changeStream.push([\n 'begin',\n {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n {commitWatermark: tx},\n ]);\n return (backfillTx = tx);\n };\n\n const commitTx = () => {\n if (backfillTx) {\n void changeStream.push([\n 'commit',\n {tag: 'commit'},\n {watermark: backfillTx},\n ]);\n changeStream.release(backfillTx);\n }\n backfillTx = null;\n };\n\n for await (const msg of this.#backfillStreamer(state.request)) {\n // Before sending `backfill-completed`, the main replication stream\n // may need to catch up, and/or the current transaction may need to be\n // committed to open a new transaction that's up to backfill watermark.\n const mustWaitBeforeFlush =\n msg.tag === 'backfill-completed' &&\n (this.#changeStreamReached(lc, msg.watermark) \|\|\n (backfillTx !== null && backfillTx < msg.watermark));\n\n // If necessary, yield the reservation to the main stream.\n if (\n backfillTx &&\n (changeStream.waiterDelay() > 0 \|\| mustWaitBeforeFlush)\n ) {\n commitTx();\n }\n\n mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n if (\n msg.tag === 'backfill' &&\n msg.rowValues.length > 0 &&\n msg.relation.rowKey.columns.length === 0\n ) {\n throw new MissingRowKeyError(state.request);\n }\n\n // Reserve the changeStreamer if not in a transaction.\n if ((backfillTx ??= await beginTxFor(msg)) === null) {\n lc.info?.(\n `backfill stream canceled: ${state.canceledReason}`,\n state.request,\n );\n this.#checkAndStartBackfill(); // start the next backfill if present\n return; // this backfill is canceled\n }\n\n // `await` to allow the change streamer to exert back pressure\n // on backfills.\n await changeStream.push(['data', msg]);\n }\n\n // Flush any final tx and release the stream.\n backfillTx && commitTx();\n lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n }\n\n #backfillRunningFor(table: Identifier): RunningBackfillState \| null {\n const state = this.#runningBackfill;\n return state?.request.table.schema === table.schema &&\n state.request.table.name === table.name\n ? state\n : null;\n }\n\n /\n Stops the running backfill for the specified `reason`. If `instance` is\n * specified, the running backfill is stopped only if it is that instance.\n * This allows the running backfill itself to clear backfill state without\n * accidentally stopping a different (e.g. subsequent) backfill.\n /\n #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n const backfill = this.#runningBackfill;\n if (backfill && backfill === (instance ?? backfill)) {\n backfill.canceledReason = reason;\n this.#runningBackfill = null;\n reason && this.#lc.info?.(`canceling backfill:`, reason);\n }\n }\n\n #setRequiredBackfill(source: string, req: BackfillRequest) {\n const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.set(req.table, req);\n }\n\n #deleteRequiredBackfill(source: string, id: Identifier) {\n const req = this.#requiredBackfills.get(id);\n if (req) {\n const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.delete(id);\n }\n }\n\n /\n Implements {@link Listener.onChange()}, invoked by the\n * {@link ChangeStreamMultiplexer}.\n /\n onChange(message: ChangeStreamMessage): void {\n if (message[0] === 'begin') {\n this.#currentTxWatermark = message[2].commitWatermark;\n return;\n }\n if (message[0] === 'commit') {\n this.#currentTxWatermark = null;\n this.#setLastStatusWatermark(message[2]);\n // Every commit is a candidate for starting the next backfill\n // (if one is not currently running).\n this.#checkAndStartBackfill();\n return;\n }\n if (message[0] === 'status') {\n this.#setLastStatusWatermark(message[2]);\n return;\n }\n if (message[0] !== 'data') {\n return;\n }\n const change = message[1];\n const {tag} = change;\n switch (tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest) {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n });\n if (this.#backfillRunningFor(table)) {\n this.#stopRunningBackfill(`TableMetadata updated`);\n }\n }\n break;\n }\n case 'create-table': {\n const {\n spec: {schema, name},\n metadata = null,\n backfill,\n } = change;\n\n if (backfill) {\n this.#setRequiredBackfill(tag, {\n table: {schema, name, metadata},\n columns: backfill,\n });\n }\n break;\n }\n case 'rename-table': {\n const {old, new: newTable} = change;\n const backfillRequest = this.#requiredBackfills.get(old);\n if (backfillRequest) {\n const {schema, name} = newTable;\n this.#deleteRequiredBackfill(tag, old);\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, schema, name},\n });\n if (this.#backfillRunningFor(old)) {\n this.#stopRunningBackfill(`table renamed`);\n }\n }\n break;\n }\n case 'drop-table': {\n const {id} = change;\n const backfillRequest = this.#requiredBackfills.get(id);\n if (backfillRequest) {\n this.#deleteRequiredBackfill(tag, id);\n if (this.#backfillRunningFor(id)) {\n this.#stopRunningBackfill(`table dropped`);\n }\n }\n break;\n }\n case 'add-column': {\n const {\n table,\n tableMetadata: metadata = null,\n column,\n backfill,\n } = change;\n if (backfill) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (!backfillRequest) {\n this.#setRequiredBackfill(tag, {\n table: {...table, metadata},\n columns: {[column.name]: backfill},\n });\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n columns: {\n ...backfillRequest.columns,\n [column.name]: backfill,\n },\n });\n // Note: The running backfill need not be canceled if a\n // new column is added. The new column will be backfilled\n // by its own stream after the current backfill completes.\n }\n }\n break;\n }\n case 'update-column': {\n const {\n table,\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && oldName in backfillRequest.columns) {\n const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: {...otherCols, [newName]: colSpec},\n });\n const backfill = this.#backfillRunningFor(table);\n if (backfill && oldName in backfill.request.columns) {\n this.#stopRunningBackfill(`column renamed`);\n }\n }\n }\n break;\n }\n case 'drop-column': {\n const {table, column} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && column in backfillRequest.columns) {\n const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n if (Object.keys(remaining).length === 0) {\n this.#deleteRequiredBackfill(tag, table);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: remaining,\n });\n }\n const backfill = this.#backfillRunningFor(table);\n if (backfill && column in backfill.request.columns) {\n this.#stopRunningBackfill(`column dropped`);\n }\n }\n break;\n }\n case 'update': {\n const {relation, key, new: row} = change;\n const backfill = this.#backfillRunningFor(relation);\n const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n if (backfill?.request.table.metadata && key !== null) {\n // A corner case that backfill is unable to correctly handle is\n // when a row's key changes; this is decomposed into a delete\n // of the old key and a set of the new key in the replica change\n // log, at which point the backfill algorithm assumes that the\n // (old) row is deleted but does not know to backfill the new row.\n // In these corner cases, the current backfill is canceled and\n // retried if its version precedes this update.\n for (const col of Object.keys(\n backfill.request.table.metadata.rowKey,\n )) {\n if (key[col] !== row[col]) {\n backfill.minWatermark = txWatermark;\n this.#lc.info?.(\n `key for row as changed (col: ${col}). ` +\n `backfill data must not predate ${backfill.minWatermark}`,\n );\n break;\n }\n }\n }\n break;\n }\n case 'backfill-completed': {\n const {relation, columns} = change;\n const backfillRequest = this.#requiredBackfills.get(relation);\n assert(\n backfillRequest,\n () => `No BackfillRequest completed backfill ${stringify(change)}`,\n );\n const remaining = Object.entries(backfillRequest.columns).filter(\n ([col]) =>\n !(columns.includes(col) \|\| relation.rowKey.columns.includes(col)),\n );\n if (remaining.length === 0) {\n this.#deleteRequiredBackfill(tag, relation);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: Object.fromEntries(remaining),\n });\n }\n // Technically the backfill is already stopping, but this method\n // cleans up the state that tracks it.\n this.#stopRunningBackfill();\n break;\n }\n }\n }\n\n cancel(): void {\n this.#stopRunningBackfill(`change stream canceled`);\n clearTimeout(this.#backfillRetryTimer);\n }\n}\n\nabstract class BackfillStreamError extends Error {\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(\n `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n {cause},\n );\n }\n}\n\n/\n Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n \n In terms of backfill, however, non-empty tables without a row key cannot\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n \n The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n /\nclass MissingRowKeyError extends BackfillStreamError {\n readonly name = 'MissingRowKeyError';\n\n constructor(bf: BackfillRequest, cause?: unknown) {\n super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n }\n}\n\n/\n Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n readonly name = 'SchemaIncompatibilityError';\n\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(bf, msg, cause);\n }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;AAYA,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKE,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;CACvB;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKR,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKS,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKJ,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKI,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKJ,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKI,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKG,uBACN,CAAC,KAAKC,oBACN,KAAKZ,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKa,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,MAAM,MAAM,KAAKR,kBAAkB,KAAKD,gBAAgB,UAAU;EAClE,KAAKP,IAAI,OACP,uCAAuC,KAAKQ,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKZ;EAI1B,IAAI,aAA4B;;;;EAKhC,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKa,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKX;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;GACjC;GACA,aAAa;EACf;EAEA,WAAW,MAAM,OAAO,KAAKD,kBAAkB,MAAM,OAAO,GAAG;GAI7D,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKc,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;GAG7C,IACE,eACC,aAAa,YAAY,IAAI,KAAK,sBAEnC,SAAS;GAGX,uBAAwB,MAAM;GAE9B,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;GAI5C,KAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;IACnD,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;IACA,KAAKN,uBAAuB;IAC5B;GACF;GAIA,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;EACvC;EAGA,cAAc,SAAS;EACvB,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;CACjE;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKE;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKb,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKiB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKR,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKQ,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKlB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKS,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKU,oBAAoB,KAAK,GAChC,KAAKL,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKT,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKoB,wBAAwB,KAAK,GAAG;KACrC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKU,oBAAoB,GAAG,GAC9B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKd,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKoB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKd,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKS,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKT,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKS,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKU,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKL,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKd,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKoB,wBAAwB,KAAK,KAAK;UAEvC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKU,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKL,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKlB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKoB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKX,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKA,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;CACvC;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}
1	+ {"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#commitThresholdBytes","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n stateVersionFromString,\n stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n ChangeStreamMessage,\n Identifier,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n Cancelable,\n ChangeStreamMultiplexer,\n Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n return `${schema}.${name}`;\n}\n\n/*\n A {@link MessageBackfill} or {@link BackfillCompleted} paired with the\n * approximate number of upstream bytes it represents. The BackfillManager uses\n * `byteSize` to bound the size of each replica transaction, committing and\n * reopening once {@link COMMIT_THRESHOLD_BYTES} is reached.\n /\nexport type BackfillMessage = {\n message: MessageBackfill \| BackfillCompleted;\n byteSize: number;\n};\n\ntype BackfillStreamer = (\n req: BackfillRequest,\n) => AsyncGenerator<BackfillMessage>;\n\n/\n Commit (and reopen) the current backfill transaction once this many bytes of\n * backfill data have accumulated in it. Bounds the size of a single replica\n * COMMIT/checkpoint so it can't monopolize the replica.\n /\nconst COMMIT_THRESHOLD_BYTES = 8 1024 * 1024;\n\ntype RunningBackfillState = {\n request: BackfillRequest;\n canceledReason?: string \| undefined;\n minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n watermark: string;\n reached: () => void;\n};\n\n/*\n The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n \n The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n \n The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n /\nexport class BackfillManager implements Cancelable, Listener {\n readonly #lc: LogContext;\n\n /\n Tracks the metadata of required backfills based on schema changes\n * and initial backfill requests.\n /\n readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n tableKey,\n );\n readonly #changeStreamer: ChangeStreamMultiplexer;\n readonly #backfillStreamer: BackfillStreamer;\n readonly #jsonFormat: JSONFormat;\n\n /\n The current running backfill. The backfill request is always also in\n * `#requiredBackfills` (technically, it can be a subset of what's in\n * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n * upon completion.\n /\n #runningBackfill: RunningBackfillState \| null = null;\n\n /* The last seen watermark in the change stream. /\n #lastStatusWatermark: string \| null = null;\n\n readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n /* The watermark of the current transaction in the change stream. /\n #currentTxWatermark: string \| null = null;\n\n readonly #commitThresholdBytes: number;\n\n constructor(\n lc: LogContext,\n changeStreamer: ChangeStreamMultiplexer,\n backfillStreamer: BackfillStreamer,\n jsonFormat: JSONFormat = JSON_STRINGIFIED,\n minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n commitThresholdBytes = COMMIT_THRESHOLD_BYTES,\n ) {\n this.#lc = lc.withContext('component', 'backfill-manager');\n this.#changeStreamer = changeStreamer;\n this.#backfillStreamer = backfillStreamer;\n this.#jsonFormat = jsonFormat;\n this.#minBackoffMs = minBackoffMs;\n this.#maxBackoffMs = maxBackoffMs;\n this.#retryDelayMs = minBackoffMs;\n this.#commitThresholdBytes = commitThresholdBytes;\n }\n\n run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n this.#lc.info?.(\n `starting backfill manager with ${initialRequests.length} initial requests`,\n {requests: initialRequests},\n );\n this.#lastStatusWatermark = lastWatermark;\n initialRequests.forEach(req =>\n this.#setRequiredBackfill('initial-request', req),\n );\n this.#checkAndStartBackfill();\n }\n\n #setLastStatusWatermark({watermark}: {watermark: string}) {\n // Only allow the watermark to move forward. This prevents a backfill\n // transaction (whose watermark is unrelated to change-stream state)\n // from moving the watermark backwards.\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n this.#lastStatusWatermark = watermark;\n for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n const awaiting = this.#awaitingStatusWatermarks[i];\n if (watermark >= awaiting.watermark) {\n awaiting.reached();\n this.#awaitingStatusWatermarks.splice(i, 1);\n }\n }\n }\n }\n\n #changeStreamReached(\n lc: LogContext,\n watermark: string,\n ): Promise<void> \| null {\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n const {promise, resolve: reached} = resolver();\n this.#awaitingStatusWatermarks.push({watermark, reached});\n lc.info?.(\n `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n );\n return promise;\n }\n return null;\n }\n\n readonly #minBackoffMs: number;\n readonly #maxBackoffMs: number;\n #retryDelayMs: number;\n #backfillRetryTimer: NodeJS.Timeout \| undefined;\n\n #checkAndStartBackfill() {\n if (\n !this.#backfillRetryTimer &&\n !this.#runningBackfill &&\n this.#requiredBackfills.size\n ) {\n // Pick a random backfill to avoid head-of-line blocking by a\n // problematic backfill (e.g. awaiting a primary key). This is\n // simpler that adding logic to classify (and declassify)\n // problematic backfills.\n const candidates = [...this.#requiredBackfills.values()];\n const request = candidates[randInt(0, candidates.length - 1)];\n const state = {request, minWatermark: ''};\n const lc = this.#lc.withContext('table', request.table.name);\n\n this.#runningBackfill = state;\n void this.#runBackfill(lc, state)\n .then(() => {\n this.#stopRunningBackfill('backfill exited', state);\n this.#retryDelayMs = this.#minBackoffMs; // reset on success\n })\n // For unexpected errors (e.g. upstream replication slot\n // unavailability), retry with exponential backoff.\n .catch(e => {\n this.#stopRunningBackfill(String(e), state);\n this.#retryBackfillWithBackoff(e);\n });\n }\n }\n\n #retryBackfillWithBackoff(e: unknown) {\n const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n this.#lc[log]?.(\n `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n e,\n );\n this.#backfillRetryTimer = setTimeout(() => {\n this.#backfillRetryTimer = undefined;\n this.#checkAndStartBackfill();\n }, this.#retryDelayMs);\n\n this.#retryDelayMs = Math.min(this.#retryDelayMs 2, this.#maxBackoffMs);\n }\n\n async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n const changeStream = this.#changeStreamer; // Purely for readability\n\n // backfillTx is set if and only if a changeStreamer reservation has been\n // acquired and the backfill stream is inside a transaction.\n let backfillTx: string \| null = null;\n let uncommittedBytes = 0;\n\n /*\n @returns the new tx watermark, or null if backfill was cancelled\n /\n const beginTxFor = async (\n msg: MessageBackfill \| BackfillCompleted,\n ): Promise<string \| null> => {\n assert(backfillTx === null, 'Expected no active backfill transaction');\n const lastWatermark = await changeStream.reserve('backfill');\n\n // After obtaining the changeStream reservation, check if the stream\n // had changes that resulted in invalidating / canceling this backfill.\n if (\n state.canceledReason \|\|\n (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n ) {\n if (state.canceledReason === undefined) {\n assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n this.#stopRunningBackfill(\n `row key change at ${state.minWatermark} ` +\n `postdates backfill watermark at ${msg.watermark}`,\n state,\n );\n }\n changeStream.release(lastWatermark);\n return null;\n }\n\n const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n let tx = stateVersionToString({\n major,\n minor: BigInt(minor) + 1n,\n });\n\n if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n // At this point it must be the case that the #changeStreamReached() the\n // backfill watermark. Given that guarantee, ensure that the version of the\n // transaction containing the backfill-completed message is at least up\n // to the backfill watermark, so that the final database state version is\n // never earlier than the version of any backfilled rows.\n tx = msg.watermark;\n }\n\n void changeStream.push([\n 'begin',\n {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n {commitWatermark: tx},\n ]);\n return (backfillTx = tx);\n };\n\n const commitTx = () => {\n if (backfillTx) {\n void changeStream.push([\n 'commit',\n {tag: 'commit'},\n {watermark: backfillTx},\n ]);\n changeStream.release(backfillTx);\n }\n backfillTx = null;\n uncommittedBytes = 0;\n };\n\n for await (const {message: msg, byteSize} of this.#backfillStreamer(\n state.request,\n )) {\n // Before sending `backfill-completed`, the main replication stream\n // may need to catch up, and/or the current transaction may need to be\n // committed to open a new transaction that's up to backfill watermark.\n const mustWaitBeforeFlush =\n msg.tag === 'backfill-completed' &&\n (this.#changeStreamReached(lc, msg.watermark) \|\|\n (backfillTx !== null && backfillTx < msg.watermark));\n\n // Commit (and later reopen) the transaction if the main stream is\n // waiting on the reservation, if we must catch up before completing, or\n // if the size of current transaction has reached the commit threshold.\n if (\n backfillTx &&\n (changeStream.waiterDelay() > 0 \|\|\n mustWaitBeforeFlush \|\|\n uncommittedBytes >= this.#commitThresholdBytes)\n ) {\n commitTx();\n }\n\n mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n if (\n msg.tag === 'backfill' &&\n msg.rowValues.length > 0 &&\n msg.relation.rowKey.columns.length === 0\n ) {\n throw new MissingRowKeyError(state.request);\n }\n\n // Reserve the changeStreamer if not in a transaction.\n if ((backfillTx ??= await beginTxFor(msg)) === null) {\n lc.info?.(\n `backfill stream canceled: ${state.canceledReason}`,\n state.request,\n );\n this.#checkAndStartBackfill(); // start the next backfill if present\n return; // this backfill is canceled\n }\n\n // `await` to allow the change streamer to exert back pressure\n // on backfills.\n await changeStream.push(['data', msg]);\n uncommittedBytes += byteSize;\n }\n\n // Flush any final tx and release the stream.\n backfillTx && commitTx();\n lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n }\n\n #backfillRunningFor(table: Identifier): RunningBackfillState \| null {\n const state = this.#runningBackfill;\n return state?.request.table.schema === table.schema &&\n state.request.table.name === table.name\n ? state\n : null;\n }\n\n /\n Stops the running backfill for the specified `reason`. If `instance` is\n * specified, the running backfill is stopped only if it is that instance.\n * This allows the running backfill itself to clear backfill state without\n * accidentally stopping a different (e.g. subsequent) backfill.\n /\n #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n const backfill = this.#runningBackfill;\n if (backfill && backfill === (instance ?? backfill)) {\n backfill.canceledReason = reason;\n this.#runningBackfill = null;\n reason && this.#lc.info?.(`canceling backfill:`, reason);\n }\n }\n\n #setRequiredBackfill(source: string, req: BackfillRequest) {\n const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.set(req.table, req);\n }\n\n #deleteRequiredBackfill(source: string, id: Identifier) {\n const req = this.#requiredBackfills.get(id);\n if (req) {\n const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.delete(id);\n }\n }\n\n /\n Implements {@link Listener.onChange()}, invoked by the\n * {@link ChangeStreamMultiplexer}.\n /\n onChange(message: ChangeStreamMessage): void {\n if (message[0] === 'begin') {\n this.#currentTxWatermark = message[2].commitWatermark;\n return;\n }\n if (message[0] === 'commit') {\n this.#currentTxWatermark = null;\n this.#setLastStatusWatermark(message[2]);\n // Every commit is a candidate for starting the next backfill\n // (if one is not currently running).\n this.#checkAndStartBackfill();\n return;\n }\n if (message[0] === 'status') {\n this.#setLastStatusWatermark(message[2]);\n return;\n }\n if (message[0] !== 'data') {\n return;\n }\n const change = message[1];\n const {tag} = change;\n switch (tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest) {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n });\n if (this.#backfillRunningFor(table)) {\n this.#stopRunningBackfill(`TableMetadata updated`);\n }\n }\n break;\n }\n case 'create-table': {\n const {\n spec: {schema, name},\n metadata = null,\n backfill,\n } = change;\n\n if (backfill) {\n this.#setRequiredBackfill(tag, {\n table: {schema, name, metadata},\n columns: backfill,\n });\n }\n break;\n }\n case 'rename-table': {\n const {old, new: newTable} = change;\n const backfillRequest = this.#requiredBackfills.get(old);\n if (backfillRequest) {\n const {schema, name} = newTable;\n this.#deleteRequiredBackfill(tag, old);\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, schema, name},\n });\n if (this.#backfillRunningFor(old)) {\n this.#stopRunningBackfill(`table renamed`);\n }\n }\n break;\n }\n case 'drop-table': {\n const {id} = change;\n const backfillRequest = this.#requiredBackfills.get(id);\n if (backfillRequest) {\n this.#deleteRequiredBackfill(tag, id);\n if (this.#backfillRunningFor(id)) {\n this.#stopRunningBackfill(`table dropped`);\n }\n }\n break;\n }\n case 'add-column': {\n const {\n table,\n tableMetadata: metadata = null,\n column,\n backfill,\n } = change;\n if (backfill) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (!backfillRequest) {\n this.#setRequiredBackfill(tag, {\n table: {...table, metadata},\n columns: {[column.name]: backfill},\n });\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n columns: {\n ...backfillRequest.columns,\n [column.name]: backfill,\n },\n });\n // Note: The running backfill need not be canceled if a\n // new column is added. The new column will be backfilled\n // by its own stream after the current backfill completes.\n }\n }\n break;\n }\n case 'update-column': {\n const {\n table,\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && oldName in backfillRequest.columns) {\n const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: {...otherCols, [newName]: colSpec},\n });\n const backfill = this.#backfillRunningFor(table);\n if (backfill && oldName in backfill.request.columns) {\n this.#stopRunningBackfill(`column renamed`);\n }\n }\n }\n break;\n }\n case 'drop-column': {\n const {table, column} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && column in backfillRequest.columns) {\n const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n if (Object.keys(remaining).length === 0) {\n this.#deleteRequiredBackfill(tag, table);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: remaining,\n });\n }\n const backfill = this.#backfillRunningFor(table);\n if (backfill && column in backfill.request.columns) {\n this.#stopRunningBackfill(`column dropped`);\n }\n }\n break;\n }\n case 'update': {\n const {relation, key, new: row} = change;\n const backfill = this.#backfillRunningFor(relation);\n const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n if (backfill?.request.table.metadata && key !== null) {\n // A corner case that backfill is unable to correctly handle is\n // when a row's key changes; this is decomposed into a delete\n // of the old key and a set of the new key in the replica change\n // log, at which point the backfill algorithm assumes that the\n // (old) row is deleted but does not know to backfill the new row.\n // In these corner cases, the current backfill is canceled and\n // retried if its version precedes this update.\n for (const col of Object.keys(\n backfill.request.table.metadata.rowKey,\n )) {\n if (key[col] !== row[col]) {\n backfill.minWatermark = txWatermark;\n this.#lc.info?.(\n `key for row as changed (col: ${col}). ` +\n `backfill data must not predate ${backfill.minWatermark}`,\n );\n break;\n }\n }\n }\n break;\n }\n case 'backfill-completed': {\n const {relation, columns} = change;\n const backfillRequest = this.#requiredBackfills.get(relation);\n assert(\n backfillRequest,\n () => `No BackfillRequest completed backfill ${stringify(change)}`,\n );\n const remaining = Object.entries(backfillRequest.columns).filter(\n ([col]) =>\n !(columns.includes(col) \|\| relation.rowKey.columns.includes(col)),\n );\n if (remaining.length === 0) {\n this.#deleteRequiredBackfill(tag, relation);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: Object.fromEntries(remaining),\n });\n }\n // Technically the backfill is already stopping, but this method\n // cleans up the state that tracks it.\n this.#stopRunningBackfill();\n break;\n }\n }\n }\n\n cancel(): void {\n this.#stopRunningBackfill(`change stream canceled`);\n clearTimeout(this.#backfillRetryTimer);\n }\n}\n\nabstract class BackfillStreamError extends Error {\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(\n `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n {cause},\n );\n }\n}\n\n/\n Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n \n In terms of backfill, however, non-empty tables without a row key cannot\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n \n The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n /\nclass MissingRowKeyError extends BackfillStreamError {\n readonly name = 'MissingRowKeyError';\n\n constructor(bf: BackfillRequest, cause?: unknown) {\n super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n }\n}\n\n/\n Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n readonly name = 'SchemaIncompatibilityError';\n\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(bf, msg, cause);\n }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;CAC5C,OAAO,GAAG,OAAO,GAAG;AACtB;;;;;;AAsBA,IAAM,yBAAyB,IAAI,OAAO;AAQ1C,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,QACF;CACA;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,CAAC;;CAGjE,sBAAqC;CAErC;CAEA,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf,uBAAuB,wBACvB;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kBAAkB;EACzD,KAAKE,kBAAkB;EACvB,KAAKC,oBAAoB;EACzB,KAAKC,cAAc;EACnB,KAAKG,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKC,gBAAgB;EACrB,KAAKH,wBAAwB;CAC/B;CAEA,IAAI,eAAuB,iBAAoC;EAC7D,KAAKN,IAAI,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,gBAAe,CAC5B;EACA,KAAKU,uBAAuB;EAC5B,gBAAgB,SAAQ,QACtB,KAAKC,qBAAqB,mBAAmB,GAAG,CAClD;EACA,KAAKC,uBAAuB;CAC9B;CAEA,wBAAwB,EAAC,aAAiC;EAIxD,KAAK,KAAKF,wBAAwB,MAAM,WAAW;GACjD,KAAKA,uBAAuB;GAC5B,KAAK,IAAI,IAAI,KAAKL,0BAA0B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,KAAKA,0BAA0B;IAChD,IAAI,aAAa,SAAS,WAAW;KACnC,SAAS,QAAQ;KACjB,KAAKA,0BAA0B,OAAO,GAAG,CAAC;IAC5C;GACF;EACF;CACF;CAEA,qBACE,IACA,WACsB;EACtB,KAAK,KAAKK,wBAAwB,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,SAAS;GAC7C,KAAKL,0BAA0B,KAAK;IAAC;IAAW;GAAO,CAAC;GACxD,GAAG,OACD,iCAAiC,KAAKK,qBAAqB,aAAa,WAC1E;GACA,OAAO;EACT;EACA,OAAO;CACT;CAEA;CACA;CACA;CACA;CAEA,yBAAyB;EACvB,IACE,CAAC,KAAKG,uBACN,CAAC,KAAKC,oBACN,KAAKb,mBAAmB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,KAAKA,mBAAmB,OAAO,CAAC;GACvD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,CAAC;GAC3D,MAAM,QAAQ;IAAC;IAAS,cAAc;GAAE;GACxC,MAAM,KAAK,KAAKD,IAAI,YAAY,SAAS,QAAQ,MAAM,IAAI;GAE3D,KAAKc,mBAAmB;GACxB,KAAUC,aAAa,IAAI,KAAK,EAC7B,WAAW;IACV,KAAKC,qBAAqB,mBAAmB,KAAK;IAClD,KAAKP,gBAAgB,KAAKF;GAC5B,CAAC,EAGA,OAAM,MAAK;IACV,KAAKS,qBAAqB,OAAO,CAAC,GAAG,KAAK;IAC1C,KAAKC,0BAA0B,CAAC;GAClC,CAAC;EACL;CACF;CAEA,0BAA0B,GAAY;EACpC,MAAM,MAAM,KAAKR,kBAAkB,KAAKD,gBAAgB,UAAU;EAClE,KAAKR,IAAI,OACP,uCAAuC,KAAKS,cAAc,MAC1D,CACF;EACA,KAAKI,sBAAsB,iBAAiB;GAC1C,KAAKA,sBAAsB,KAAA;GAC3B,KAAKD,uBAAuB;EAC9B,GAAG,KAAKH,aAAa;EAErB,KAAKA,gBAAgB,KAAK,IAAI,KAAKA,gBAAgB,GAAG,KAAKD,aAAa;CAC1E;CAEA,MAAMO,aAAa,IAAgB,OAA6B;EAC9D,MAAM,eAAe,KAAKb;EAI1B,IAAI,aAA4B;EAChC,IAAI,mBAAmB;;;;EAKvB,MAAM,aAAa,OACjB,QAC2B;GAC3B,OAAO,eAAe,MAAM,yCAAyC;GACrE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,UAAU;GAI3D,IACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;IACA,IAAI,MAAM,mBAAmB,KAAA,GAAW;KACtC,OAAO,IAAI,QAAQ,YAAY,+BAA+B;KAC9D,KAAKc,qBACH,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,KACF;IACF;IACA,aAAa,QAAQ,aAAa;IAClC,OAAO;GACT;GAEA,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,aAAa;GAChE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,KAAK,IAAI;GACzB,CAAC;GAED,IAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,WAM/C,KAAK,IAAI;GAGX,aAAkB,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,KAAKZ;KAAa,SAAS;IAAI;IACpD,EAAC,iBAAiB,GAAE;GACtB,CAAC;GACD,OAAQ,aAAa;EACvB;EAEA,MAAM,iBAAiB;GACrB,IAAI,YAAY;IACd,aAAkB,KAAK;KACrB;KACA,EAAC,KAAK,SAAQ;KACd,EAAC,WAAW,WAAU;IACxB,CAAC;IACD,aAAa,QAAQ,UAAU;GACjC;GACA,aAAa;GACb,mBAAmB;EACrB;EAEA,WAAW,MAAM,EAAC,SAAS,KAAK,cAAa,KAAKD,kBAChD,MAAM,OACR,GAAG;GAID,MAAM,sBACJ,IAAI,QAAQ,yBACX,KAAKe,qBAAqB,IAAI,IAAI,SAAS,KACzC,eAAe,QAAQ,aAAa,IAAI;GAK7C,IACE,eACC,aAAa,YAAY,IAAI,KAC5B,uBACA,oBAAoB,KAAKZ,wBAE3B,SAAS;GAGX,uBAAwB,MAAM;GAE9B,IACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,GAEvC,MAAM,IAAI,mBAAmB,MAAM,OAAO;GAI5C,KAAK,eAAe,MAAM,WAAW,GAAG,OAAO,MAAM;IACnD,GAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,OACR;IACA,KAAKM,uBAAuB;IAC5B;GACF;GAIA,MAAM,aAAa,KAAK,CAAC,QAAQ,GAAG,CAAC;GACrC,oBAAoB;EACtB;EAGA,cAAc,SAAS;EACvB,GAAG,QAAQ,0BAA0B,MAAM,kBAAkB,EAAE;CACjE;CAEA,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,KAAKE;EACnB,OAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;CACN;;;;;;;CAQA,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,KAAKA;EACtB,IAAI,YAAY,cAAc,YAAY,WAAW;GACnD,SAAS,iBAAiB;GAC1B,KAAKA,mBAAmB;GACxB,UAAU,KAAKd,IAAI,OAAO,uBAAuB,MAAM;EACzD;CACF;CAEA,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,KAAKC,mBAAmB,IAAI,IAAI,KAAK,IAAI,YAAY;EACpE,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;EAChE,KAAKC,mBAAmB,IAAI,IAAI,OAAO,GAAG;CAC5C;CAEA,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,KAAKA,mBAAmB,IAAI,EAAE;EAC1C,IAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;GAC/D,KAAKD,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,IAAG,CAAC;GAChE,KAAKC,mBAAmB,OAAO,EAAE;EACnC;CACF;;;;;CAMA,SAAS,SAAoC;EAC3C,IAAI,QAAQ,OAAO,SAAS;GAC1B,KAAKkB,sBAAsB,QAAQ,GAAG;GACtC;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKA,sBAAsB;GAC3B,KAAKC,wBAAwB,QAAQ,EAAE;GAGvC,KAAKR,uBAAuB;GAC5B;EACF;EACA,IAAI,QAAQ,OAAO,UAAU;GAC3B,KAAKQ,wBAAwB,QAAQ,EAAE;GACvC;EACF;EACA,IAAI,QAAQ,OAAO,QACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;EACd,QAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,KAAKnB,mBAAmB,IAAI,KAAK;IACzD,IAAI,iBAAiB;KACnB,KAAKU,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;KAC5C,CAAC;KACD,IAAI,KAAKU,oBAAoB,KAAK,GAChC,KAAKL,qBAAqB,uBAAuB;IAErD;IACA;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;IAEJ,IAAI,UACF,KAAKL,qBAAqB,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;KAAQ;KAC9B,SAAS;IACX,CAAC;IAEH;GACF;GACA,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,GAAG;IACvD,IAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;KACvB,KAAKqB,wBAAwB,KAAK,GAAG;KACrC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;MAAI;KAChD,CAAC;KACD,IAAI,KAAKU,oBAAoB,GAAG,GAC9B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;IAEb,IADwB,KAAKf,mBAAmB,IAAI,EAChD,GAAiB;KACnB,KAAKqB,wBAAwB,KAAK,EAAE;KACpC,IAAI,KAAKD,oBAAoB,EAAE,GAC7B,KAAKL,qBAAqB,eAAe;IAE7C;IACA;GACF;GACA,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;IACJ,IAAI,UAAU;KACZ,MAAM,kBAAkB,KAAKf,mBAAmB,IAAI,KAAK;KACzD,IAAI,CAAC,iBACH,KAAKU,qBAAqB,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;MAAQ;MAC1B,SAAS,GAAE,OAAO,OAAO,SAAQ;KACnC,CAAC;UAED,KAAKA,qBAAqB,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;MAAQ;MAC1C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;MACjB;KACF,CAAC;IAKL;IACA;GACF;GACA,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;IACJ,IAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,KAAKV,mBAAmB,IAAI,KAAK;KACzD,IAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;MAC3D,KAAKU,qBAAqB,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;OAAO;MAC5C,CAAC;MACD,MAAM,WAAW,KAAKU,oBAAoB,KAAK;MAC/C,IAAI,YAAY,WAAW,SAAS,QAAQ,SAC1C,KAAKL,qBAAqB,gBAAgB;KAE9C;IACF;IACA;GACF;GACA,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,KAAKf,mBAAmB,IAAI,KAAK;IACzD,IAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;KAC5D,IAAI,OAAO,KAAK,SAAS,EAAE,WAAW,GACpC,KAAKqB,wBAAwB,KAAK,KAAK;UAEvC,KAAKX,qBAAqB,KAAK;MAC7B,GAAG;MACH,SAAS;KACX,CAAC;KAEH,MAAM,WAAW,KAAKU,oBAAoB,KAAK;KAC/C,IAAI,YAAY,UAAU,SAAS,QAAQ,SACzC,KAAKL,qBAAqB,gBAAgB;IAE9C;IACA;GACF;GACA,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,KAAKK,oBAAoB,QAAQ;IAClD,MAAM,cAAc,KAAK,KAAKF,qBAAqB,aAAa;IAChE,IAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,MAClC,GACE,IAAI,IAAI,SAAS,IAAI,MAAM;MACzB,SAAS,eAAe;MACxB,KAAKnB,IAAI,OACP,gCAAgC,IAAI,oCACA,SAAS,cAC/C;MACA;KACF;;IAGJ;GACF;GACA,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,KAAKC,mBAAmB,IAAI,QAAQ;IAC5D,OACE,uBACM,yCAAyC,UAAU,MAAM,GACjE;IACA,MAAM,YAAY,OAAO,QAAQ,gBAAgB,OAAO,EAAE,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,GAAG,KAAK,SAAS,OAAO,QAAQ,SAAS,GAAG,EACnE;IACA,IAAI,UAAU,WAAW,GACvB,KAAKqB,wBAAwB,KAAK,QAAQ;SAE1C,KAAKX,qBAAqB,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,SAAS;IACvC,CAAC;IAIH,KAAKK,qBAAqB;IAC1B;GACF;EACF;CACF;CAEA,SAAe;EACb,KAAKA,qBAAqB,wBAAwB;EAClD,aAAa,KAAKH,mBAAmB;CACvC;AACF;AAEA,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,OAAO,EAAE,KAAK,GAAG,EAAE,KAAK,OAC7C,EAAC,MAAK,CACR;CACF;AACF;;;;;;;;;;;;;;;;;;;;AAqBA,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;EAChD,MAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,KAAK;CAChE;AACF;;;;;;;AAQA,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;EAC7D,MAAM,IAAI,KAAK,KAAK;CACtB;AACF"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts CHANGED Viewed

@@ -1,5 +1,6 @@
 import type { LogContext } from '@rocicorp/logger';
-import type { BackfillCompleted, BackfillRequest, MessageBackfill } from '../protocol/current.ts';
+import { type BackfillMessage } from '../common/backfill-manager.ts';
+import type { BackfillRequest } from '../protocol/current.ts';
 import type { Replica } from './schema/shard.ts';
 type StreamOptions = {
     /**
@@ -21,6 +22,6 @@ type StreamOptions = {
  * made at a transaction snapshot corresponding to specific LSN, obtained by
  * creating a short-lived replication slot.
  */
-export declare function streamBackfill(lc: LogContext, upstreamURI: string, { slot, publications }: Pick<Replica, 'slot' | 'publications'>, bf: BackfillRequest, opts?: StreamOptions): AsyncGenerator<MessageBackfill | BackfillCompleted>;
+export declare function streamBackfill(lc: LogContext, upstreamURI: string, { slot, publications }: Pick<Replica, 'slot' | 'publications'>, bf: BackfillRequest, opts?: StreamOptions): AsyncGenerator<BackfillMessage>;
 export {};
 //# sourceMappingURL=backfill-stream.d.ts.map

package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;~~AAmBjD~~,OAAO,KAAK,~~EACV~~,~~iBAAiB~~,~~EACjB~~,~~eAAe~~,~~EAGf~~,eAAe,~~EAChB~~,MAAM,wBAAwB,CAAC;AAahC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAChC,CAAC;AAYF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,~~GAAG,iBAAiB,~~CAAC,~~CA8FrD~~"}
1	+ {"version":3,"file":"backfill-stream.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/backfill-stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAkBjD,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAEV,eAAe,EAGhB,MAAM,wBAAwB,CAAC;AAahC,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAI/C,KAAK,aAAa,GAAG;IACnB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEzC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CAChC,CAAC;AAYF;;;;;GAKG;AACH,wBAAuB,cAAc,CACnC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,EAAC,IAAI,EAAE,YAAY,EAAC,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAAC,EAC5D,EAAE,EAAE,eAAe,EACnB,IAAI,GAAE,aAAkB,GACvB,cAAc,CAAC,eAAe,CAAC,CAsGjC"}

package/out/zero-cache/src/services/change-source/pg/backfill-stream.js CHANGED Viewed

@@ -28,7 +28,10 @@ var SAMPLE_OR_LIMIT_RE = /\sTABLESAMPLE\s+BERNOULLI\b|\sLIMIT\s+\d/i;
 async function* streamBackfill(lc, upstreamURI, { slot, publications }, bf, opts = {}) {
 	lc = lc.withContext("component", "backfill").withContext("table", bf.table.name);
 	const { flushThresholdBytes = POSTGRES_COPY_CHUNK_SIZE, textCopy = false } = opts;
-	const db = await connectPgClient(lc, upstreamURI, "backfill-stream", { ["max_lifetime"]: 7200 });
+	const db = await connectPgClient(lc, upstreamURI, "backfill-stream", {
+		["max_lifetime"]: 1440 * 60,
+		liveness: "keepalive"
+	});
 	let tx;
 	let watermark;
 	try {
@@ -106,10 +109,13 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 		totalBytes += chunk.byteLength;
 		if (bufferedBytes >= flushThresholdBytes) {
 			yield {
-				tag: "backfill",
-				...backfill,
-				rowValues,
-				status
+				message: {
+					tag: "backfill",
+					...backfill,
+					rowValues,
+					status
+				},
+				byteSize: bufferedBytes
 			};
 			totalMsgs++;
 			logFlushed();
@@ -121,18 +127,24 @@ async function* stream(lc, tx, backfill, { getTotalRows, getTotalBytes }, copyCo
 	await copyDone;
 	if (rowValues.length > 0) {
 		yield {
-			tag: "backfill",
-			...backfill,
-			rowValues,
-			status
+			message: {
+				tag: "backfill",
+				...backfill,
+				rowValues,
+				status
+			},
+			byteSize: bufferedBytes
 		};
 		totalMsgs++;
 		logFlushed();
 	}
 	yield {
-		tag: "backfill-completed",
-		...backfill,
-		status
+		message: {
+			tag: "backfill-completed",
+			...backfill,
+			status
+		},
+		byteSize: 0
 	};
 	elapsed = (performance.now() - start).toFixed(3);
 	lc.info?.(`Finished streaming ${status.rows} rows, ${totalMsgs} msgs, ${totalBytes} bytes (${elapsed} ms)`);