npm - @rocicorp/zero - Versions diffs - 1.2.0 → 1.3.0-canary.0 - Mend

@rocicorp/zero 1.2.0 → 1.3.0-canary.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (303) hide show

package/out/zero-cache/src/server/syncer.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport {AuthSessionImpl, type ValidateLegacyJWT} from '../auth/auth.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n config: Pick<NormalizedZeroConfig, 'query' \| 'getQueries'>,\n) {\n const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n if (!queryConfig?.url) {\n return undefined;\n }\n\n return {\n url: queryConfig.url,\n forwardCookies: queryConfig.forwardCookies ?? false,\n };\n}\n\nexport default function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n\n startOtelAuto(createLogContext(config, {worker: 'syncer'}, false));\n const lc = createLogContext(config, {worker: 'syncer'}, true);\n initEventSink(lc, config);\n\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const {cvr, upstream} = config;\n assert(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set');\n assert(upstream.maxConnsPerWorker, 'upstream.maxConnsPerWorker must be set');\n\n const replicaFile = replicaFileName(config.replica.file, fileMode);\n lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n const cvrDB = pgClient(lc, cvr.db, {\n max: cvr.maxConnsPerWorker,\n connection: {['application_name']: `zero-sync-worker-${pid}-cvr`},\n });\n\n const upstreamDB = pgClient(lc, upstream.db, {\n max: upstream.maxConnsPerWorker,\n connection: {['application_name']: `zero-sync-worker-${pid}-upstream`},\n });\n\n const dbWarmup = Promise.allSettled([\n warmupConnections(lc, cvrDB, 'cvr'),\n warmupConnections(lc, upstreamDB, 'upstream'),\n ]);\n\n const tmpDir = config.storageDBTmpDir ?? tmpdir();\n const operatorStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `sync-worker-${randomUUID()}`),\n );\n const writeAuthzStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `mutagen-${randomUUID()}`),\n );\n\n const shard = getShardID(config);\n\n const viewSyncerFactory = (\n id: string,\n sub: Subscription<ReplicaState>,\n drainCoordinator: DrainCoordinator,\n validateLegacyJWT: ValidateLegacyJWT \| undefined,\n ) => {\n const logger = lc\n .withContext('component', 'view-syncer')\n .withContext('clientGroupID', id)\n .withContext('instance', randomID());\n lc.debug?.(\n `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n );\n\n // Create the custom query transformer if configured\n const customQueryConfig = getCustomQueryConfig(config);\n const customQueryTransformer =\n customQueryConfig &&\n new CustomQueryTransformer(logger, customQueryConfig, shard);\n\n const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n const priorityOpRunningYieldThresholdMs = Math.max(\n config.yieldThresholdMs / 4,\n 2,\n );\n const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n const authSession = new AuthSessionImpl(\n logger.withContext('component', 'auth-session'),\n id,\n validateLegacyJWT,\n );\n\n return new ViewSyncerService(\n config,\n logger,\n shard,\n config.taskID,\n id,\n cvrDB,\n new PipelineDriver(\n logger,\n config.log,\n new Snapshotter(logger, replicaFile, shard),\n shard,\n operatorStorage.createClientGroupStorage(id),\n id,\n inspectorDelegate,\n () =>\n isPriorityOpRunning()\n ? priorityOpRunningYieldThresholdMs\n : normalYieldThresholdMs,\n config.enableQueryPlanner,\n config,\n ),\n sub,\n drainCoordinator,\n config.log.slowHydrateThreshold,\n inspectorDelegate,\n customQueryTransformer,\n runPriorityOp,\n authSession,\n );\n };\n\n const mutagenFactory = (id: string) =>\n new MutagenService(\n lc.withContext('component', 'mutagen').withContext('clientGroupID', id),\n shard,\n id,\n upstreamDB,\n config,\n writeAuthzStorage,\n );\n\n const pusherFactory =\n config.push.url === undefined && config.mutate.url === undefined\n ? undefined\n : (id: string) =>\n new PusherService(\n config,\n {\n ...config.push,\n ...config.mutate,\n url: must(\n config.push.url ?? config.mutate.url,\n 'No push or mutate URL configured',\n ),\n },\n lc.withContext('clientGroupID', id),\n id,\n );\n\n const syncer = new Syncer(\n lc,\n config,\n viewSyncerFactory,\n mutagenFactory,\n pusherFactory,\n parent,\n );\n\n startAnonymousTelemetry(lc, config);\n\n void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,SAAS,WAAW;AAClB,QAAO,QAAQ,GAAG,OAAO,iBAAiB,CAAC,SAAS,GAAG;;AAGzD,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;AAE9D,KAAI,CAAC,aAAa,IAChB;AAGF,QAAO;EACL,KAAK,YAAY;EACjB,gBAAgB,YAAY,kBAAkB;EAC/C;;AAGH,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;AAElE,eAAc,iBAAiB,QAAQ,EAAC,QAAQ,UAAS,EAAE,MAAM,CAAC;CAClE,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,UAAS,EAAE,KAAK;AAC7D,eAAc,IAAI,OAAO;AAEzB,QAAO,KAAK,SAAS,GAAG,gCAAgC;CACxD,MAAM,WAAW,MAAQ,KAAK,IAAI,sBAAsB;CAExD,MAAM,EAAC,KAAK,aAAY;AACxB,QAAO,IAAI,mBAAmB,oCAAoC;AAClE,QAAO,SAAS,mBAAmB,yCAAyC;CAE5E,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,SAAS;AAClE,IAAG,QAAQ,0BAA0B,cAAc;CAEnD,MAAM,QAAQ,SAAS,IAAI,IAAI,IAAI;EACjC,KAAK,IAAI;EACT,YAAY,GAAE,qBAAqB,oBAAoB,IAAI,OAAM;EAClE,CAAC;CAEF,MAAM,aAAa,SAAS,IAAI,SAAS,IAAI;EAC3C,KAAK,SAAS;EACd,YAAY,GAAE,qBAAqB,oBAAoB,IAAI,YAAW;EACvE,CAAC;CAEF,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,MAAM,EACnC,kBAAkB,IAAI,YAAY,WAAW,CAC9C,CAAC;CAEF,MAAM,SAAS,OAAO,mBAAmB,QAAQ;CACjD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,YAAY,GAAG,CACjD;CACD,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,YAAY,GAAG,CAC7C;CAED,MAAM,QAAQ,WAAW,OAAO;CAEhC,MAAM,qBACJ,IACA,KACA,kBACA,sBACG;EACH,MAAM,SAAS,GACZ,YAAY,aAAa,cAAc,CACvC,YAAY,iBAAiB,GAAG,CAChC,YAAY,YAAY,UAAU,CAAC;AACtC,KAAG,QACD,gDAAgD,OAAO,qBACxD;EAGD,MAAM,oBAAoB,qBAAqB,OAAO;EACtD,MAAM,yBACJ,qBACA,IAAI,uBAAuB,QAAQ,mBAAmB,MAAM;EAE9D,MAAM,oBAAoB,IAAI,kBAAkB,uBAAuB;EAEvE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,EACD;EACD,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,EAAE;EAEnE,MAAM,cAAc,IAAI,gBACtB,OAAO,YAAY,aAAa,eAAe,EAC/C,IACA,kBACD;AAED,SAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,MAAM,EAC3C,OACA,gBAAgB,yBAAyB,GAAG,EAC5C,IACA,yBAEE,qBAAqB,GACjB,oCACA,wBACN,OAAO,oBACP,OACD,EACD,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,wBACA,eACA,YACD;;CAGH,MAAM,kBAAkB,OACtB,IAAI,eACF,GAAG,YAAY,aAAa,UAAU,CAAC,YAAY,iBAAiB,GAAG,EACvE,OACA,IACA,YACA,QACA,kBACD;CAoBH,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBACA,gBArBA,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,KACC,OACC,IAAI,cACF,QACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,mCACD;EACF,EACD,GAAG,YAAY,iBAAiB,GAAG,EACnC,GACD,EAQP,OACD;AAED,yBAAwB,IAAI,OAAO;AAE9B,UAAS,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC,CAAC;AAE/D,QAAO,eAAe,IAAI,QAAQ,OAAO;;AAI3C,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
1	+ {"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n type ConnectionContextManager,\n ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\n\nfunction randomID() {\n return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n config: Pick<NormalizedZeroConfig, 'query' \| 'getQueries'>,\n) {\n const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n if (!queryConfig?.url) {\n return undefined;\n }\n\n return {\n url: queryConfig.url,\n apiKey: queryConfig.apiKey,\n allowedClientHeaders: queryConfig.allowedClientHeaders,\n forwardCookies: queryConfig.forwardCookies ?? false,\n };\n}\n\nexport default function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n\n startOtelAuto(createLogContext(config, {worker: 'syncer'}, false));\n const lc = createLogContext(config, {worker: 'syncer'}, true);\n initEventSink(lc, config);\n\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const {cvr, upstream, enableCrudMutations} = config;\n\n const replicaFile = replicaFileName(config.replica.file, fileMode);\n lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n const cvrDB = pgClient(lc, cvr.db, {\n max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n connection: {['application_name']: `zero-sync-worker-${pid}-cvr`},\n });\n\n const upstreamDB = enableCrudMutations\n ? pgClient(lc, upstream.db, {\n max: must(\n upstream.maxConnsPerWorker,\n 'upstream.maxConnsPerWorker must be set',\n ),\n connection: {['application_name']: `zero-sync-worker-${pid}-upstream`},\n })\n : undefined;\n\n const dbWarmup = Promise.allSettled([\n warmupConnections(lc, cvrDB, 'cvr'),\n upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n ]);\n\n const tmpDir = config.storageDBTmpDir ?? tmpdir();\n const operatorStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `sync-worker-${randomUUID()}`),\n );\n const writeAuthzStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `mutagen-${randomUUID()}`),\n );\n\n const shard = getShardID(config);\n const customQueryConfig = getCustomQueryConfig(config);\n const pushConfig =\n config.push.url === undefined && config.mutate.url === undefined\n ? undefined\n : {\n ...config.push,\n ...config.mutate,\n url: must(\n config.push.url ?? config.mutate.url,\n 'No push or mutate URL configured',\n ),\n };\n\n /** @deprecated used in JWT validation */\n let validateLegacyJWT: ValidateLegacyJWT \| undefined = undefined;\n\n const tokenOptions = tokenConfigOptions(config.auth ?? {});\n if (tokenOptions.length === 1) {\n validateLegacyJWT = async (token, {userID}) => {\n if (!userID) {\n throw new ProtocolErrorWithLevel(\n {\n kind: 'Unauthorized',\n message: 'UserID is required for JWT validation.',\n origin: 'zeroCache',\n },\n 'warn',\n );\n }\n\n const decoded = await verifyToken(config.auth, token, {\n subject: userID,\n ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n ...(config.auth?.audience && {\n audience: config.auth.audience,\n }),\n });\n return {\n type: 'jwt',\n raw: token,\n decoded,\n };\n };\n }\n\n const viewSyncerFactory = (\n id: string,\n sub: Subscription<ReplicaState>,\n drainCoordinator: DrainCoordinator,\n ) => {\n const logger = lc\n .withContext('component', 'view-syncer')\n .withContext('clientGroupID', id)\n .withContext('instance', randomID());\n\n const customQueryTransformer =\n customQueryConfig && new CustomQueryTransformer(logger, shard);\n const contextManager = new ConnectionContextManagerImpl(\n logger,\n config.auth.revalidateIntervalSeconds,\n config.auth.retransformIntervalSeconds,\n customQueryConfig,\n pushConfig,\n validateLegacyJWT,\n );\n\n lc.debug?.(\n `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n );\n\n const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n const priorityOpRunningYieldThresholdMs = Math.max(\n config.yieldThresholdMs / 4,\n 2,\n );\n const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n return new ViewSyncerService(\n config,\n logger,\n shard,\n config.taskID,\n id,\n cvrDB,\n new PipelineDriver(\n logger,\n config.log,\n new Snapshotter(logger, replicaFile, shard),\n shard,\n operatorStorage.createClientGroupStorage(id),\n id,\n inspectorDelegate,\n () =>\n isPriorityOpRunning()\n ? priorityOpRunningYieldThresholdMs\n : normalYieldThresholdMs,\n config.enableQueryPlanner,\n config,\n ),\n sub,\n drainCoordinator,\n config.log.slowHydrateThreshold,\n inspectorDelegate,\n contextManager,\n customQueryTransformer,\n runPriorityOp,\n );\n };\n\n const mutagenFactory = upstreamDB\n ? (id: string) =>\n new MutagenService(\n lc\n .withContext('component', 'mutagen')\n .withContext('clientGroupID', id),\n shard,\n id,\n upstreamDB,\n config,\n writeAuthzStorage,\n )\n : undefined;\n\n const pusherFactory =\n pushConfig === undefined\n ? undefined\n : (id: string, contextManager: ConnectionContextManager) =>\n new PusherService(\n config,\n lc.withContext('clientGroupID', id),\n id,\n contextManager,\n );\n\n const syncer = new Syncer(\n lc,\n config,\n viewSyncerFactory,\n mutagenFactory,\n pusherFactory,\n parent,\n validateLegacyJWT,\n );\n\n startAnonymousTelemetry(lc, config);\n\n void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8CA,SAAS,WAAW;AAClB,QAAO,QAAQ,GAAG,OAAO,iBAAiB,CAAC,SAAS,GAAG;;AAGzD,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;AAE9D,KAAI,CAAC,aAAa,IAChB;AAGF,QAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,gBAAgB,YAAY,kBAAkB;EAC/C;;AAGH,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;AAElE,eAAc,iBAAiB,QAAQ,EAAC,QAAQ,UAAS,EAAE,MAAM,CAAC;CAClE,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,UAAS,EAAE,KAAK;AAC7D,eAAc,IAAI,OAAO;AAEzB,QAAO,KAAK,SAAS,GAAG,gCAAgC;CACxD,MAAM,WAAW,MAAQ,KAAK,IAAI,sBAAsB;CAExD,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,SAAS;AAClE,IAAG,QAAQ,0BAA0B,cAAc;CAEnD,MAAM,QAAQ,SAAS,IAAI,IAAI,IAAI;EACjC,KAAK,KAAK,IAAI,mBAAmB,oCAAoC;EACrE,YAAY,GAAE,qBAAqB,oBAAoB,IAAI,OAAM;EAClE,CAAC;CAEF,MAAM,aAAa,sBACf,SAAS,IAAI,SAAS,IAAI;EACxB,KAAK,KACH,SAAS,mBACT,yCACD;EACD,YAAY,GAAE,qBAAqB,oBAAoB,IAAI,YAAW;EACvE,CAAC,GACF,KAAA;CAEJ,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,MAAM,EACnC,aAAa,kBAAkB,IAAI,YAAY,WAAW,GAAG,YAC9D,CAAC;CAEF,MAAM,SAAS,OAAO,mBAAmB,QAAQ;CACjD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,YAAY,GAAG,CACjD;CACD,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,YAAY,GAAG,CAC7C;CAED,MAAM,QAAQ,WAAW,OAAO;CAChC,MAAM,oBAAoB,qBAAqB,OAAO;CACtD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,mCACD;EACF;;CAGP,IAAI,oBAAmD,KAAA;AAGvD,KADqB,mBAAmB,OAAO,QAAQ,EAAE,CAAC,CACzC,WAAW,EAC1B,qBAAoB,OAAO,OAAO,EAAC,aAAY;AAC7C,MAAI,CAAC,OACH,OAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;GACT,EACD,OACD;AAUH,SAAO;GACL,MAAM;GACN,KAAK;GACL,SAVc,MAAM,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,QAAO;IACvD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,UACvB;IACF,CAAC;GAKD;;CAIL,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,aAAa,cAAc,CACvC,YAAY,iBAAiB,GAAG,CAChC,YAAY,YAAY,UAAU,CAAC;EAEtC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,MAAM;EAChE,MAAM,iBAAiB,IAAI,6BACzB,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,kBACD;AAED,KAAG,QACD,gDAAgD,OAAO,qBACxD;EAED,MAAM,oBAAoB,IAAI,kBAAkB,uBAAuB;EAEvE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,EACD;EACD,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,EAAE;AAEnE,SAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,MAAM,EAC3C,OACA,gBAAgB,yBAAyB,GAAG,EAC5C,IACA,yBAEE,qBAAqB,GACjB,oCACA,wBACN,OAAO,oBACP,OACD,EACD,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,gBACA,wBACA,cACD;;CA4BH,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,UAAU,CACnC,YAAY,iBAAiB,GAAG,EACnC,OACA,IACA,YACA,QACA,kBACD,GACH,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,mBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,GAAG,EACnC,IACA,eACD,EAQP,QACA,kBACD;AAED,yBAAwB,IAAI,OAAO;AAE9B,UAAS,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC,CAAC;AAE/D,QAAO,eAAe,IAAI,QAAQ,OAAO;;AAI3C,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}

package/out/zero-cache/src/server/worker-urls.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"~~AAcA~~,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}
1	+ {"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}

package/out/zero-cache/src/server/worker-urls.js CHANGED Viewed

@@ -1,7 +1,8 @@
 //#region ../zero-cache/src/server/worker-urls.ts
+var tsRe = /\.ts$/;
 function resolve(path) {
 	const { url } = import.meta;
-	if (url.endsWith(".js")) path = path.replace(/\.ts$/, ".js");
+	if (url.endsWith(".js")) path = path.replace(tsRe, ".js");
 	return new URL(path, url);
 }
 var CHANGE_STREAMER_URL = resolve("./change-streamer.ts");

package/out/zero-cache/src/server/worker-urls.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nfunction resolve(path: string): URL {\n const {url} = import.meta;\n if (url.endsWith('.js')) {\n // When compiled, change .ts to .js\n path = path.replace(~~/\\.ts$/,~~ '.js');\n }\n return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;AACrB,KAAI,IAAI,SAAS,MAAM,CAErB,QAAO,KAAK,QAAQ,~~SAAS~~,MAAM;~~AAErC~~,QAAO,IAAI,IAAI,MAAM,IAAI;;AAM3B,IAAa,sBAAsB,QAAQ,uBAAuB;AAClE,IAAa,WAAW,QAAQ,YAAY;AACjB,QAAQ,eAAe;AAClD,IAAa,aAAa,QAAQ,cAAc;AAChD,IAAa,iBAAiB,QAAQ,kBAAkB;AACxD,IAAa,aAAa,QAAQ,cAAc;AAChD,IAAa,mBAAmB,QAAQ,oBAAoB"}
1	+ {"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n const {url} = import.meta;\n if (url.endsWith('.js')) {\n // When compiled, change .ts to .js\n path = path.replace(tsRe, '.js');\n }\n return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO,OAAO;AACrB,KAAI,IAAI,SAAS,MAAM,CAErB,QAAO,KAAK,QAAQ,MAAM,MAAM;AAElC,QAAO,IAAI,IAAI,MAAM,IAAI;;AAM3B,IAAa,sBAAsB,QAAQ,uBAAuB;AAClE,IAAa,WAAW,QAAQ,YAAY;AACjB,QAAQ,eAAe;AAClD,IAAa,aAAa,QAAQ,cAAc;AAChD,IAAa,iBAAiB,QAAQ,kBAAkB;AACxD,IAAa,aAAa,QAAQ,cAAc;AAChD,IAAa,mBAAmB,QAAQ,oBAAoB"}

package/out/zero-cache/src/services/change-source/change-source.d.ts CHANGED Viewed

@@ -22,5 +22,9 @@ export interface ChangeSource {
      * with a corresponding sink for upstream acknowledgements.
      */
     startStream(afterWatermark: string, backfillRequests?: BackfillRequest[]): Promise<ChangeStream>;
+    /**
+     * Releases connections and resources held by this change source.
+     */
+    stop(): Promise<void>;
 }
 //# sourceMappingURL=change-source.d.ts.map

package/out/zero-cache/src/services/change-source/change-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-source/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EACV,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAErC;;;;OAIG;IACH,IAAI,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;CAClC,CAAC,CAAC,+EAA+E;AAElF,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,gBAAgB,IAAI,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAEpE;;;OAGG;IACH,WAAW,CACT,cAAc,EAAE,MAAM,EACtB,gBAAgB,CAAC,EAAE,eAAe,EAAE,GACnC,OAAO,CAAC,YAAY,CAAC,CAAC;~~CAC1B~~"}
1	+ {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-source/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAE,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,KAAK,EACV,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAErC;;;;OAIG;IACH,IAAI,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;CAClC,CAAC,CAAC,+EAA+E;AAElF,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,gBAAgB,IAAI,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAEpE;;;OAGG;IACH,WAAW,CACT,cAAc,EAAE,MAAM,EACtB,gBAAgB,CAAC,EAAE,eAAe,EAAE,GACnC,OAAO,CAAC,YAAY,CAAC,CAAC;IAEzB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;AAgBzD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;gBA+BxD,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B;IAWxC,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE;~~IA2O7D~~;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IA6M5C,MAAM,IAAI,IAAI;CAIf;AAED,uBAAe,mBAAoB,SAAQ,KAAK;gBAClC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAO9D;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;gBAEjC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAG9D"}
1	+ {"version":3,"file":"backfill-manager.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAOjD,OAAO,EAAmB,KAAK,UAAU,EAAC,MAAM,wBAAwB,CAAC;AAKzE,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EAEnB,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EACV,UAAU,EACV,uBAAuB,EACvB,QAAQ,EACT,MAAM,gCAAgC,CAAC;AAMxC,KAAK,gBAAgB,GAAG,CACtB,GAAG,EAAE,eAAe,KACjB,cAAc,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;AAgBzD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,eAAgB,YAAW,UAAU,EAAE,QAAQ;;gBA+BxD,EAAE,EAAE,UAAU,EACd,cAAc,EAAE,uBAAuB,EACvC,gBAAgB,EAAE,gBAAgB,EAClC,UAAU,GAAE,UAA6B,EACzC,YAAY,SAA0B,EACtC,YAAY,SAA0B;IAWxC,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE;IAsP7D;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IA6M5C,MAAM,IAAI,IAAI;CAIf;AAED,uBAAe,mBAAoB,SAAQ,KAAK;gBAClC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAO9D;AA6BD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,mBAAmB;IACjE,QAAQ,CAAC,IAAI,gCAAgC;gBAEjC,EAAE,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAG9D"}

package/out/zero-cache/src/services/change-source/common/backfill-manager.js CHANGED Viewed

@@ -139,10 +139,11 @@ var BackfillManager = class {
 				return null;
 			}
 			const { major, minor = 0n } = stateVersionFromString(lastWatermark);
-			const tx = stateVersionToString({
+			let tx = stateVersionToString({
 				major,
 				minor: BigInt(minor) + 1n
 			});
+			if (msg.tag === "backfill-completed" && tx < msg.watermark) tx = msg.watermark;
 			changeStream.push([
 				"begin",
 				{
@@ -166,7 +167,7 @@ var BackfillManager = class {
 			backfillTx = null;
 		};
 		for await (const msg of this.#backfillStreamer(state.request)) {
-			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && this.#changeStreamReached(lc, msg.watermark);
+			const mustWaitBeforeFlush = msg.tag === "backfill-completed" && (this.#changeStreamReached(lc, msg.watermark) || backfillTx !== null && backfillTx < msg.watermark);
 			if (backfillTx && (changeStream.waiterDelay() > 0 || mustWaitBeforeFlush)) commitTx();
 			mustWaitBeforeFlush && await mustWaitBeforeFlush;
 			if (msg.tag === "backfill" && msg.rowValues.length > 0 && msg.relation.rowKey.columns.length === 0) throw new MissingRowKeyError(state.request);

package/out/zero-cache/src/services/change-source/common/backfill-manager.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n stateVersionFromString,\n stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n ChangeStreamMessage,\n Identifier,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n Cancelable,\n ChangeStreamMultiplexer,\n Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n return `${schema}.${name}`;\n}\n\ntype BackfillStreamer = (\n req: BackfillRequest,\n) => AsyncGenerator<MessageBackfill \| BackfillCompleted>;\n\ntype RunningBackfillState = {\n request: BackfillRequest;\n canceledReason?: string \| undefined;\n minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n watermark: string;\n reached: () => void;\n};\n\n/*\n The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n \n The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n \n The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n /\nexport class BackfillManager implements Cancelable, Listener {\n readonly #lc: LogContext;\n\n /\n Tracks the metadata of required backfills based on schema changes\n * and initial backfill requests.\n /\n readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n tableKey,\n );\n readonly #changeStreamer: ChangeStreamMultiplexer;\n readonly #backfillStreamer: BackfillStreamer;\n readonly #jsonFormat: JSONFormat;\n\n /\n The current running backfill. The backfill request is always also in\n * `#requiredBackfills` (technically, it can be a subset of what's in\n * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n * upon completion.\n /\n #runningBackfill: RunningBackfillState \| null = null;\n\n /* The last seen watermark in the change stream. /\n #lastStatusWatermark: string \| null = null;\n\n readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n /* The watermark of the current transaction in the change stream. /\n #currentTxWatermark: string \| null = null;\n\n constructor(\n lc: LogContext,\n changeStreamer: ChangeStreamMultiplexer,\n backfillStreamer: BackfillStreamer,\n jsonFormat: JSONFormat = JSON_STRINGIFIED,\n minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n ) {\n this.#lc = lc.withContext('component', 'backfill-manager');\n this.#changeStreamer = changeStreamer;\n this.#backfillStreamer = backfillStreamer;\n this.#jsonFormat = jsonFormat;\n this.#minBackoffMs = minBackoffMs;\n this.#maxBackoffMs = maxBackoffMs;\n this.#retryDelayMs = minBackoffMs;\n }\n\n run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n this.#lc.info?.(\n `starting backfill manager with ${initialRequests.length} initial requests`,\n {requests: initialRequests},\n );\n this.#lastStatusWatermark = lastWatermark;\n initialRequests.forEach(req =>\n this.#setRequiredBackfill('initial-request', req),\n );\n this.#checkAndStartBackfill();\n }\n\n #setLastStatusWatermark({watermark}: {watermark: string}) {\n // Only allow the watermark to move forward. This prevents a backfill\n // transaction (whose watermark is unrelated to change-stream state)\n // from moving the watermark backwards.\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n this.#lastStatusWatermark = watermark;\n for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n const awaiting = this.#awaitingStatusWatermarks[i];\n if (watermark >= awaiting.watermark) {\n awaiting.reached();\n this.#awaitingStatusWatermarks.splice(i, 1);\n }\n }\n }\n }\n\n #changeStreamReached(\n lc: LogContext,\n watermark: string,\n ): Promise<void> \| null {\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n const {promise, resolve: reached} = resolver();\n this.#awaitingStatusWatermarks.push({watermark, reached});\n lc.info?.(\n `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n );\n return promise;\n }\n return null;\n }\n\n readonly #minBackoffMs: number;\n readonly #maxBackoffMs: number;\n #retryDelayMs: number;\n #backfillRetryTimer: NodeJS.Timeout \| undefined;\n\n #checkAndStartBackfill() {\n if (\n !this.#backfillRetryTimer &&\n !this.#runningBackfill &&\n this.#requiredBackfills.size\n ) {\n // Pick a random backfill to avoid head-of-line blocking by a\n // problematic backfill (e.g. awaiting a primary key). This is\n // simpler that adding logic to classify (and declassify)\n // problematic backfills.\n const candidates = [...this.#requiredBackfills.values()];\n const request = candidates[randInt(0, candidates.length - 1)];\n const state = {request, minWatermark: ''};\n const lc = this.#lc.withContext('table', request.table.name);\n\n this.#runningBackfill = state;\n void this.#runBackfill(lc, state)\n .then(() => {\n this.#stopRunningBackfill('backfill exited', state);\n this.#retryDelayMs = this.#minBackoffMs; // reset on success\n })\n // For unexpected errors (e.g. upstream replication slot\n // unavailability), retry with exponential backoff.\n .catch(e => {\n this.#stopRunningBackfill(String(e), state);\n this.#retryBackfillWithBackoff(e);\n });\n }\n }\n\n #retryBackfillWithBackoff(e: unknown) {\n const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n this.#lc[log]?.(\n `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n e,\n );\n this.#backfillRetryTimer = setTimeout(() => {\n this.#backfillRetryTimer = undefined;\n this.#checkAndStartBackfill();\n }, this.#retryDelayMs);\n\n this.#retryDelayMs = Math.min(this.#retryDelayMs 2, this.#maxBackoffMs);\n }\n\n async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n const changeStream = this.#changeStreamer; // Purely for readability\n\n // backfillTx is set if and only if a changeStreamer reservation has been\n // acquired and the backfill stream is inside a transaction.\n let backfillTx: string \| null = null;\n\n /*\n @returns the new tx watermark, or null if backfill was cancelled\n /\n const beginTxFor = async (\n msg: MessageBackfill \| BackfillCompleted,\n ): Promise<string \| null> => {\n assert(backfillTx === null, 'Expected no active backfill transaction');\n const lastWatermark = await changeStream.reserve('backfill');\n\n // After obtaining the changeStream reservation, check if the stream\n // had changes that resulted in invalidating / canceling this backfill.\n if (\n state.canceledReason \|\|\n (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n ) {\n if (state.canceledReason === undefined) {\n assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n this.#stopRunningBackfill(\n `row key change at ${state.minWatermark} ` +\n `postdates backfill watermark at ${msg.watermark}`,\n state,\n );\n }\n changeStream.release(lastWatermark);\n return null;\n }\n\n const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n const tx = stateVersionToString({\n major,\n minor: BigInt(minor) + 1n,\n });\n\n void changeStream.push([\n 'begin',\n {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n {commitWatermark: tx},\n ]);\n return (backfillTx = tx);\n };\n\n const commitTx = () => {\n if (backfillTx) {\n void changeStream.push([\n 'commit',\n {tag: 'commit'},\n {watermark: backfillTx},\n ]);\n changeStream.release(backfillTx);\n }\n backfillTx = null;\n };\n\n for await (const msg of this.#backfillStreamer(state.request)) {\n // Before sending `backfill-completed`, the main replication stream\n // may need to catch up.\n const mustWaitBeforeFlush =\n msg.tag === 'backfill-completed' &&\n this.#changeStreamReached(lc, msg.watermark);\n\n // If necessary, yield the reservation to the main stream.\n if (\n backfillTx &&\n (changeStream.waiterDelay() > 0 \|\| mustWaitBeforeFlush)\n ) {\n commitTx();\n }\n\n mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n if (\n msg.tag === 'backfill' &&\n msg.rowValues.length > 0 &&\n msg.relation.rowKey.columns.length === 0\n ) {\n throw new MissingRowKeyError(state.request);\n }\n\n // Reserve the changeStreamer if not in a transaction.\n if ((backfillTx ??= await beginTxFor(msg)) === null) {\n lc.info?.(\n `backfill stream canceled: ${state.canceledReason}`,\n state.request,\n );\n this.#checkAndStartBackfill(); // start the next backfill if present\n return; // this backfill is canceled\n }\n\n // `await` to allow the change streamer to exert back pressure\n // on backfills.\n await changeStream.push(['data', msg]);\n }\n\n // Flush any final tx and release the stream.\n backfillTx && commitTx();\n lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n }\n\n #backfillRunningFor(table: Identifier): RunningBackfillState \| null {\n const state = this.#runningBackfill;\n return state?.request.table.schema === table.schema &&\n state.request.table.name === table.name\n ? state\n : null;\n }\n\n /\n Stops the running backfill for the specified `reason`. If `instance` is\n * specified, the running backfill is stopped only if it is that instance.\n * This allows the running backfill itself to clear backfill state without\n * accidentally stopping a different (e.g. subsequent) backfill.\n /\n #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n const backfill = this.#runningBackfill;\n if (backfill && backfill === (instance ?? backfill)) {\n backfill.canceledReason = reason;\n this.#runningBackfill = null;\n reason && this.#lc.info?.(`canceling backfill:`, reason);\n }\n }\n\n #setRequiredBackfill(source: string, req: BackfillRequest) {\n const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.set(req.table, req);\n }\n\n #deleteRequiredBackfill(source: string, id: Identifier) {\n const req = this.#requiredBackfills.get(id);\n if (req) {\n const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.delete(id);\n }\n }\n\n /\n Implements {@link Listener.onChange()}, invoked by the\n * {@link ChangeStreamMultiplexer}.\n /\n onChange(message: ChangeStreamMessage): void {\n if (message[0] === 'begin') {\n this.#currentTxWatermark = message[2].commitWatermark;\n return;\n }\n if (message[0] === 'commit') {\n this.#currentTxWatermark = null;\n this.#setLastStatusWatermark(message[2]);\n // Every commit is a candidate for starting the next backfill\n // (if one is not currently running).\n this.#checkAndStartBackfill();\n return;\n }\n if (message[0] === 'status') {\n this.#setLastStatusWatermark(message[2]);\n return;\n }\n if (message[0] !== 'data') {\n return;\n }\n const change = message[1];\n const {tag} = change;\n switch (tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest) {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n });\n if (this.#backfillRunningFor(table)) {\n this.#stopRunningBackfill(`TableMetadata updated`);\n }\n }\n break;\n }\n case 'create-table': {\n const {\n spec: {schema, name},\n metadata = null,\n backfill,\n } = change;\n\n if (backfill) {\n this.#setRequiredBackfill(tag, {\n table: {schema, name, metadata},\n columns: backfill,\n });\n }\n break;\n }\n case 'rename-table': {\n const {old, new: newTable} = change;\n const backfillRequest = this.#requiredBackfills.get(old);\n if (backfillRequest) {\n const {schema, name} = newTable;\n this.#deleteRequiredBackfill(tag, old);\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, schema, name},\n });\n if (this.#backfillRunningFor(old)) {\n this.#stopRunningBackfill(`table renamed`);\n }\n }\n break;\n }\n case 'drop-table': {\n const {id} = change;\n const backfillRequest = this.#requiredBackfills.get(id);\n if (backfillRequest) {\n this.#deleteRequiredBackfill(tag, id);\n if (this.#backfillRunningFor(id)) {\n this.#stopRunningBackfill(`table dropped`);\n }\n }\n break;\n }\n case 'add-column': {\n const {\n table,\n tableMetadata: metadata = null,\n column,\n backfill,\n } = change;\n if (backfill) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (!backfillRequest) {\n this.#setRequiredBackfill(tag, {\n table: {...table, metadata},\n columns: {[column.name]: backfill},\n });\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n columns: {\n ...backfillRequest.columns,\n [column.name]: backfill,\n },\n });\n // Note: The running backfill need not be canceled if a\n // new column is added. The new column will be backfilled\n // by its own stream after the current backfill completes.\n }\n }\n break;\n }\n case 'update-column': {\n const {\n table,\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && oldName in backfillRequest.columns) {\n const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: {...otherCols, [newName]: colSpec},\n });\n const backfill = this.#backfillRunningFor(table);\n if (backfill && oldName in backfill.request.columns) {\n this.#stopRunningBackfill(`column renamed`);\n }\n }\n }\n break;\n }\n case 'drop-column': {\n const {table, column} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && column in backfillRequest.columns) {\n const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n if (Object.keys(remaining).length === 0) {\n this.#deleteRequiredBackfill(tag, table);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: remaining,\n });\n }\n const backfill = this.#backfillRunningFor(table);\n if (backfill && column in backfill.request.columns) {\n this.#stopRunningBackfill(`column dropped`);\n }\n }\n break;\n }\n case 'update': {\n const {relation, key, new: row} = change;\n const backfill = this.#backfillRunningFor(relation);\n const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n if (backfill?.request.table.metadata && key !== null) {\n // A corner case that backfill is unable to correctly handle is\n // when a row's key changes; this is decomposed into a delete\n // of the old key and a set of the new key in the replica change\n // log, at which point the backfill algorithm assumes that the\n // (old) row is deleted but does not know to backfill the new row.\n // In these corner cases, the current backfill is canceled and\n // retried if its version precedes this update.\n for (const col of Object.keys(\n backfill.request.table.metadata.rowKey,\n )) {\n if (key[col] !== row[col]) {\n backfill.minWatermark = txWatermark;\n this.#lc.info?.(\n `key for row as changed (col: ${col}). ` +\n `backfill data must not predate ${backfill.minWatermark}`,\n );\n break;\n }\n }\n }\n break;\n }\n case 'backfill-completed': {\n const {relation, columns} = change;\n const backfillRequest = this.#requiredBackfills.get(relation);\n assert(\n backfillRequest,\n () => `No BackfillRequest completed backfill ${stringify(change)}`,\n );\n const remaining = Object.entries(backfillRequest.columns).filter(\n ([col]) =>\n !(columns.includes(col) \|\| relation.rowKey.columns.includes(col)),\n );\n if (remaining.length === 0) {\n this.#deleteRequiredBackfill(tag, relation);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: Object.fromEntries(remaining),\n });\n }\n // Technically the backfill is already stopping, but this method\n // cleans up the state that tracks it.\n this.#stopRunningBackfill();\n break;\n }\n }\n }\n\n cancel(): void {\n this.#stopRunningBackfill(`change stream canceled`);\n clearTimeout(this.#backfillRetryTimer);\n }\n}\n\nabstract class BackfillStreamError extends Error {\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(\n `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n {cause},\n );\n }\n}\n\n/\n Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n \n In terms of backfill, however, non-empty tables without a row key cannot\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n \n The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n /\nclass MissingRowKeyError extends BackfillStreamError {\n readonly name = 'MissingRowKeyError';\n\n constructor(bf: BackfillRequest, cause?: unknown) {\n super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n }\n}\n\n/\n Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n readonly name = 'SchemaIncompatibilityError';\n\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(bf, msg, cause);\n }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;AAC5C,QAAO,GAAG,OAAO,GAAG;;AAatB,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,SACD;CACD;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,EAAE;;CAGlE,sBAAqC;CAErC,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,iBAAuB;AACvB,QAAA,mBAAyB;AACzB,QAAA,aAAmB;AACnB,QAAA,eAAqB;AACrB,QAAA,eAAqB;AACrB,QAAA,eAAqB;;CAGvB,IAAI,eAAuB,iBAAoC;AAC7D,QAAA,GAAS,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,iBAAgB,CAC5B;AACD,QAAA,sBAA4B;AAC5B,kBAAgB,SAAQ,QACtB,MAAA,oBAA0B,mBAAmB,IAAI,CAClD;AACD,QAAA,uBAA6B;;CAG/B,wBAAwB,EAAC,aAAiC;AAIxD,OAAK,MAAA,uBAA6B,MAAM,WAAW;AACjD,SAAA,sBAA4B;AAC5B,QAAK,IAAI,IAAI,MAAA,yBAA+B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,MAAA,yBAA+B;AAChD,QAAI,aAAa,SAAS,WAAW;AACnC,cAAS,SAAS;AAClB,WAAA,yBAA+B,OAAO,GAAG,EAAE;;;;;CAMnD,qBACE,IACA,WACsB;AACtB,OAAK,MAAA,uBAA6B,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,UAAU;AAC9C,SAAA,yBAA+B,KAAK;IAAC;IAAW;IAAQ,CAAC;AACzD,MAAG,OACD,iCAAiC,MAAA,oBAA0B,aAAa,YACzE;AACD,UAAO;;AAET,SAAO;;CAGT;CACA;CACA;CACA;CAEA,yBAAyB;AACvB,MACE,CAAC,MAAA,sBACD,CAAC,MAAA,mBACD,MAAA,kBAAwB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,MAAA,kBAAwB,QAAQ,CAAC;GACxD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,EAAE;GAC5D,MAAM,QAAQ;IAAC;IAAS,cAAc;IAAG;GACzC,MAAM,KAAK,MAAA,GAAS,YAAY,SAAS,QAAQ,MAAM,KAAK;AAE5D,SAAA,kBAAwB;AACnB,SAAA,YAAkB,IAAI,MAAM,CAC9B,WAAW;AACV,UAAA,oBAA0B,mBAAmB,MAAM;AACnD,UAAA,eAAqB,MAAA;KACrB,CAGD,OAAM,MAAK;AACV,UAAA,oBAA0B,OAAO,EAAE,EAAE,MAAM;AAC3C,UAAA,yBAA+B,EAAE;KACjC;;;CAIR,0BAA0B,GAAY;EACpC,MAAM,MAAM,MAAA,iBAAuB,MAAA,eAAqB,UAAU;AAClE,QAAA,GAAS,OACP,uCAAuC,MAAA,aAAmB,MAC1D,EACD;AACD,QAAA,qBAA2B,iBAAiB;AAC1C,SAAA,qBAA2B,KAAA;AAC3B,SAAA,uBAA6B;KAC5B,MAAA,aAAmB;AAEtB,QAAA,eAAqB,KAAK,IAAI,MAAA,eAAqB,GAAG,MAAA,aAAmB;;CAG3E,OAAA,YAAmB,IAAgB,OAA6B;EAC9D,MAAM,eAAe,MAAA;EAIrB,IAAI,aAA4B;;;;EAKhC,MAAM,aAAa,OACjB,QAC2B;AAC3B,UAAO,eAAe,MAAM,0CAA0C;GACtE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,WAAW;AAI5D,OACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;AACA,QAAI,MAAM,mBAAmB,KAAA,GAAW;AACtC,YAAO,IAAI,QAAQ,YAAY,gCAAgC;AAC/D,WAAA,oBACE,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,MACD;;AAEH,iBAAa,QAAQ,cAAc;AACnC,WAAO;;GAGT,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,cAAc;GACjE,MAAM,KAAK,qBAAqB;IAC9B;IACA,OAAO,OAAO,MAAM,GAAG;IACxB,CAAC;AAEG,gBAAa,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,MAAA;KAAkB,SAAS;KAAK;IACrD,EAAC,iBAAiB,IAAG;IACtB,CAAC;AACF,UAAQ,aAAa;;EAGvB,MAAM,iBAAiB;AACrB,OAAI,YAAY;AACT,iBAAa,KAAK;KACrB;KACA,EAAC,KAAK,UAAS;KACf,EAAC,WAAW,YAAW;KACxB,CAAC;AACF,iBAAa,QAAQ,WAAW;;AAElC,gBAAa;;AAGf,aAAW,MAAM,OAAO,MAAA,iBAAuB,MAAM,QAAQ,EAAE;GAG7D,MAAM,sBACJ,IAAI,QAAQ,wBACZ,MAAA,oBAA0B,IAAI,IAAI,UAAU;AAG9C,OACE,eACC,aAAa,aAAa,GAAG,KAAK,qBAEnC,WAAU;AAGZ,0BAAwB,MAAM;AAE9B,OACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,EAEvC,OAAM,IAAI,mBAAmB,MAAM,QAAQ;AAI7C,QAAK,eAAe,MAAM,WAAW,IAAI,MAAM,MAAM;AACnD,OAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,QACP;AACD,UAAA,uBAA6B;AAC7B;;AAKF,SAAM,aAAa,KAAK,CAAC,QAAQ,IAAI,CAAC;;AAIxC,gBAAc,UAAU;AACxB,KAAG,QAAQ,0BAA0B,MAAM,kBAAkB,GAAG;;CAGlE,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,MAAA;AACd,SAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;;;;;;;;CASN,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,MAAA;AACjB,MAAI,YAAY,cAAc,YAAY,WAAW;AACnD,YAAS,iBAAiB;AAC1B,SAAA,kBAAwB;AACxB,aAAU,MAAA,GAAS,OAAO,uBAAuB,OAAO;;;CAI5D,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,MAAA,kBAAwB,IAAI,IAAI,MAAM,GAAG,YAAY;AACpE,QAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,QAAA,kBAAwB,IAAI,IAAI,OAAO,IAAI;;CAG7C,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,MAAA,kBAAwB,IAAI,GAAG;AAC3C,MAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;AAC/D,SAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,SAAA,kBAAwB,OAAO,GAAG;;;;;;;CAQtC,SAAS,SAAoC;AAC3C,MAAI,QAAQ,OAAO,SAAS;AAC1B,SAAA,qBAA2B,QAAQ,GAAG;AACtC;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,qBAA2B;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AAGxC,SAAA,uBAA6B;AAC7B;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AACxC;;AAEF,MAAI,QAAQ,OAAO,OACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;AACd,UAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,iBAAiB;AACnB,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC5C,CAAC;AACF,SAAI,MAAA,mBAAyB,MAAM,CACjC,OAAA,oBAA0B,wBAAwB;;AAGtD;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;AAEJ,QAAI,SACF,OAAA,oBAA0B,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;MAAS;KAC/B,SAAS;KACV,CAAC;AAEJ;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,IAAI;AACxD,QAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;AACvB,WAAA,uBAA6B,KAAK,IAAI;AACtC,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;OAAK;MAChD,CAAC;AACF,SAAI,MAAA,mBAAyB,IAAI,CAC/B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;AAEb,QADwB,MAAA,kBAAwB,IAAI,GAAG,EAClC;AACnB,WAAA,uBAA6B,KAAK,GAAG;AACrC,SAAI,MAAA,mBAAyB,GAAG,CAC9B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;AACJ,QAAI,UAAU;KACZ,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,CAAC,gBACH,OAAA,oBAA0B,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;OAAS;MAC3B,SAAS,GAAE,OAAO,OAAO,UAAS;MACnC,CAAC;SAEF,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC3C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;OAChB;MACF,CAAC;;AAMN;;GAEF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;AAC3D,YAAA,oBAA0B,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;QAAQ;OAC5C,CAAC;MACF,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,UAAI,YAAY,WAAW,SAAS,QAAQ,QAC1C,OAAA,oBAA0B,iBAAiB;;;AAIjD;;GAEF,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;AAC5D,SAAI,OAAO,KAAK,UAAU,CAAC,WAAW,EACpC,OAAA,uBAA6B,KAAK,MAAM;SAExC,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,SAAS;MACV,CAAC;KAEJ,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,SAAI,YAAY,UAAU,SAAS,QAAQ,QACzC,OAAA,oBAA0B,iBAAiB;;AAG/C;;GAEF,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,MAAA,mBAAyB,SAAS;IACnD,MAAM,cAAc,KAAK,MAAA,oBAA0B,cAAc;AACjE,QAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,OACjC,CACC,KAAI,IAAI,SAAS,IAAI,MAAM;AACzB,eAAS,eAAe;AACxB,YAAA,GAAS,OACP,gCAAgC,IAAI,oCACA,SAAS,eAC9C;AACD;;;AAIN;;GAEF,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,SAAS;AAC7D,WACE,uBACM,yCAAyC,UAAU,OAAO,GACjE;IACD,MAAM,YAAY,OAAO,QAAQ,gBAAgB,QAAQ,CAAC,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,IAAI,IAAI,SAAS,OAAO,QAAQ,SAAS,IAAI,EACnE;AACD,QAAI,UAAU,WAAW,EACvB,OAAA,uBAA6B,KAAK,SAAS;QAE3C,OAAA,oBAA0B,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,UAAU;KACvC,CAAC;AAIJ,UAAA,qBAA2B;AAC3B;;;;CAKN,SAAe;AACb,QAAA,oBAA0B,yBAAyB;AACnD,eAAa,MAAA,mBAAyB;;;AAI1C,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,KAAK,OAC7C,EAAC,OAAM,CACR;;;;;;;;;;;;;;;;;;;;;;AAuBL,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;AAChD,QAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,MAAM;;;;;;;;;AAUnE,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QAAM,IAAI,KAAK,MAAM"}
1	+ {"version":3,"file":"backfill-manager.js","names":["#lc","#requiredBackfills","#changeStreamer","#backfillStreamer","#jsonFormat","#awaitingStatusWatermarks","#minBackoffMs","#maxBackoffMs","#retryDelayMs","#lastStatusWatermark","#setRequiredBackfill","#checkAndStartBackfill","#backfillRetryTimer","#runningBackfill","#runBackfill","#stopRunningBackfill","#retryBackfillWithBackoff","#changeStreamReached","#currentTxWatermark","#setLastStatusWatermark","#backfillRunningFor","#deleteRequiredBackfill"],"sources":["../../../../../../../zero-cache/src/services/change-source/common/backfill-manager.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {CustomKeyMap} from '../../../../../shared/src/custom-key-map.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {randInt} from '../../../../../shared/src/rand.ts';\nimport {JSON_STRINGIFIED, type JSONFormat} from '../../../types/lite.ts';\nimport {\n stateVersionFromString,\n stateVersionToString,\n} from '../../../types/state-version.ts';\nimport type {\n BackfillCompleted,\n BackfillRequest,\n ChangeStreamMessage,\n Identifier,\n MessageBackfill,\n} from '../protocol/current.ts';\nimport type {\n Cancelable,\n ChangeStreamMultiplexer,\n Listener,\n} from './change-stream-multiplexer.ts';\n\nfunction tableKey({schema, name}: Identifier) {\n return `${schema}.${name}`;\n}\n\ntype BackfillStreamer = (\n req: BackfillRequest,\n) => AsyncGenerator<MessageBackfill \| BackfillCompleted>;\n\ntype RunningBackfillState = {\n request: BackfillRequest;\n canceledReason?: string \| undefined;\n minWatermark: string;\n};\n\nconst MIN_BACKOFF_INTERVAL_MS = 2_000;\nconst MAX_BACKOFF_INTERVAL_MS = 60_000;\n\ntype AwaitingStatusWatermark = {\n watermark: string;\n reached: () => void;\n};\n\n/*\n The BackfillManager initiates backfills for BackfillRequests from the\n * change-streamer (i.e. unfinished backfills from previous sessions)\n * or for new backfills signaled by `create-table` or `add-column` messages\n * from the change-source.\n \n The BackfillManager registers itself as a change stream listener in order\n * to track necessary backfills, and potentially invalidate the in-progress\n * backfill (e.g. due to a schema change) so that it can be retried at a\n * new snapshot.\n \n The manager also handles low priority streaming of the backfill messages\n * using the {@link ChangeStreamMultiplexer}, implementing a policy of always\n * releasing its reservation if another producer (i.e. the main change stream)\n * has messages to stream.\n /\nexport class BackfillManager implements Cancelable, Listener {\n readonly #lc: LogContext;\n\n /\n Tracks the metadata of required backfills based on schema changes\n * and initial backfill requests.\n /\n readonly #requiredBackfills = new CustomKeyMap<Identifier, BackfillRequest>(\n tableKey,\n );\n readonly #changeStreamer: ChangeStreamMultiplexer;\n readonly #backfillStreamer: BackfillStreamer;\n readonly #jsonFormat: JSONFormat;\n\n /\n The current running backfill. The backfill request is always also in\n * `#requiredBackfills` (technically, it can be a subset of what's in\n * `#requiredBackfills`); the request is removed from `#requiredBackfills`\n * upon completion.\n /\n #runningBackfill: RunningBackfillState \| null = null;\n\n /* The last seen watermark in the change stream. /\n #lastStatusWatermark: string \| null = null;\n\n readonly #awaitingStatusWatermarks: AwaitingStatusWatermark[] = [];\n\n /* The watermark of the current transaction in the change stream. /\n #currentTxWatermark: string \| null = null;\n\n constructor(\n lc: LogContext,\n changeStreamer: ChangeStreamMultiplexer,\n backfillStreamer: BackfillStreamer,\n jsonFormat: JSONFormat = JSON_STRINGIFIED,\n minBackoffMs = MIN_BACKOFF_INTERVAL_MS,\n maxBackoffMs = MAX_BACKOFF_INTERVAL_MS,\n ) {\n this.#lc = lc.withContext('component', 'backfill-manager');\n this.#changeStreamer = changeStreamer;\n this.#backfillStreamer = backfillStreamer;\n this.#jsonFormat = jsonFormat;\n this.#minBackoffMs = minBackoffMs;\n this.#maxBackoffMs = maxBackoffMs;\n this.#retryDelayMs = minBackoffMs;\n }\n\n run(lastWatermark: string, initialRequests: BackfillRequest[]) {\n this.#lc.info?.(\n `starting backfill manager with ${initialRequests.length} initial requests`,\n {requests: initialRequests},\n );\n this.#lastStatusWatermark = lastWatermark;\n initialRequests.forEach(req =>\n this.#setRequiredBackfill('initial-request', req),\n );\n this.#checkAndStartBackfill();\n }\n\n #setLastStatusWatermark({watermark}: {watermark: string}) {\n // Only allow the watermark to move forward. This prevents a backfill\n // transaction (whose watermark is unrelated to change-stream state)\n // from moving the watermark backwards.\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n this.#lastStatusWatermark = watermark;\n for (let i = this.#awaitingStatusWatermarks.length - 1; i >= 0; i--) {\n const awaiting = this.#awaitingStatusWatermarks[i];\n if (watermark >= awaiting.watermark) {\n awaiting.reached();\n this.#awaitingStatusWatermarks.splice(i, 1);\n }\n }\n }\n }\n\n #changeStreamReached(\n lc: LogContext,\n watermark: string,\n ): Promise<void> \| null {\n if ((this.#lastStatusWatermark ?? '') < watermark) {\n const {promise, resolve: reached} = resolver();\n this.#awaitingStatusWatermarks.push({watermark, reached});\n lc.info?.(\n `waiting for change stream (at ${this.#lastStatusWatermark}) to reach ${watermark}`,\n );\n return promise;\n }\n return null;\n }\n\n readonly #minBackoffMs: number;\n readonly #maxBackoffMs: number;\n #retryDelayMs: number;\n #backfillRetryTimer: NodeJS.Timeout \| undefined;\n\n #checkAndStartBackfill() {\n if (\n !this.#backfillRetryTimer &&\n !this.#runningBackfill &&\n this.#requiredBackfills.size\n ) {\n // Pick a random backfill to avoid head-of-line blocking by a\n // problematic backfill (e.g. awaiting a primary key). This is\n // simpler that adding logic to classify (and declassify)\n // problematic backfills.\n const candidates = [...this.#requiredBackfills.values()];\n const request = candidates[randInt(0, candidates.length - 1)];\n const state = {request, minWatermark: ''};\n const lc = this.#lc.withContext('table', request.table.name);\n\n this.#runningBackfill = state;\n void this.#runBackfill(lc, state)\n .then(() => {\n this.#stopRunningBackfill('backfill exited', state);\n this.#retryDelayMs = this.#minBackoffMs; // reset on success\n })\n // For unexpected errors (e.g. upstream replication slot\n // unavailability), retry with exponential backoff.\n .catch(e => {\n this.#stopRunningBackfill(String(e), state);\n this.#retryBackfillWithBackoff(e);\n });\n }\n }\n\n #retryBackfillWithBackoff(e: unknown) {\n const log = this.#retryDelayMs === this.#maxBackoffMs ? 'error' : 'warn';\n this.#lc[log]?.(\n `Error running backfill. Retrying in ${this.#retryDelayMs} ms`,\n e,\n );\n this.#backfillRetryTimer = setTimeout(() => {\n this.#backfillRetryTimer = undefined;\n this.#checkAndStartBackfill();\n }, this.#retryDelayMs);\n\n this.#retryDelayMs = Math.min(this.#retryDelayMs 2, this.#maxBackoffMs);\n }\n\n async #runBackfill(lc: LogContext, state: RunningBackfillState) {\n const changeStream = this.#changeStreamer; // Purely for readability\n\n // backfillTx is set if and only if a changeStreamer reservation has been\n // acquired and the backfill stream is inside a transaction.\n let backfillTx: string \| null = null;\n\n /*\n @returns the new tx watermark, or null if backfill was cancelled\n /\n const beginTxFor = async (\n msg: MessageBackfill \| BackfillCompleted,\n ): Promise<string \| null> => {\n assert(backfillTx === null, 'Expected no active backfill transaction');\n const lastWatermark = await changeStream.reserve('backfill');\n\n // After obtaining the changeStream reservation, check if the stream\n // had changes that resulted in invalidating / canceling this backfill.\n if (\n state.canceledReason \|\|\n (msg.tag === 'backfill' && msg.watermark < state.minWatermark)\n ) {\n if (state.canceledReason === undefined) {\n assert(msg.tag === 'backfill', 'Expected backfill message tag'); // TypeScript should have figured this out.\n this.#stopRunningBackfill(\n `row key change at ${state.minWatermark} ` +\n `postdates backfill watermark at ${msg.watermark}`,\n state,\n );\n }\n changeStream.release(lastWatermark);\n return null;\n }\n\n const {major, minor = 0n} = stateVersionFromString(lastWatermark);\n let tx = stateVersionToString({\n major,\n minor: BigInt(minor) + 1n,\n });\n\n if (msg.tag === 'backfill-completed' && tx < msg.watermark) {\n // At this point it must be the case that the #changeStreamReached() the\n // backfill watermark. Given that guarantee, ensure that the version of the\n // transaction containing the backfill-completed message is at least up\n // to the backfill watermark, so that the final database state version is\n // never earlier than the version of any backfilled rows.\n tx = msg.watermark;\n }\n\n void changeStream.push([\n 'begin',\n {tag: 'begin', json: this.#jsonFormat, skipAck: true},\n {commitWatermark: tx},\n ]);\n return (backfillTx = tx);\n };\n\n const commitTx = () => {\n if (backfillTx) {\n void changeStream.push([\n 'commit',\n {tag: 'commit'},\n {watermark: backfillTx},\n ]);\n changeStream.release(backfillTx);\n }\n backfillTx = null;\n };\n\n for await (const msg of this.#backfillStreamer(state.request)) {\n // Before sending `backfill-completed`, the main replication stream\n // may need to catch up, and/or the current transaction may need to be\n // committed to open a new transaction that's up to backfill watermark.\n const mustWaitBeforeFlush =\n msg.tag === 'backfill-completed' &&\n (this.#changeStreamReached(lc, msg.watermark) \|\|\n (backfillTx !== null && backfillTx < msg.watermark));\n\n // If necessary, yield the reservation to the main stream.\n if (\n backfillTx &&\n (changeStream.waiterDelay() > 0 \|\| mustWaitBeforeFlush)\n ) {\n commitTx();\n }\n\n mustWaitBeforeFlush && (await mustWaitBeforeFlush);\n\n if (\n msg.tag === 'backfill' &&\n msg.rowValues.length > 0 &&\n msg.relation.rowKey.columns.length === 0\n ) {\n throw new MissingRowKeyError(state.request);\n }\n\n // Reserve the changeStreamer if not in a transaction.\n if ((backfillTx ??= await beginTxFor(msg)) === null) {\n lc.info?.(\n `backfill stream canceled: ${state.canceledReason}`,\n state.request,\n );\n this.#checkAndStartBackfill(); // start the next backfill if present\n return; // this backfill is canceled\n }\n\n // `await` to allow the change streamer to exert back pressure\n // on backfills.\n await changeStream.push(['data', msg]);\n }\n\n // Flush any final tx and release the stream.\n backfillTx && commitTx();\n lc.debug?.(`backfill stream exited`, state.canceledReason ?? '');\n }\n\n #backfillRunningFor(table: Identifier): RunningBackfillState \| null {\n const state = this.#runningBackfill;\n return state?.request.table.schema === table.schema &&\n state.request.table.name === table.name\n ? state\n : null;\n }\n\n /\n Stops the running backfill for the specified `reason`. If `instance` is\n * specified, the running backfill is stopped only if it is that instance.\n * This allows the running backfill itself to clear backfill state without\n * accidentally stopping a different (e.g. subsequent) backfill.\n /\n #stopRunningBackfill(reason?: string, instance?: RunningBackfillState) {\n const backfill = this.#runningBackfill;\n if (backfill && backfill === (instance ?? backfill)) {\n backfill.canceledReason = reason;\n this.#runningBackfill = null;\n reason && this.#lc.info?.(`canceling backfill:`, reason);\n }\n }\n\n #setRequiredBackfill(source: string, req: BackfillRequest) {\n const action = this.#requiredBackfills.has(req.table) ? 'updated' : 'added';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.set(req.table, req);\n }\n\n #deleteRequiredBackfill(source: string, id: Identifier) {\n const req = this.#requiredBackfills.get(id);\n if (req) {\n const action = source === 'backfill-completed' ? 'completed' : 'dropped';\n this.#lc.info?.(`Backfill ${action}: ${source}`, {backfill: req});\n this.#requiredBackfills.delete(id);\n }\n }\n\n /\n Implements {@link Listener.onChange()}, invoked by the\n * {@link ChangeStreamMultiplexer}.\n /\n onChange(message: ChangeStreamMessage): void {\n if (message[0] === 'begin') {\n this.#currentTxWatermark = message[2].commitWatermark;\n return;\n }\n if (message[0] === 'commit') {\n this.#currentTxWatermark = null;\n this.#setLastStatusWatermark(message[2]);\n // Every commit is a candidate for starting the next backfill\n // (if one is not currently running).\n this.#checkAndStartBackfill();\n return;\n }\n if (message[0] === 'status') {\n this.#setLastStatusWatermark(message[2]);\n return;\n }\n if (message[0] !== 'data') {\n return;\n }\n const change = message[1];\n const {tag} = change;\n switch (tag) {\n case 'update-table-metadata': {\n const {table, new: metadata} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest) {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n });\n if (this.#backfillRunningFor(table)) {\n this.#stopRunningBackfill(`TableMetadata updated`);\n }\n }\n break;\n }\n case 'create-table': {\n const {\n spec: {schema, name},\n metadata = null,\n backfill,\n } = change;\n\n if (backfill) {\n this.#setRequiredBackfill(tag, {\n table: {schema, name, metadata},\n columns: backfill,\n });\n }\n break;\n }\n case 'rename-table': {\n const {old, new: newTable} = change;\n const backfillRequest = this.#requiredBackfills.get(old);\n if (backfillRequest) {\n const {schema, name} = newTable;\n this.#deleteRequiredBackfill(tag, old);\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, schema, name},\n });\n if (this.#backfillRunningFor(old)) {\n this.#stopRunningBackfill(`table renamed`);\n }\n }\n break;\n }\n case 'drop-table': {\n const {id} = change;\n const backfillRequest = this.#requiredBackfills.get(id);\n if (backfillRequest) {\n this.#deleteRequiredBackfill(tag, id);\n if (this.#backfillRunningFor(id)) {\n this.#stopRunningBackfill(`table dropped`);\n }\n }\n break;\n }\n case 'add-column': {\n const {\n table,\n tableMetadata: metadata = null,\n column,\n backfill,\n } = change;\n if (backfill) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (!backfillRequest) {\n this.#setRequiredBackfill(tag, {\n table: {...table, metadata},\n columns: {[column.name]: backfill},\n });\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n table: {...backfillRequest.table, metadata},\n columns: {\n ...backfillRequest.columns,\n [column.name]: backfill,\n },\n });\n // Note: The running backfill need not be canceled if a\n // new column is added. The new column will be backfilled\n // by its own stream after the current backfill completes.\n }\n }\n break;\n }\n case 'update-column': {\n const {\n table,\n old: {name: oldName},\n new: {name: newName},\n } = change;\n if (oldName !== newName) {\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && oldName in backfillRequest.columns) {\n const {[oldName]: colSpec, ...otherCols} = backfillRequest.columns;\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: {...otherCols, [newName]: colSpec},\n });\n const backfill = this.#backfillRunningFor(table);\n if (backfill && oldName in backfill.request.columns) {\n this.#stopRunningBackfill(`column renamed`);\n }\n }\n }\n break;\n }\n case 'drop-column': {\n const {table, column} = change;\n const backfillRequest = this.#requiredBackfills.get(table);\n if (backfillRequest && column in backfillRequest.columns) {\n const {[column]: _excluded, ...remaining} = backfillRequest.columns;\n if (Object.keys(remaining).length === 0) {\n this.#deleteRequiredBackfill(tag, table);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: remaining,\n });\n }\n const backfill = this.#backfillRunningFor(table);\n if (backfill && column in backfill.request.columns) {\n this.#stopRunningBackfill(`column dropped`);\n }\n }\n break;\n }\n case 'update': {\n const {relation, key, new: row} = change;\n const backfill = this.#backfillRunningFor(relation);\n const txWatermark = must(this.#currentTxWatermark, `not in a tx`);\n if (backfill?.request.table.metadata && key !== null) {\n // A corner case that backfill is unable to correctly handle is\n // when a row's key changes; this is decomposed into a delete\n // of the old key and a set of the new key in the replica change\n // log, at which point the backfill algorithm assumes that the\n // (old) row is deleted but does not know to backfill the new row.\n // In these corner cases, the current backfill is canceled and\n // retried if its version precedes this update.\n for (const col of Object.keys(\n backfill.request.table.metadata.rowKey,\n )) {\n if (key[col] !== row[col]) {\n backfill.minWatermark = txWatermark;\n this.#lc.info?.(\n `key for row as changed (col: ${col}). ` +\n `backfill data must not predate ${backfill.minWatermark}`,\n );\n break;\n }\n }\n }\n break;\n }\n case 'backfill-completed': {\n const {relation, columns} = change;\n const backfillRequest = this.#requiredBackfills.get(relation);\n assert(\n backfillRequest,\n () => `No BackfillRequest completed backfill ${stringify(change)}`,\n );\n const remaining = Object.entries(backfillRequest.columns).filter(\n ([col]) =>\n !(columns.includes(col) \|\| relation.rowKey.columns.includes(col)),\n );\n if (remaining.length === 0) {\n this.#deleteRequiredBackfill(tag, relation);\n } else {\n this.#setRequiredBackfill(tag, {\n ...backfillRequest,\n columns: Object.fromEntries(remaining),\n });\n }\n // Technically the backfill is already stopping, but this method\n // cleans up the state that tracks it.\n this.#stopRunningBackfill();\n break;\n }\n }\n }\n\n cancel(): void {\n this.#stopRunningBackfill(`change stream canceled`);\n clearTimeout(this.#backfillRetryTimer);\n }\n}\n\nabstract class BackfillStreamError extends Error {\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(\n `Cannot backfill ${bf.table.schema}.${bf.table.name}` +\n `[${Object.keys(bf.columns).join(',')}]: ${msg}`,\n {cause},\n );\n }\n}\n\n/\n Background: The zero-cache supports replication of tables without a\n * PRIMARY KEY to facilitate the onboarding process. These rows can be\n * INSERT'ed, but postgres will rightfully prohibit UPDATEs and DELETEs\n * on such tables because the rows cannot be identified by a key. Supporting\n * this mode of replication allows the user to \"fix\" the setup by adding the\n * primary key, after which the table can be published downstream without\n * requiring a resync of the data.\n \n In terms of backfill, however, non-empty tables without a row key cannot\n * be backfilled, because backfill retries would result in writing duplicating\n * rows. (Empty tables, on the other hand, are fine because there is no data\n * to be deduped.)\n \n The MissingRowKeyError is used to signal that the table cannot be backfilled\n * in its current state. For simplicity, it is handled like runtime errors and\n * retried with backoff, with which it can eventually succeed if (1) a primary\n * key is added or (2) the table is emptied, e.g. via a TRUNCATE.\n /\nclass MissingRowKeyError extends BackfillStreamError {\n readonly name = 'MissingRowKeyError';\n\n constructor(bf: BackfillRequest, cause?: unknown) {\n super(bf, `\"${bf.table.name}\" is missing a PRIMARY KEY`, cause);\n }\n}\n\n/\n Error type for backfill stream implementations to throw indicating that\n * the backfill request failed due to a schema incompatibility error. This\n * type of error does not need exponential backoff, as the retry happens\n * naturally once the invalidating schema change is processed and committed.\n */\nexport class SchemaIncompatibilityError extends BackfillStreamError {\n readonly name = 'SchemaIncompatibilityError';\n\n constructor(bf: BackfillRequest, msg: string, cause?: unknown) {\n super(bf, msg, cause);\n }\n}\n"],"mappings":";;;;;;;;;AAyBA,SAAS,SAAS,EAAC,QAAQ,QAAmB;AAC5C,QAAO,GAAG,OAAO,GAAG;;AAatB,IAAM,0BAA0B;AAChC,IAAM,0BAA0B;;;;;;;;;;;;;;;;;AAuBhC,IAAa,kBAAb,MAA6D;CAC3D;;;;;CAMA,qBAA8B,IAAI,aAChC,SACD;CACD;CACA;CACA;;;;;;;CAQA,mBAAgD;;CAGhD,uBAAsC;CAEtC,4BAAgE,EAAE;;CAGlE,sBAAqC;CAErC,YACE,IACA,gBACA,kBACA,aAAA,KACA,eAAe,yBACf,eAAe,yBACf;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,mBAAmB;AAC1D,QAAA,iBAAuB;AACvB,QAAA,mBAAyB;AACzB,QAAA,aAAmB;AACnB,QAAA,eAAqB;AACrB,QAAA,eAAqB;AACrB,QAAA,eAAqB;;CAGvB,IAAI,eAAuB,iBAAoC;AAC7D,QAAA,GAAS,OACP,kCAAkC,gBAAgB,OAAO,oBACzD,EAAC,UAAU,iBAAgB,CAC5B;AACD,QAAA,sBAA4B;AAC5B,kBAAgB,SAAQ,QACtB,MAAA,oBAA0B,mBAAmB,IAAI,CAClD;AACD,QAAA,uBAA6B;;CAG/B,wBAAwB,EAAC,aAAiC;AAIxD,OAAK,MAAA,uBAA6B,MAAM,WAAW;AACjD,SAAA,sBAA4B;AAC5B,QAAK,IAAI,IAAI,MAAA,yBAA+B,SAAS,GAAG,KAAK,GAAG,KAAK;IACnE,MAAM,WAAW,MAAA,yBAA+B;AAChD,QAAI,aAAa,SAAS,WAAW;AACnC,cAAS,SAAS;AAClB,WAAA,yBAA+B,OAAO,GAAG,EAAE;;;;;CAMnD,qBACE,IACA,WACsB;AACtB,OAAK,MAAA,uBAA6B,MAAM,WAAW;GACjD,MAAM,EAAC,SAAS,SAAS,YAAW,UAAU;AAC9C,SAAA,yBAA+B,KAAK;IAAC;IAAW;IAAQ,CAAC;AACzD,MAAG,OACD,iCAAiC,MAAA,oBAA0B,aAAa,YACzE;AACD,UAAO;;AAET,SAAO;;CAGT;CACA;CACA;CACA;CAEA,yBAAyB;AACvB,MACE,CAAC,MAAA,sBACD,CAAC,MAAA,mBACD,MAAA,kBAAwB,MACxB;GAKA,MAAM,aAAa,CAAC,GAAG,MAAA,kBAAwB,QAAQ,CAAC;GACxD,MAAM,UAAU,WAAW,QAAQ,GAAG,WAAW,SAAS,EAAE;GAC5D,MAAM,QAAQ;IAAC;IAAS,cAAc;IAAG;GACzC,MAAM,KAAK,MAAA,GAAS,YAAY,SAAS,QAAQ,MAAM,KAAK;AAE5D,SAAA,kBAAwB;AACnB,SAAA,YAAkB,IAAI,MAAM,CAC9B,WAAW;AACV,UAAA,oBAA0B,mBAAmB,MAAM;AACnD,UAAA,eAAqB,MAAA;KACrB,CAGD,OAAM,MAAK;AACV,UAAA,oBAA0B,OAAO,EAAE,EAAE,MAAM;AAC3C,UAAA,yBAA+B,EAAE;KACjC;;;CAIR,0BAA0B,GAAY;EACpC,MAAM,MAAM,MAAA,iBAAuB,MAAA,eAAqB,UAAU;AAClE,QAAA,GAAS,OACP,uCAAuC,MAAA,aAAmB,MAC1D,EACD;AACD,QAAA,qBAA2B,iBAAiB;AAC1C,SAAA,qBAA2B,KAAA;AAC3B,SAAA,uBAA6B;KAC5B,MAAA,aAAmB;AAEtB,QAAA,eAAqB,KAAK,IAAI,MAAA,eAAqB,GAAG,MAAA,aAAmB;;CAG3E,OAAA,YAAmB,IAAgB,OAA6B;EAC9D,MAAM,eAAe,MAAA;EAIrB,IAAI,aAA4B;;;;EAKhC,MAAM,aAAa,OACjB,QAC2B;AAC3B,UAAO,eAAe,MAAM,0CAA0C;GACtE,MAAM,gBAAgB,MAAM,aAAa,QAAQ,WAAW;AAI5D,OACE,MAAM,kBACL,IAAI,QAAQ,cAAc,IAAI,YAAY,MAAM,cACjD;AACA,QAAI,MAAM,mBAAmB,KAAA,GAAW;AACtC,YAAO,IAAI,QAAQ,YAAY,gCAAgC;AAC/D,WAAA,oBACE,qBAAqB,MAAM,aAAa,mCACH,IAAI,aACzC,MACD;;AAEH,iBAAa,QAAQ,cAAc;AACnC,WAAO;;GAGT,MAAM,EAAC,OAAO,QAAQ,OAAM,uBAAuB,cAAc;GACjE,IAAI,KAAK,qBAAqB;IAC5B;IACA,OAAO,OAAO,MAAM,GAAG;IACxB,CAAC;AAEF,OAAI,IAAI,QAAQ,wBAAwB,KAAK,IAAI,UAM/C,MAAK,IAAI;AAGN,gBAAa,KAAK;IACrB;IACA;KAAC,KAAK;KAAS,MAAM,MAAA;KAAkB,SAAS;KAAK;IACrD,EAAC,iBAAiB,IAAG;IACtB,CAAC;AACF,UAAQ,aAAa;;EAGvB,MAAM,iBAAiB;AACrB,OAAI,YAAY;AACT,iBAAa,KAAK;KACrB;KACA,EAAC,KAAK,UAAS;KACf,EAAC,WAAW,YAAW;KACxB,CAAC;AACF,iBAAa,QAAQ,WAAW;;AAElC,gBAAa;;AAGf,aAAW,MAAM,OAAO,MAAA,iBAAuB,MAAM,QAAQ,EAAE;GAI7D,MAAM,sBACJ,IAAI,QAAQ,yBACX,MAAA,oBAA0B,IAAI,IAAI,UAAU,IAC1C,eAAe,QAAQ,aAAa,IAAI;AAG7C,OACE,eACC,aAAa,aAAa,GAAG,KAAK,qBAEnC,WAAU;AAGZ,0BAAwB,MAAM;AAE9B,OACE,IAAI,QAAQ,cACZ,IAAI,UAAU,SAAS,KACvB,IAAI,SAAS,OAAO,QAAQ,WAAW,EAEvC,OAAM,IAAI,mBAAmB,MAAM,QAAQ;AAI7C,QAAK,eAAe,MAAM,WAAW,IAAI,MAAM,MAAM;AACnD,OAAG,OACD,6BAA6B,MAAM,kBACnC,MAAM,QACP;AACD,UAAA,uBAA6B;AAC7B;;AAKF,SAAM,aAAa,KAAK,CAAC,QAAQ,IAAI,CAAC;;AAIxC,gBAAc,UAAU;AACxB,KAAG,QAAQ,0BAA0B,MAAM,kBAAkB,GAAG;;CAGlE,oBAAoB,OAAgD;EAClE,MAAM,QAAQ,MAAA;AACd,SAAO,OAAO,QAAQ,MAAM,WAAW,MAAM,UAC3C,MAAM,QAAQ,MAAM,SAAS,MAAM,OACjC,QACA;;;;;;;;CASN,qBAAqB,QAAiB,UAAiC;EACrE,MAAM,WAAW,MAAA;AACjB,MAAI,YAAY,cAAc,YAAY,WAAW;AACnD,YAAS,iBAAiB;AAC1B,SAAA,kBAAwB;AACxB,aAAU,MAAA,GAAS,OAAO,uBAAuB,OAAO;;;CAI5D,qBAAqB,QAAgB,KAAsB;EACzD,MAAM,SAAS,MAAA,kBAAwB,IAAI,IAAI,MAAM,GAAG,YAAY;AACpE,QAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,QAAA,kBAAwB,IAAI,IAAI,OAAO,IAAI;;CAG7C,wBAAwB,QAAgB,IAAgB;EACtD,MAAM,MAAM,MAAA,kBAAwB,IAAI,GAAG;AAC3C,MAAI,KAAK;GACP,MAAM,SAAS,WAAW,uBAAuB,cAAc;AAC/D,SAAA,GAAS,OAAO,YAAY,OAAO,IAAI,UAAU,EAAC,UAAU,KAAI,CAAC;AACjE,SAAA,kBAAwB,OAAO,GAAG;;;;;;;CAQtC,SAAS,SAAoC;AAC3C,MAAI,QAAQ,OAAO,SAAS;AAC1B,SAAA,qBAA2B,QAAQ,GAAG;AACtC;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,qBAA2B;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AAGxC,SAAA,uBAA6B;AAC7B;;AAEF,MAAI,QAAQ,OAAO,UAAU;AAC3B,SAAA,uBAA6B,QAAQ,GAAG;AACxC;;AAEF,MAAI,QAAQ,OAAO,OACjB;EAEF,MAAM,SAAS,QAAQ;EACvB,MAAM,EAAC,QAAO;AACd,UAAQ,KAAR;GACE,KAAK,yBAAyB;IAC5B,MAAM,EAAC,OAAO,KAAK,aAAY;IAC/B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,iBAAiB;AACnB,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC5C,CAAC;AACF,SAAI,MAAA,mBAAyB,MAAM,CACjC,OAAA,oBAA0B,wBAAwB;;AAGtD;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EACJ,MAAM,EAAC,QAAQ,QACf,WAAW,MACX,aACE;AAEJ,QAAI,SACF,OAAA,oBAA0B,KAAK;KAC7B,OAAO;MAAC;MAAQ;MAAM;MAAS;KAC/B,SAAS;KACV,CAAC;AAEJ;;GAEF,KAAK,gBAAgB;IACnB,MAAM,EAAC,KAAK,KAAK,aAAY;IAC7B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,IAAI;AACxD,QAAI,iBAAiB;KACnB,MAAM,EAAC,QAAQ,SAAQ;AACvB,WAAA,uBAA6B,KAAK,IAAI;AACtC,WAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAQ;OAAK;MAChD,CAAC;AACF,SAAI,MAAA,mBAAyB,IAAI,CAC/B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EAAC,OAAM;AAEb,QADwB,MAAA,kBAAwB,IAAI,GAAG,EAClC;AACnB,WAAA,uBAA6B,KAAK,GAAG;AACrC,SAAI,MAAA,mBAAyB,GAAG,CAC9B,OAAA,oBAA0B,gBAAgB;;AAG9C;;GAEF,KAAK,cAAc;IACjB,MAAM,EACJ,OACA,eAAe,WAAW,MAC1B,QACA,aACE;AACJ,QAAI,UAAU;KACZ,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,CAAC,gBACH,OAAA,oBAA0B,KAAK;MAC7B,OAAO;OAAC,GAAG;OAAO;OAAS;MAC3B,SAAS,GAAE,OAAO,OAAO,UAAS;MACnC,CAAC;SAEF,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,OAAO;OAAC,GAAG,gBAAgB;OAAO;OAAS;MAC3C,SAAS;OACP,GAAG,gBAAgB;QAClB,OAAO,OAAO;OAChB;MACF,CAAC;;AAMN;;GAEF,KAAK,iBAAiB;IACpB,MAAM,EACJ,OACA,KAAK,EAAC,MAAM,WACZ,KAAK,EAAC,MAAM,cACV;AACJ,QAAI,YAAY,SAAS;KACvB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,SAAI,mBAAmB,WAAW,gBAAgB,SAAS;MACzD,MAAM,GAAE,UAAU,SAAS,GAAG,cAAa,gBAAgB;AAC3D,YAAA,oBAA0B,KAAK;OAC7B,GAAG;OACH,SAAS;QAAC,GAAG;SAAY,UAAU;QAAQ;OAC5C,CAAC;MACF,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,UAAI,YAAY,WAAW,SAAS,QAAQ,QAC1C,OAAA,oBAA0B,iBAAiB;;;AAIjD;;GAEF,KAAK,eAAe;IAClB,MAAM,EAAC,OAAO,WAAU;IACxB,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,MAAM;AAC1D,QAAI,mBAAmB,UAAU,gBAAgB,SAAS;KACxD,MAAM,GAAE,SAAS,WAAW,GAAG,cAAa,gBAAgB;AAC5D,SAAI,OAAO,KAAK,UAAU,CAAC,WAAW,EACpC,OAAA,uBAA6B,KAAK,MAAM;SAExC,OAAA,oBAA0B,KAAK;MAC7B,GAAG;MACH,SAAS;MACV,CAAC;KAEJ,MAAM,WAAW,MAAA,mBAAyB,MAAM;AAChD,SAAI,YAAY,UAAU,SAAS,QAAQ,QACzC,OAAA,oBAA0B,iBAAiB;;AAG/C;;GAEF,KAAK,UAAU;IACb,MAAM,EAAC,UAAU,KAAK,KAAK,QAAO;IAClC,MAAM,WAAW,MAAA,mBAAyB,SAAS;IACnD,MAAM,cAAc,KAAK,MAAA,oBAA0B,cAAc;AACjE,QAAI,UAAU,QAAQ,MAAM,YAAY,QAAQ;UAQzC,MAAM,OAAO,OAAO,KACvB,SAAS,QAAQ,MAAM,SAAS,OACjC,CACC,KAAI,IAAI,SAAS,IAAI,MAAM;AACzB,eAAS,eAAe;AACxB,YAAA,GAAS,OACP,gCAAgC,IAAI,oCACA,SAAS,eAC9C;AACD;;;AAIN;;GAEF,KAAK,sBAAsB;IACzB,MAAM,EAAC,UAAU,YAAW;IAC5B,MAAM,kBAAkB,MAAA,kBAAwB,IAAI,SAAS;AAC7D,WACE,uBACM,yCAAyC,UAAU,OAAO,GACjE;IACD,MAAM,YAAY,OAAO,QAAQ,gBAAgB,QAAQ,CAAC,QACvD,CAAC,SACA,EAAE,QAAQ,SAAS,IAAI,IAAI,SAAS,OAAO,QAAQ,SAAS,IAAI,EACnE;AACD,QAAI,UAAU,WAAW,EACvB,OAAA,uBAA6B,KAAK,SAAS;QAE3C,OAAA,oBAA0B,KAAK;KAC7B,GAAG;KACH,SAAS,OAAO,YAAY,UAAU;KACvC,CAAC;AAIJ,UAAA,qBAA2B;AAC3B;;;;CAKN,SAAe;AACb,QAAA,oBAA0B,yBAAyB;AACnD,eAAa,MAAA,mBAAyB;;;AAI1C,IAAe,sBAAf,cAA2C,MAAM;CAC/C,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QACE,mBAAmB,GAAG,MAAM,OAAO,GAAG,GAAG,MAAM,KAAA,GACzC,OAAO,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,KAAK,OAC7C,EAAC,OAAM,CACR;;;;;;;;;;;;;;;;;;;;;;AAuBL,IAAM,qBAAN,cAAiC,oBAAoB;CACnD,OAAgB;CAEhB,YAAY,IAAqB,OAAiB;AAChD,QAAM,IAAI,IAAI,GAAG,MAAM,KAAK,6BAA6B,MAAM;;;;;;;;;AAUnE,IAAa,6BAAb,cAAgD,oBAAoB;CAClE,OAAgB;CAEhB,YAAY,IAAqB,KAAa,OAAiB;AAC7D,QAAM,IAAI,KAAK,MAAM"}

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAQnE,OAAO,EAIL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAQpE,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CA+B7E;~~AAuED~~;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}
1	+ {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,EAEL,KAAK,UAAU,EAChB,MAAM,0CAA0C,CAAC;AAGlD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAQnE,OAAO,EAIL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAQpE,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CA+B7E;AA2ED;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,iBAqFvB"}

package/out/zero-cache/src/services/change-source/custom/change-source.js CHANGED Viewed

@@ -24,7 +24,7 @@ async function initializeCustomChangeSource(lc, upstreamURI, shard, replicaDbFil
 	const subscriptionState = getSubscriptionState(new StatementRunner(replica));
 	replica.close();
 	if (shard.publications.length) {
-		const requested = [...shard.publications].sort();
+		const requested = shard.publications.toSorted();
 		const replicated = subscriptionState.publications.sort();
 		if (!deepEqual(requested, replicated)) throw new Error(`Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`);
 	}
@@ -50,6 +50,9 @@ var CustomChangeSource = class {
 	startLagReporter() {
 		return null;
 	}
+	stop() {
+		return Promise.resolve();
+	}
 	startStream(clientWatermark, backfillRequests = []) {
 		if (backfillRequests?.length) throw new Error("backfill is yet not supported for custom change sources");
 		return Promise.resolve(this.#startStream(clientWatermark));
@@ -105,7 +108,7 @@ async function initialSync(lc, shard, tx, upstreamURI, context) {
 					const { commitWatermark } = change[2];
 					lc.info?.(`initial sync of shard ${id} at replicaVersion ${commitWatermark}`);
 					statusPublisher.publish(lc, "Initializing", `Copying upstream tables at version ${commitWatermark}`, 5e3);
-					initReplicationState(tx, [...publications].sort(), commitWatermark, context, false);
+					initReplicationState(tx, publications.toSorted(), commitWatermark, context, false);
 					processor.processMessage(lc, change);
 					break;
 				}

package/out/zero-cache/src/services/change-source/custom/change-source.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = [...shard.publications].sort();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n [...publications].sort(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,CAAC,GAAG,MAAM,aAAa,CAAC,MAAM;EAChD,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,YAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,CAAC,GAAG,aAAa,CAAC,MAAM,EACxB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}
1	+ {"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. /\nexport type ServerContext = JSONObject;\n\n/\n Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n /\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = shard.publications.toSorted();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n stop(): Promise<void> {\n return Promise.resolve();\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/\n Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n \n Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n publications.toSorted(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,MAAM,aAAa,UAAU;EAC/C,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,OAAsB;AACpB,SAAO,QAAQ,SAAS;;CAG1B,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,YAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,aAAa,UAAU,EACvB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAejD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;~~AAEhE~~,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;~~AA6BjD~~;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAsC7E;~~AA+bD~~,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;~~AA8vBxD~~,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
1	+ {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAejD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAGhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA2BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,SAAI,GACtB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAsC7E;AAqcD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAmwBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js CHANGED Viewed

@@ -22,6 +22,7 @@ import { validate } from "./schema/validation.js";
 import { dropShard, getInternalShardConfig, getReplicaAtVersion, internalPublicationPrefix, legacyReplicationSlot, replicaIdentitiesForTablesWithoutPrimaryKeys, replicationSlotExpression } from "./schema/shard.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { initReplica } from "../common/replica-schema.js";
+import "../../../types/pg-versions.js";
 import { BackfillManager } from "../common/backfill-manager.js";
 import { ChangeStreamMultiplexer } from "../common/change-stream-multiplexer.js";
 import { updateShardSchema } from "./schema/init.js";
@@ -32,7 +33,6 @@ import postgres from "postgres";
 import { nanoid } from "nanoid";
 import { PG_ADMIN_SHUTDOWN, PG_INSUFFICIENT_PRIVILEGE, PG_OBJECT_IN_USE } from "@drdgvhbh/postgres-error-codes";
 //#region ../zero-cache/src/services/change-source/pg/change-source.ts
-var PG_17 = 17e4;
 /**
 * Initializes a Postgres change source, including the initial sync of the
 * replica, before streaming changes from the corresponding logical replication
@@ -57,7 +57,7 @@ async function checkAndUpdateUpstream(lc, sql, shard, { replicaVersion, publicat
 	await updateShardSchema(lc, sql, shard, replicaVersion);
 	const upstreamReplica = await getReplicaAtVersion(lc, sql, shard, replicaVersion, initialSyncContext);
 	if (!upstreamReplica) throw new AutoResetSignal(`No replication slot for replica at version ${replicaVersion}`);
-	const requested = [...shard.publications].sort();
+	const requested = shard.publications.toSorted();
 	const replicated = upstreamReplica.publications.filter((p) => !p.startsWith(internalPublicationPrefix(shard))).sort();
 	if (!deepEqual(requested, replicated)) {
 		lc.warn?.(`Dropping shard to change publications to: [${requested}]`);
@@ -94,6 +94,7 @@ var PostgresChangeSource = class {
 	constructor(lc, upstreamUri, shard, replica, context, lagReportIntervalMs) {
 		this.#lc = lc.withContext("component", "change-source");
 		this.#db = pgClient(lc, upstreamUri, {
+			max: 1,
 			["idle_timeout"]: 60,
 			connection: { ["application_name"]: "zero-replication-monitor" }
 		});
@@ -103,12 +104,16 @@ var PostgresChangeSource = class {
 		this.#context = context;
 		this.#lagReporter = lagReportIntervalMs > 0 ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, lagReportIntervalMs) : null;
 	}
+	async stop() {
+		this.#lagReporter?.stop();
+		await this.#db.end();
+	}
 	async startLagReporter() {
 		if (this.#lagReporter) try {
 			return await this.#lagReporter.initiateLagReport(true);
 		} catch (e) {
 			if (isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {
-				const functionName = (this.#lagReporter.pgVersion ?? 0) >= PG_17 ? "pg_logical_emit_message(boolean, text, text, boolean)" : "pg_logical_emit_message(boolean, text, text)";
+				const functionName = (this.#lagReporter.pgVersion ?? 0) >= 17e4 ? "pg_logical_emit_message(boolean, text, text, boolean)" : "pg_logical_emit_message(boolean, text, text)";
 				this.#lc.warn?.("\n\nUnable to initiate replication lag reports due to insufficient privileges.\nTo enable replication lag reporting, run:", `\n\tGRANT EXECUTE ON FUNCTION ${functionName} TO <your_db_user>;\n\n`, e);
 			} else this.#lc.error?.(`Unexpected error while initiating lag reports. Lag reports will be disabled.`, e);
 		}
@@ -354,7 +359,7 @@ var LagReporter = class LagReporter {
 		this.#expectingLagReport = lagReport;
 		let commitTimeMs;
 		let lsn;
-		if (pgVersion >= PG_17) [{commitTimeMs, lsn}] = await this.#db`
+		if (pgVersion >= 17e4) [{commitTimeMs, lsn}] = await this.#db`
         WITH CTE AS (SELECT extract(epoch from now()) * 1000 AS "commitTimeMs")
         SELECT "commitTimeMs", pg_logical_emit_message(
           false,
@@ -414,6 +419,10 @@ var LagReporter = class LagReporter {
 			return this.#processLagReport(this.#expectingLagReport, majorVersionToString(lsn));
 		}
 	}
+	stop() {
+		clearTimeout(this.#timer);
+		this.#timer = void 0;
+	}
 	#scheduleNextReport(delayMs) {
 		this.#expectingLagReport = null;
 		clearTimeout(this.#timer);