npm - @rocicorp/zero - Versions diffs - 0.25.13-canary.8 → 0.25.13 - Mend

@rocicorp/zero 0.25.13-canary.8 → 0.25.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (19) hide show

package/out/zero/package.json.js CHANGED Viewed

@@ -1,4 +1,4 @@
-const version = "0.25.13-canary.8";
+const version = "0.25.13";
 const packageJson = {
   version
 };

package/out/zero-cache/src/server/main.js CHANGED Viewed

@@ -82,7 +82,7 @@ async function runWorker(parent, env) {
       "message",
       () => {
         processes.addSubprocess(
-          startReplicaBackupProcess(config),
+          startReplicaBackupProcess(lc, config),
           "supporting",
           "litestream"
         );

package/out/zero-cache/src/server/main.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"main.js","sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import {resolver} from '@rocicorp/resolver';\nimport path from 'node:path';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n restoreReplica,\n startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(createLogContext(config, {worker: 'dispatcher'}, false));\n const lc = createLogContext(config, {worker: 'dispatcher'}, true);\n initEventSink(lc, config);\n\n const processes = new ProcessManager(lc, parent);\n\n const {numSyncWorkers: numSyncers} = config;\n if (config.upstream.maxConns < numSyncers) {\n throw new Error(\n `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n if (config.cvr.maxConns < numSyncers) {\n throw new Error(\n `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n\n const internalFlags: string[] =\n numSyncers === 0\n ? []\n : [\n '--upstream-max-conns-per-worker',\n String(Math.floor(config.upstream.maxConns / numSyncers)),\n '--cvr-max-conns-per-worker',\n String(Math.floor(config.cvr.maxConns / numSyncers)),\n ];\n\n function loadWorker(\n moduleUrl: URL,\n type: WorkerType,\n id?: string \| number,\n ...args: string[]\n ): Worker {\n const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n return processes.addWorker(worker, type, name);\n }\n\n const {\n taskID,\n changeStreamer: {mode: changeStreamerMode, uri: changeStreamerURI},\n litestream,\n } = config;\n const runChangeStreamer =\n changeStreamerMode === 'dedicated' && changeStreamerURI === undefined;\n\n let restoreStart = new Date();\n if (litestream.backupURL \|\| (litestream.executable && !runChangeStreamer)) {\n try {\n restoreStart = await restoreReplica(lc, config);\n } catch (e) {\n if (runChangeStreamer) {\n // If the restore failed, e.g. due to a corrupt backup, the\n // replication-manager recovers by re-syncing.\n lc.error?.('error restoring backup. resyncing the replica.', e);\n } else {\n // View-syncers, on the other hand, have no option other than to retry\n // until a valid backup has been published. This is achieved by\n // shutting down and letting the container runner retry with its\n // configured policy.\n throw e;\n }\n }\n }\n\n const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n resolver();\n const changeStreamer = runChangeStreamer\n ? loadWorker(\n CHANGE_STREAMER_URL,\n 'supporting',\n undefined,\n String(restoreStart.getTime()),\n ).once('message', changeStreamerStarted)\n : (changeStreamerStarted() ?? undefined);\n\n const {promise: reaperReady, resolve: reaperStarted} = resolver();\n if (numSyncers > 0) {\n loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n } else {\n reaperStarted();\n }\n\n // Wait for the change-streamer to be ready to guarantee that a replica\n // file is present.\n await changeStreamerReady;\n\n if (runChangeStreamer && litestream.backupURL) {\n // Start a backup replicator and corresponding litestream backup process.\n const {promise: backupReady, resolve} = resolver();\n const mode: ReplicaFileMode = 'backup';\n loadWorker(REPLICATOR_URL, 'supporting', mode, mode).once(\n // Wait for the Replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the lock when the\n // replicator first prepares the db file.\n 'message',\n () => {\n processes.addSubprocess(\n startReplicaBackupProcess(config),\n 'supporting',\n 'litestream',\n );\n resolve();\n },\n );\n await backupReady;\n }\n\n // Before starting the view-syncers, ensure that the reaper has started\n // up, indicating that any CVR db migrations have been performed.\n await reaperReady;\n\n const syncers: Worker[] = [];\n if (numSyncers) {\n const mode: ReplicaFileMode =\n runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n const {promise: replicaReady, resolve} = resolver();\n const replicator = loadWorker(\n REPLICATOR_URL,\n 'supporting',\n mode,\n mode,\n ).once('message', () => {\n subscribeTo(lc, replicator);\n resolve();\n });\n await replicaReady;\n\n const notifier = createNotifierFrom(lc, replicator);\n for (let i = 0; i < numSyncers; i++) {\n syncers.push(loadWorker(SYNCER_URL, 'user-facing', i + 1, mode));\n }\n syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n }\n let mutator: Worker \| undefined;\n if (clientConnectionBifurcated) {\n mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n }\n\n lc.info?.('waiting for workers to be ready ...');\n const logWaiting = setInterval(\n () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n 10_000,\n );\n await processes.allWorkersReady();\n clearInterval(logWaiting);\n lc.info?.(`all workers ready (${Date.now() - startMs} ms)`);\n\n parent.send(['ready', {ready: true}]);\n\n try {\n await runUntilKilled(\n lc,\n parent,\n new WorkerDispatcher(\n lc,\n taskID,\n parent,\n syncers,\n mutator,\n changeStreamer,\n ),\n );\n } catch (err) {\n processes.logErrorAndExit(err, 'dispatcher');\n }\n\n await processes.done();\n}\n\nif (!singleProcessMode()) {\n void exitAfter(() => runWorker(must(parentWorker), process.env));\n}\n"],"names":[],"mappings":";;;;;;;;;;;;;AAwCA,eAA8B,UAC5B,QACA,KACe;AACf,QAAM,UAAU,KAAK,IAAA;AACrB,QAAM,SAAS,wBAAwB,EAAC,KAAI;AAE5C,gBAAc,iBAAiB,QAAQ,EAAC,QAAQ,aAAA,GAAe,KAAK,CAAC;AACrE,QAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,aAAA,GAAe,IAAI;AAChE,gBAAc,IAAI,MAAM;AAExB,QAAM,YAAY,IAAI,eAAe,IAAI,MAAM;AAE/C,QAAM,EAAC,gBAAgB,WAAA,IAAc;AACrC,MAAI,OAAO,SAAS,WAAW,YAAY;AACzC,UAAM,IAAI;AAAA,MACR,sCAAsC,OAAO,SAAS,QAAQ,SAAS,UAAU;AAAA,IAAA;AAAA,EAGrF;AACA,MAAI,OAAO,IAAI,WAAW,YAAY;AACpC,UAAM,IAAI;AAAA,MACR,iCAAiC,OAAO,IAAI,QAAQ,SAAS,UAAU;AAAA,IAAA;AAAA,EAG3E;AAEA,QAAM,gBACJ,eAAe,IACX,KACA;AAAA,IACE;AAAA,IACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;AAAA,IACxD;AAAA,IACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;AAAA,EAAA;AAG3D,WAAS,WACP,WACA,MACA,OACG,MACK;AACR,UAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;AACpE,UAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,EAAE,MAAM;AACpE,WAAO,UAAU,UAAU,QAAQ,MAAM,IAAI;AAAA,EAC/C;AAEA,QAAM;AAAA,IACJ;AAAA,IACA,gBAAgB,EAAC,MAAM,oBAAoB,KAAK,kBAAA;AAAA,IAChD;AAAA,EAAA,IACE;AACJ,QAAM,oBACJ,uBAAuB,eAAe,sBAAsB;AAE9D,MAAI,mCAAmB,KAAA;AACvB,MAAI,WAAW,aAAc,WAAW,cAAc,CAAC,mBAAoB;AACzE,QAAI;AACF,qBAAe,MAAM,eAAe,IAAI,MAAM;AAAA,IAChD,SAAS,GAAG;AACV,UAAI,mBAAmB;AAGrB,WAAG,QAAQ,kDAAkD,CAAC;AAAA,MAChE,OAAO;AAKL,cAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,QAAM,EAAC,SAAS,qBAAqB,SAAS,sBAAA,IAC5C,SAAA;AACF,QAAM,iBAAiB,oBACnB;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,aAAa,QAAA,CAAS;AAAA,EAAA,EAC7B,KAAK,WAAW,qBAAqB,IACtC,2BAA2B;AAEhC,QAAM,EAAC,SAAS,aAAa,SAAS,cAAA,IAAiB,SAAA;AACvD,MAAI,aAAa,GAAG;AAClB,eAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;AAAA,EACpE,OAAO;AACL,kBAAA;AAAA,EACF;AAIA,QAAM;AAEN,MAAI,qBAAqB,WAAW,WAAW;AAE7C,UAAM,EAAC,SAAS,aAAa,QAAA,IAAW,SAAA;AACxC,UAAM,OAAwB;AAC9B,eAAW,gBAAgB,cAAc,MAAM,IAAI,EAAE;AAAA;AAAA;AAAA;AAAA,MAInD;AAAA,MACA,MAAM;AACJ,kBAAU;AAAA,UACR,0BAA0B,MAAM;AAAA,UAChC;AAAA,UACA;AAAA,QAAA;AAEF,gBAAA;AAAA,MACF;AAAA,IAAA;AAEF,UAAM;AAAA,EACR;AAIA,QAAM;AAEN,QAAM,UAAoB,CAAA;AAC1B,MAAI,YAAY;AACd,UAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;AAC/D,UAAM,EAAC,SAAS,cAAc,QAAA,IAAW,SAAA;AACzC,UAAM,aAAa;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,EACA,KAAK,WAAW,MAAM;AACtB,kBAAY,IAAI,UAAU;AAC1B,cAAA;AAAA,IACF,CAAC;AACD,UAAM;AAEN,UAAM,WAAW,mBAAmB,IAAI,UAAU;AAClD,aAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,cAAQ,KAAK,WAAW,YAAY,eAAe,IAAI,GAAG,IAAI,CAAC;AAAA,IACjE;AACA,YAAQ,QAAQ,CAAA,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;AAAA,EACzE;AACA,MAAI;AAKJ,KAAG,OAAO,qCAAqC;AAC/C,QAAM,aAAa;AAAA,IACjB,MAAM,GAAG,OAAO,qBAAqB,UAAU,eAAe,KAAK,IAAI,CAAC,EAAE;AAAA,IAC1E;AAAA,EAAA;AAEF,QAAM,UAAU,gBAAA;AAChB,gBAAc,UAAU;AACxB,KAAG,OAAO,sBAAsB,KAAK,QAAQ,OAAO,MAAM;AAE1D,SAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAA,CAAK,CAAC;AAEpC,MAAI;AACF,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA,IAAI;AAAA,QACF;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAAA,IACF;AAAA,EAEJ,SAAS,KAAK;AACZ,cAAU,gBAAgB,KAAK,YAAY;AAAA,EAC7C;AAEA,QAAM,UAAU,KAAA;AAClB;AAEA,IAAI,CAAC,qBAAqB;AACxB,OAAK,UAAU,MAAM,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CAAC;AACjE;"}
1	+ {"version":3,"file":"main.js","sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import {resolver} from '@rocicorp/resolver';\nimport path from 'node:path';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n restoreReplica,\n startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(createLogContext(config, {worker: 'dispatcher'}, false));\n const lc = createLogContext(config, {worker: 'dispatcher'}, true);\n initEventSink(lc, config);\n\n const processes = new ProcessManager(lc, parent);\n\n const {numSyncWorkers: numSyncers} = config;\n if (config.upstream.maxConns < numSyncers) {\n throw new Error(\n `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n if (config.cvr.maxConns < numSyncers) {\n throw new Error(\n `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n\n const internalFlags: string[] =\n numSyncers === 0\n ? []\n : [\n '--upstream-max-conns-per-worker',\n String(Math.floor(config.upstream.maxConns / numSyncers)),\n '--cvr-max-conns-per-worker',\n String(Math.floor(config.cvr.maxConns / numSyncers)),\n ];\n\n function loadWorker(\n moduleUrl: URL,\n type: WorkerType,\n id?: string \| number,\n ...args: string[]\n ): Worker {\n const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n return processes.addWorker(worker, type, name);\n }\n\n const {\n taskID,\n changeStreamer: {mode: changeStreamerMode, uri: changeStreamerURI},\n litestream,\n } = config;\n const runChangeStreamer =\n changeStreamerMode === 'dedicated' && changeStreamerURI === undefined;\n\n let restoreStart = new Date();\n if (litestream.backupURL \|\| (litestream.executable && !runChangeStreamer)) {\n try {\n restoreStart = await restoreReplica(lc, config);\n } catch (e) {\n if (runChangeStreamer) {\n // If the restore failed, e.g. due to a corrupt backup, the\n // replication-manager recovers by re-syncing.\n lc.error?.('error restoring backup. resyncing the replica.', e);\n } else {\n // View-syncers, on the other hand, have no option other than to retry\n // until a valid backup has been published. This is achieved by\n // shutting down and letting the container runner retry with its\n // configured policy.\n throw e;\n }\n }\n }\n\n const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n resolver();\n const changeStreamer = runChangeStreamer\n ? loadWorker(\n CHANGE_STREAMER_URL,\n 'supporting',\n undefined,\n String(restoreStart.getTime()),\n ).once('message', changeStreamerStarted)\n : (changeStreamerStarted() ?? undefined);\n\n const {promise: reaperReady, resolve: reaperStarted} = resolver();\n if (numSyncers > 0) {\n loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n } else {\n reaperStarted();\n }\n\n // Wait for the change-streamer to be ready to guarantee that a replica\n // file is present.\n await changeStreamerReady;\n\n if (runChangeStreamer && litestream.backupURL) {\n // Start a backup replicator and corresponding litestream backup process.\n const {promise: backupReady, resolve} = resolver();\n const mode: ReplicaFileMode = 'backup';\n loadWorker(REPLICATOR_URL, 'supporting', mode, mode).once(\n // Wait for the Replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the lock when the\n // replicator first prepares the db file.\n 'message',\n () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, config),\n 'supporting',\n 'litestream',\n );\n resolve();\n },\n );\n await backupReady;\n }\n\n // Before starting the view-syncers, ensure that the reaper has started\n // up, indicating that any CVR db migrations have been performed.\n await reaperReady;\n\n const syncers: Worker[] = [];\n if (numSyncers) {\n const mode: ReplicaFileMode =\n runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n const {promise: replicaReady, resolve} = resolver();\n const replicator = loadWorker(\n REPLICATOR_URL,\n 'supporting',\n mode,\n mode,\n ).once('message', () => {\n subscribeTo(lc, replicator);\n resolve();\n });\n await replicaReady;\n\n const notifier = createNotifierFrom(lc, replicator);\n for (let i = 0; i < numSyncers; i++) {\n syncers.push(loadWorker(SYNCER_URL, 'user-facing', i + 1, mode));\n }\n syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n }\n let mutator: Worker \| undefined;\n if (clientConnectionBifurcated) {\n mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n }\n\n lc.info?.('waiting for workers to be ready ...');\n const logWaiting = setInterval(\n () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n 10_000,\n );\n await processes.allWorkersReady();\n clearInterval(logWaiting);\n lc.info?.(`all workers ready (${Date.now() - startMs} ms)`);\n\n parent.send(['ready', {ready: true}]);\n\n try {\n await runUntilKilled(\n lc,\n parent,\n new WorkerDispatcher(\n lc,\n taskID,\n parent,\n syncers,\n mutator,\n changeStreamer,\n ),\n );\n } catch (err) {\n processes.logErrorAndExit(err, 'dispatcher');\n }\n\n await processes.done();\n}\n\nif (!singleProcessMode()) {\n void exitAfter(() => runWorker(must(parentWorker), process.env));\n}\n"],"names":[],"mappings":";;;;;;;;;;;;;AAwCA,eAA8B,UAC5B,QACA,KACe;AACf,QAAM,UAAU,KAAK,IAAA;AACrB,QAAM,SAAS,wBAAwB,EAAC,KAAI;AAE5C,gBAAc,iBAAiB,QAAQ,EAAC,QAAQ,aAAA,GAAe,KAAK,CAAC;AACrE,QAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,aAAA,GAAe,IAAI;AAChE,gBAAc,IAAI,MAAM;AAExB,QAAM,YAAY,IAAI,eAAe,IAAI,MAAM;AAE/C,QAAM,EAAC,gBAAgB,WAAA,IAAc;AACrC,MAAI,OAAO,SAAS,WAAW,YAAY;AACzC,UAAM,IAAI;AAAA,MACR,sCAAsC,OAAO,SAAS,QAAQ,SAAS,UAAU;AAAA,IAAA;AAAA,EAGrF;AACA,MAAI,OAAO,IAAI,WAAW,YAAY;AACpC,UAAM,IAAI;AAAA,MACR,iCAAiC,OAAO,IAAI,QAAQ,SAAS,UAAU;AAAA,IAAA;AAAA,EAG3E;AAEA,QAAM,gBACJ,eAAe,IACX,KACA;AAAA,IACE;AAAA,IACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,UAAU,CAAC;AAAA,IACxD;AAAA,IACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,UAAU,CAAC;AAAA,EAAA;AAG3D,WAAS,WACP,WACA,MACA,OACG,MACK;AACR,UAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,aAAa;AACpE,UAAM,OAAO,KAAK,SAAS,UAAU,QAAQ,KAAK,KAAK,KAAK,EAAE,MAAM;AACpE,WAAO,UAAU,UAAU,QAAQ,MAAM,IAAI;AAAA,EAC/C;AAEA,QAAM;AAAA,IACJ;AAAA,IACA,gBAAgB,EAAC,MAAM,oBAAoB,KAAK,kBAAA;AAAA,IAChD;AAAA,EAAA,IACE;AACJ,QAAM,oBACJ,uBAAuB,eAAe,sBAAsB;AAE9D,MAAI,mCAAmB,KAAA;AACvB,MAAI,WAAW,aAAc,WAAW,cAAc,CAAC,mBAAoB;AACzE,QAAI;AACF,qBAAe,MAAM,eAAe,IAAI,MAAM;AAAA,IAChD,SAAS,GAAG;AACV,UAAI,mBAAmB;AAGrB,WAAG,QAAQ,kDAAkD,CAAC;AAAA,MAChE,OAAO;AAKL,cAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAEA,QAAM,EAAC,SAAS,qBAAqB,SAAS,sBAAA,IAC5C,SAAA;AACF,QAAM,iBAAiB,oBACnB;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,aAAa,QAAA,CAAS;AAAA,EAAA,EAC7B,KAAK,WAAW,qBAAqB,IACtC,2BAA2B;AAEhC,QAAM,EAAC,SAAS,aAAa,SAAS,cAAA,IAAiB,SAAA;AACvD,MAAI,aAAa,GAAG;AAClB,eAAW,YAAY,YAAY,EAAE,KAAK,WAAW,aAAa;AAAA,EACpE,OAAO;AACL,kBAAA;AAAA,EACF;AAIA,QAAM;AAEN,MAAI,qBAAqB,WAAW,WAAW;AAE7C,UAAM,EAAC,SAAS,aAAa,QAAA,IAAW,SAAA;AACxC,UAAM,OAAwB;AAC9B,eAAW,gBAAgB,cAAc,MAAM,IAAI,EAAE;AAAA;AAAA;AAAA;AAAA,MAInD;AAAA,MACA,MAAM;AACJ,kBAAU;AAAA,UACR,0BAA0B,IAAI,MAAM;AAAA,UACpC;AAAA,UACA;AAAA,QAAA;AAEF,gBAAA;AAAA,MACF;AAAA,IAAA;AAEF,UAAM;AAAA,EACR;AAIA,QAAM;AAEN,QAAM,UAAoB,CAAA;AAC1B,MAAI,YAAY;AACd,UAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;AAC/D,UAAM,EAAC,SAAS,cAAc,QAAA,IAAW,SAAA;AACzC,UAAM,aAAa;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA,EACA,KAAK,WAAW,MAAM;AACtB,kBAAY,IAAI,UAAU;AAC1B,cAAA;AAAA,IACF,CAAC;AACD,UAAM;AAEN,UAAM,WAAW,mBAAmB,IAAI,UAAU;AAClD,aAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,cAAQ,KAAK,WAAW,YAAY,eAAe,IAAI,GAAG,IAAI,CAAC;AAAA,IACjE;AACA,YAAQ,QAAQ,CAAA,WAAU,wBAAwB,IAAI,QAAQ,QAAQ,CAAC;AAAA,EACzE;AACA,MAAI;AAKJ,KAAG,OAAO,qCAAqC;AAC/C,QAAM,aAAa;AAAA,IACjB,MAAM,GAAG,OAAO,qBAAqB,UAAU,eAAe,KAAK,IAAI,CAAC,EAAE;AAAA,IAC1E;AAAA,EAAA;AAEF,QAAM,UAAU,gBAAA;AAChB,gBAAc,UAAU;AACxB,KAAG,OAAO,sBAAsB,KAAK,QAAQ,OAAO,MAAM;AAE1D,SAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAA,CAAK,CAAC;AAEpC,MAAI;AACF,UAAM;AAAA,MACJ;AAAA,MACA;AAAA,MACA,IAAI;AAAA,QACF;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MAAA;AAAA,IACF;AAAA,EAEJ,SAAS,KAAK;AACZ,cAAU,gBAAgB,KAAK,YAAY;AAAA,EAC7C;AAEA,QAAM,UAAU,KAAA;AAClB;AAEA,IAAI,CAAC,qBAAqB;AACxB,OAAK,UAAU,MAAM,UAAU,KAAK,YAAY,GAAG,QAAQ,GAAG,CAAC;AACjE;"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAgBjD,OAAO,KAAK,EAEV,kBAAkB,EAEnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,kDAAkD,CAAC;AAE1D,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AAYtD,OAAO,EAAC,KAAK,kBAAkB,EAAC,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAoBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,GAC9B,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAoC7E;~~AA6PD~~,qBAAa,KAAK;;gBAIJ,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,SAAS;IAiBT,GAAG,CAAC,SAAS,EAAE,WAAW;CAa3B;AAycD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
1	+ {"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAgBjD,OAAO,KAAK,EAEV,kBAAkB,EAEnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,kDAAkD,CAAC;AAE1D,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AAYtD,OAAO,EAAC,KAAK,kBAAkB,EAAC,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAoBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,GAC9B,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAoC7E;AAgSD,qBAAa,KAAK;;gBAIJ,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,SAAS;IAiBT,GAAG,CAAC,SAAS,EAAE,WAAW;CAa3B;AAycD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js CHANGED Viewed

@@ -197,17 +197,44 @@ class PostgresChangeSource {
     const slotExpression = replicationSlotExpression(this.#shard);
     const legacySlotName = legacyReplicationSlot(this.#shard);
     const result = await sql`
-    SELECT slot_name as slot, pg_terminate_backend(active_pid), active_pid as pid
+    SELECT slot_name as slot, pg_terminate_backend(active_pid) as terminated, active_pid as pid
       FROM pg_replication_slots
       WHERE (slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName})
             AND slot_name <= ${slotToKeep}`;
+    this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);
     if (result.length === 0) {
+      const shardSlots = await sql`
+      SELECT slot_name as slot, active, active_pid as pid
+        FROM pg_replication_slots
+        WHERE slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName}
+        ORDER BY slot_name`;
+      this.#lc.warn?.(
+        `slot ${slotToKeep} not found while cleaning subscribers; shard slots at time of failure: ${JSON.stringify(
+          shardSlots
+        )}`
+      );
       throw new AbortError(
         `replication slot ${slotToKeep} is missing. A different replication-manager should now be running on a new replication slot.`
       );
     }
     const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;
-    await sql`DELETE FROM ${sql(replicasTable)} WHERE slot != ${slotToKeep}`;
+    const replicasBefore = await sql`
+      SELECT slot, version FROM ${sql(replicasTable)} ORDER BY slot`;
+    this.#lc.info?.(
+      `replicas before cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(
+        replicasBefore
+      )}`
+    );
+    await sql`
+      DELETE FROM ${sql(replicasTable)}
+        WHERE slot < ${slotToKeep}`;
+    const replicasAfter = await sql`
+      SELECT slot, version FROM ${sql(replicasTable)} ORDER BY slot`;
+    this.#lc.info?.(
+      `replicas after cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(
+        replicasAfter
+      )}`
+    );
     const pids = result.filter(({ pid }) => pid !== null).map(({ pid }) => pid);
     if (pids.length) {
       this.#lc.info?.(`signaled subscriber ${pids} to shut down`);

package/out/zero-cache/src/services/change-source/pg/change-source.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-source.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"sourcesContent":["import {\n PG_ADMIN_SHUTDOWN,\n PG_OBJECT_IN_USE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {\n equals,\n intersection,\n symmetricDifferences,\n} from '../../../../../shared/src/set-utils.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {mapPostgresToLiteColumn} from '../../../db/pg-to-lite.ts';\nimport type {\n ColumnSpec,\n PublishedTableSpec,\n TableSpec,\n} from '../../../db/specs.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {\n oneAfter,\n versionFromLexi,\n versionToLexi,\n type LexiVersion,\n} from '../../../types/lexi-version.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {\n upstreamSchema,\n type ShardConfig,\n type ShardID,\n} from '../../../types/shards.ts';\nimport type {Sink} from '../../../types/streams.ts';\nimport {Subscription, type PendingResult} from '../../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../../change-streamer/change-streamer-service.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n getSubscriptionState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {JSONObject} from '../protocol/current.ts';\nimport type {\n DataChange,\n Identifier,\n MessageRelation,\n} from '../protocol/current/data.ts';\nimport type {\n ChangeStreamData,\n ChangeStreamMessage,\n Data,\n} from '../protocol/current/downstream.ts';\nimport {type InitialSyncOptions} from './initial-sync.ts';\nimport type {\n Message,\n MessageMessage,\n MessageRelation as PostgresRelation,\n} from './logical-replication/pgoutput.types.ts';\nimport {subscribe} from './logical-replication/stream.ts';\nimport {fromBigInt, toLexiVersion, type LSN} from './lsn.ts';\nimport {replicationEventSchema, type DdlUpdateEvent} from './schema/ddl.ts';\nimport {updateShardSchema} from './schema/init.ts';\nimport {getPublicationInfo, type PublishedSchema} from './schema/published.ts';\nimport {\n dropShard,\n getInternalShardConfig,\n getReplicaAtVersion,\n internalPublicationPrefix,\n legacyReplicationSlot,\n replicaIdentitiesForTablesWithoutPrimaryKeys,\n replicationSlotExpression,\n type InternalShardConfig,\n type Replica,\n} from './schema/shard.ts';\nimport {validate} from './schema/validation.ts';\nimport {initSyncSchema} from './sync-schema.ts';\n\n/*\n Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n /\nexport async function initializePostgresChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n syncOptions: InitialSyncOptions,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initSyncSchema(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n shard,\n replicaDbFile,\n upstreamURI,\n syncOptions,\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n // Check that upstream is properly setup, and throw an AutoReset to re-run\n // initial sync if not.\n const db = pgClient(lc, upstreamURI);\n try {\n const upstreamReplica = await checkAndUpdateUpstream(\n lc,\n db,\n shard,\n subscriptionState,\n );\n\n const changeSource = new PostgresChangeSource(\n lc,\n upstreamURI,\n shard,\n upstreamReplica,\n );\n\n return {subscriptionState, changeSource};\n } finally {\n await db.end();\n }\n}\n\nasync function checkAndUpdateUpstream(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n {replicaVersion, publications: subscribed}: SubscriptionState,\n) {\n // Perform any shard schema updates\n await updateShardSchema(lc, sql, shard, replicaVersion);\n\n const upstreamReplica = await getReplicaAtVersion(\n lc,\n sql,\n shard,\n replicaVersion,\n );\n if (!upstreamReplica) {\n throw new AutoResetSignal(\n `No replication slot for replica at version ${replicaVersion}`,\n );\n }\n\n // Verify that the publications match what is being replicated.\n const requested = [...shard.publications].sort();\n const replicated = upstreamReplica.publications\n .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n .sort();\n if (!deepEqual(requested, replicated)) {\n lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n throw new AutoResetSignal(\n `Requested publications [${requested}] do not match configured ` +\n `publications: [${replicated}]`,\n );\n }\n\n // Sanity check: The subscription state on the replica should have the\n // same publications. This should be guaranteed by the equivalence of the\n // replicaVersion, but it doesn't hurt to verify.\n if (!deepEqual(upstreamReplica.publications, subscribed)) {\n throw new AutoResetSignal(\n `Upstream publications [${upstreamReplica.publications}] do not ` +\n `match subscribed publications [${subscribed}]`,\n );\n }\n\n // Verify that the publications exist.\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n `.values();\n if (exists.length !== subscribed.length) {\n throw new AutoResetSignal(\n `Upstream publications [${exists.flat()}] do not contain ` +\n `all subscribed publications [${subscribed}]`,\n );\n }\n\n const {slot} = upstreamReplica;\n const result = await sql<\n {restartLSN: LSN \| null; walStatus: string \| null}[]\n > /sql/ `\n SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n WHERE slot_name = ${slot}`;\n if (result.length === 0) {\n throw new AutoResetSignal(`replication slot ${slot} is missing`);\n }\n const [{restartLSN, walStatus}] = result;\n if (restartLSN === null \|\| walStatus === 'lost') {\n throw new AutoResetSignal(\n `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n );\n }\n return upstreamReplica;\n}\n\n/\n Postgres implementation of a {@link ChangeSource} backed by a logical\n * replication stream.\n /\nclass PostgresChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replica: Replica;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replica: Replica,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replica = replica;\n }\n\n async startStream(clientWatermark: string): Promise<ChangeStream> {\n const db = pgClient(this.#lc, this.#upstreamUri, {}, 'json-as-string');\n const {slot} = this.#replica;\n\n let cleanup = promiseVoid;\n try {\n ({cleanup} = await this.#stopExistingReplicationSlotSubscribers(\n db,\n slot,\n ));\n const config = await getInternalShardConfig(db, this.#shard);\n this.#lc.info?.(`starting replication stream@${slot}`);\n return await this.#startStream(db, slot, clientWatermark, config);\n } finally {\n void cleanup.then(() => db.end());\n }\n }\n\n async #startStream(\n db: PostgresDB,\n slot: string,\n clientWatermark: string,\n shardConfig: InternalShardConfig,\n ): Promise<ChangeStream> {\n const clientStart = oneAfter(clientWatermark);\n const {messages, acks} = await subscribe(\n this.#lc,\n db,\n slot,\n [...shardConfig.publications],\n versionFromLexi(clientStart),\n );\n\n const changes = Subscription.create<ChangeStreamMessage>({\n cleanup: () => messages.cancel(),\n });\n const acker = new Acker(acks);\n\n const changeMaker = new ChangeMaker(\n this.#lc,\n this.#shard,\n shardConfig,\n this.#replica.initialSchema,\n this.#upstreamUri,\n );\n\n void (async function () {\n try {\n for await (const [lsn, msg] of messages) {\n if (msg.tag === 'keepalive') {\n changes.push(['status', msg, {watermark: versionToLexi(lsn)}]);\n continue;\n }\n let last: PendingResult \| undefined;\n for (const change of await changeMaker.makeChanges(lsn, msg)) {\n last = changes.push(change);\n }\n await last?.result; // Allow the change-streamer to push back.\n }\n } catch (e) {\n changes.fail(translateError(e));\n }\n })();\n\n this.#lc.info?.(\n `started replication stream@${slot} from ${clientWatermark} (replicaVersion: ${\n this.#replica.version\n })`,\n );\n\n return {\n changes,\n acks: {push: status => acker.ack(status[2].watermark)},\n };\n }\n\n /\n Stops replication slots associated with this shard, and returns\n * a `cleanup` task that drops any slot other than the specified\n * `slotToKeep`.\n \n Note that replication slots created after `slotToKeep` (as indicated by\n * the timestamp suffix) are preserved, as those are newly syncing replicas\n * that will soon take over the slot.\n /\n async #stopExistingReplicationSlotSubscribers(\n sql: PostgresDB,\n slotToKeep: string,\n ): Promise<{cleanup: Promise<void>}> {\n const slotExpression = replicationSlotExpression(this.#shard);\n const legacySlotName = legacyReplicationSlot(this.#shard);\n\n // Note: `slot_name <= slotToKeep` uses a string compare of the millisecond\n // timestamp, which works until it exceeds 13 digits (sometime in 2286).\n const result = await sql<{slot: string; pid: string \| null}[]>`\n SELECT slot_name as slot, pg_terminate_backend(active_pid), active_pid as pid\n FROM pg_replication_slots \n WHERE (slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName})\n AND slot_name <= ${slotToKeep}`;\n if (result.length === 0) {\n throw new AbortError(\n `replication slot ${slotToKeep} is missing. A different ` +\n `replication-manager should now be running on a new ` +\n `replication slot.`,\n );\n }\n // Clean up the replicas table.\n const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;\n await sql`DELETE FROM ${sql(replicasTable)} WHERE slot != ${slotToKeep}`;\n\n const pids = result.filter(({pid}) => pid !== null).map(({pid}) => pid);\n if (pids.length) {\n this.#lc.info?.(`signaled subscriber ${pids} to shut down`);\n }\n const otherSlots = result\n .filter(({slot}) => slot !== slotToKeep)\n .map(({slot}) => slot);\n return {\n cleanup: otherSlots.length\n ? this.#dropReplicationSlots(sql, otherSlots)\n : promiseVoid,\n };\n }\n\n async #dropReplicationSlots(sql: PostgresDB, slots: string[]) {\n this.#lc.info?.(`dropping other replication slot(s) ${slots}`);\n for (let i = 0; i < 5; i++) {\n try {\n await sql`\n SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name IN ${sql(slots)}\n `;\n this.#lc.info?.(`successfully dropped ${slots}`);\n return;\n } catch (e) {\n // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n if (\n e instanceof postgres.PostgresError &&\n e.code === PG_OBJECT_IN_USE\n ) {\n // The freeing up of the replication slot is not transactional;\n // sometimes it takes time for Postgres to consider the slot\n // inactive.\n this.#lc.debug?.(`attempt ${i + 1}: ${String(e)}`, e);\n } else {\n this.#lc.warn?.(`error dropping ${slots}`, e);\n }\n await sleep(1000);\n }\n }\n this.#lc.warn?.(`maximum attempts exceeded dropping ${slots}`);\n }\n}\n\n// Exported for testing.\nexport class Acker {\n #acks: Sink<bigint>;\n #keepaliveTimer: NodeJS.Timeout \| undefined;\n\n constructor(acks: Sink<bigint>) {\n this.#acks = acks;\n }\n\n keepalive() {\n // Sets a timeout to send a standby status update in response to\n // a primary keepalive message.\n //\n // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n //\n // A primary keepalive message is streamed to the change-streamer as a\n // 'status' message, which in turn responds with an ack. However, in the\n // event that the change-streamer is backed up processing preceding\n // changes, this timeout will fire to send a status update that does not\n // change the confirmed flush position. This timeout must be shorter than\n // the `wal_sender_timeout`, which defaults to 60 seconds.\n //\n // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n this.#keepaliveTimer ??= setTimeout(() => this.#sendAck(), 1000);\n }\n\n ack(watermark: LexiVersion) {\n this.#sendAck(watermark);\n }\n\n #sendAck(watermark?: LexiVersion) {\n clearTimeout(this.#keepaliveTimer);\n this.#keepaliveTimer = undefined;\n\n // Note: Sending '0/0' means \"keep alive but do not update confirmed_flush_lsn\"\n // https://github.com/postgres/postgres/blob/3edc67d337c2e498dad1cd200e460f7c63e512e6/src/backend/replication/walsender.c#L2457\n const lsn = watermark ? versionFromLexi(watermark) : 0n;\n this.#acks.push(lsn);\n }\n}\n\ntype ReplicationError = {\n lsn: bigint;\n msg: Message;\n err: unknown;\n lastLogTime: number;\n};\n\nconst SET_REPLICA_IDENTITY_DELAY_MS = 500;\n\nclass ChangeMaker {\n readonly #lc: LogContext;\n readonly #shardPrefix: string;\n readonly #shardConfig: InternalShardConfig;\n readonly #initialSchema: PublishedSchema;\n readonly #upstreamDB: PostgresDB;\n\n #replicaIdentityTimer: NodeJS.Timeout \| undefined;\n #error: ReplicationError \| undefined;\n\n constructor(\n lc: LogContext,\n {appID, shardNum}: ShardID,\n shardConfig: InternalShardConfig,\n initialSchema: PublishedSchema,\n upstreamURI: string,\n ) {\n this.#lc = lc;\n // Note: This matches the prefix used in pg_logical_emit_message() in pg/schema/ddl.ts.\n this.#shardPrefix = `${appID}/${shardNum}`;\n this.#shardConfig = shardConfig;\n this.#initialSchema = initialSchema;\n this.#upstreamDB = pgClient(lc, upstreamURI, {\n ['idle_timeout']: 10, // only used occasionally\n connection: {['application_name']: 'zero-schema-change-detector'},\n });\n }\n\n async makeChanges(lsn: bigint, msg: Message): Promise<ChangeStreamMessage[]> {\n if (this.#error) {\n this.#logError(this.#error);\n return [];\n }\n try {\n return await this.#makeChanges(msg);\n } catch (err) {\n this.#error = {lsn, msg, err, lastLogTime: 0};\n this.#logError(this.#error);\n\n const message = `Unable to continue replication from LSN ${fromBigInt(lsn)}`;\n const errorDetails: JSONObject = {error: message};\n if (err instanceof UnsupportedSchemaChangeError) {\n errorDetails.reason = err.description;\n errorDetails.context = err.ddlUpdate.context;\n } else {\n errorDetails.reason = String(err);\n }\n\n // Rollback the current transaction to avoid dangling transactions in\n // downstream processors (i.e. changeLog, replicator).\n return [\n ['rollback', {tag: 'rollback'}],\n ['control', {tag: 'reset-required', message, errorDetails}],\n ];\n }\n }\n\n #logError(error: ReplicationError) {\n const {lsn, msg, err, lastLogTime} = error;\n const now = Date.now();\n\n // Output an error to logs as replication messages continue to be dropped,\n // at most once a minute.\n if (now - lastLogTime > 60_000) {\n this.#lc.error?.(\n `Unable to continue replication from LSN ${fromBigInt(lsn)}: ${String(\n err,\n )}`,\n err instanceof UnsupportedSchemaChangeError\n ? err.ddlUpdate.context\n : // 'content' can be a large byte Buffer. Exclude it from logging output.\n {...msg, content: undefined},\n );\n error.lastLogTime = now;\n }\n }\n\n // oxlint-disable-next-line require-await\n async #makeChanges(msg: Message): Promise<ChangeStreamData[]> {\n switch (msg.tag) {\n case 'begin':\n return [\n [\n 'begin',\n {...msg, json: 's'},\n {commitWatermark: toLexiVersion(must(msg.commitLsn))},\n ],\n ];\n\n case 'delete': {\n if (!(msg.key ?? msg.old)) {\n throw new Error(\n `Invalid DELETE msg (missing key): ${stringify(msg)}`,\n );\n }\n return [\n [\n 'data',\n {\n ...msg,\n relation: withoutColumns(msg.relation),\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-DELETE\n key: must(msg.old ?? msg.key),\n },\n ],\n ];\n }\n\n case 'update': {\n return [\n [\n 'data',\n {\n ...msg,\n relation: withoutColumns(msg.relation),\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-UPDATE\n key: msg.old ?? msg.key,\n },\n ],\n ];\n }\n\n case 'insert':\n return [['data', {...msg, relation: withoutColumns(msg.relation)}]];\n case 'truncate':\n return [\n ['data', {...msg, relations: msg.relations.map(withoutColumns)}],\n ];\n\n case 'message':\n if (msg.prefix !== this.#shardPrefix) {\n this.#lc.debug?.('ignoring message for different shard', msg.prefix);\n return [];\n }\n return this.#handleCustomMessage(msg);\n\n case 'commit':\n return [\n ['commit', msg, {watermark: toLexiVersion(must(msg.commitLsn))}],\n ];\n\n case 'relation':\n return this.#handleRelation(msg);\n case 'type':\n return []; // Nothing need be done for custom types.\n case 'origin':\n // No need to detect replication loops since we are not a\n // PG replication source.\n return [];\n default:\n msg satisfies never;\n throw new Error(`Unexpected message type ${stringify(msg)}`);\n }\n }\n\n #preSchema: PublishedSchema \| undefined;\n\n #handleCustomMessage(msg: MessageMessage) {\n const event = this.#parseReplicationEvent(msg.content);\n // Cancel manual schema adjustment timeouts when an upstream schema change\n // is about to happen, so as to avoid interfering / redundant work.\n clearTimeout(this.#replicaIdentityTimer);\n\n if (event.type === 'ddlStart') {\n // Store the schema in order to diff it with a potential ddlUpdate.\n this.#preSchema = event.schema;\n return [];\n }\n // ddlUpdate\n const changes = this.#makeSchemaChanges(\n must(this.#preSchema, `ddlUpdate received without a ddlStart`),\n event,\n ).map(change => ['data', change] satisfies Data);\n\n this.#lc\n .withContext('query', event.context.query)\n .info?.(`${changes.length} schema change(s)`, changes);\n\n const replicaIdentities = replicaIdentitiesForTablesWithoutPrimaryKeys(\n event.schema,\n );\n if (replicaIdentities) {\n this.#replicaIdentityTimer = setTimeout(async () => {\n try {\n await replicaIdentities.apply(this.#lc, this.#upstreamDB);\n } catch (err) {\n this.#lc.warn?.(`error setting replica identities`, err);\n }\n }, SET_REPLICA_IDENTITY_DELAY_MS);\n }\n\n return changes;\n }\n\n /\n A note on operation order:\n \n Postgres will drop related indexes when columns are dropped,\n * but SQLite will error instead (https://sqlite.org/forum/forumpost/2e62dba69f?t=c&hist).\n * The current workaround is to drop indexes first.\n \n Also note that although it should not be possible to both rename and\n * add/drop tables/columns in a single statement, the operations are\n * ordered to handle that possibility, by always dropping old entities,\n * then modifying kept entities, and then adding new entities.\n \n Thus, the order of replicating DDL updates is:\n * - drop indexes\n * - drop tables\n * - alter tables\n * - drop columns\n * - alter columns\n * - add columns\n * - create tables\n * - create indexes\n \n In the future the replication logic should be improved to handle this\n * behavior in SQLite by dropping dependent indexes manually before dropping\n * columns. This, for example, would be needed to properly support changing\n * the type of a column that's indexed.\n /\n #makeSchemaChanges(\n preSchema: PublishedSchema,\n update: DdlUpdateEvent,\n ): DataChange[] {\n try {\n const [prevTbl, prevIdx] = specsByID(preSchema);\n const [nextTbl, nextIdx] = specsByID(update.schema);\n const changes: DataChange[] = [];\n\n // Validate the new table schemas\n for (const table of nextTbl.values()) {\n validate(this.#lc, table, update.schema.indexes);\n }\n\n const [droppedIdx, createdIdx] = symmetricDifferences(prevIdx, nextIdx);\n for (const id of droppedIdx) {\n const {schema, name} = must(prevIdx.get(id));\n changes.push({tag: 'drop-index', id: {schema, name}});\n }\n\n // DROP\n const [droppedTbl, createdTbl] = symmetricDifferences(prevTbl, nextTbl);\n for (const id of droppedTbl) {\n const {schema, name} = must(prevTbl.get(id));\n changes.push({tag: 'drop-table', id: {schema, name}});\n }\n // ALTER\n const tables = intersection(prevTbl, nextTbl);\n for (const id of tables) {\n changes.push(\n ...this.#getTableChanges(\n must(prevTbl.get(id)),\n must(nextTbl.get(id)),\n ),\n );\n }\n // CREATE\n for (const id of createdTbl) {\n const spec = must(nextTbl.get(id));\n changes.push({tag: 'create-table', spec});\n }\n\n // Add indexes last since they may reference tables / columns that need\n // to be created first.\n for (const id of createdIdx) {\n const spec = must(nextIdx.get(id));\n changes.push({tag: 'create-index', spec});\n }\n return changes;\n } catch (e) {\n throw new UnsupportedSchemaChangeError(String(e), update, {cause: e});\n }\n }\n\n #getTableChanges(oldTable: TableSpec, newTable: TableSpec): DataChange[] {\n const changes: DataChange[] = [];\n if (\n oldTable.schema !== newTable.schema \|\|\n oldTable.name !== newTable.name\n ) {\n changes.push({\n tag: 'rename-table',\n old: {schema: oldTable.schema, name: oldTable.name},\n new: {schema: newTable.schema, name: newTable.name},\n });\n }\n const table = {schema: newTable.schema, name: newTable.name};\n const oldColumns = columnsByID(oldTable.columns);\n const newColumns = columnsByID(newTable.columns);\n\n // DROP\n const [dropped, added] = symmetricDifferences(oldColumns, newColumns);\n for (const id of dropped) {\n const {name: column} = must(oldColumns.get(id));\n changes.push({tag: 'drop-column', table, column});\n }\n\n // ALTER\n const both = intersection(oldColumns, newColumns);\n for (const id of both) {\n const {name: oldName, ...oldSpec} = must(oldColumns.get(id));\n const {name: newName, ...newSpec} = must(newColumns.get(id));\n // The three things that we care about are:\n // 1. name\n // 2. type\n // 3. not-null\n if (\n oldName !== newName \|\|\n oldSpec.dataType !== newSpec.dataType \|\|\n oldSpec.notNull !== newSpec.notNull\n ) {\n changes.push({\n tag: 'update-column',\n table,\n old: {name: oldName, spec: oldSpec},\n new: {name: newName, spec: newSpec},\n });\n }\n }\n\n // ADD\n for (const id of added) {\n const {name, ...spec} = must(newColumns.get(id));\n const column = {name, spec};\n // Validate that the ChangeProcessor will accept the column change.\n mapPostgresToLiteColumn(table.name, column);\n changes.push({tag: 'add-column', table, column});\n }\n return changes;\n }\n\n #parseReplicationEvent(content: Uint8Array) {\n const str =\n content instanceof Buffer\n ? content.toString('utf-8')\n : new TextDecoder().decode(content);\n const json = JSON.parse(str);\n return v.parse(json, replicationEventSchema, 'passthrough');\n }\n\n /\n If `ddlDetection === true`, relation messages are irrelevant,\n * as schema changes are detected by event triggers that\n * emit custom messages.\n \n For degraded-mode replication (`ddlDetection === false`):\n * 1. query the current published schemas on upstream\n * 2. compare that with the InternalShardConfig.initialSchema\n * 3. compare that with the incoming MessageRelation\n * 4. On any discrepancy, throw an UnsupportedSchemaChangeError\n * to halt replication.\n \n Note that schemas queried in step [1] will be post-transaction\n * schemas, which are not necessarily suitable for actually processing\n * the statements in the transaction being replicated. In other words,\n * this mechanism cannot be used to reliably replicate schema changes.\n * However, they serve the purpose determining if schemas have changed.\n */\n async #handleRelation(rel: PostgresRelation): Promise<ChangeStreamData[]> {\n const {publications, ddlDetection} = this.#shardConfig;\n if (ddlDetection) {\n return [];\n }\n const currentSchema = await getPublicationInfo(\n this.#upstreamDB,\n publications,\n );\n const difference = getSchemaDifference(this.#initialSchema, currentSchema);\n if (difference !== null) {\n throw new MissingEventTriggerSupport(difference);\n }\n // Even if the currentSchema is equal to the initialSchema, the\n // MessageRelation itself must be checked to detect transient\n // schema changes within the transaction (e.g. adding and dropping\n // a table, or renaming a column and then renaming it back).\n const orel = this.#initialSchema.tables.find(\n t => t.oid === rel.relationOid,\n );\n if (!orel) {\n // Can happen if a table is created and then dropped in the same transaction.\n throw new MissingEventTriggerSupport(\n `relation not in initialSchema: ${stringify(rel)}`,\n );\n }\n if (relationDifferent(orel, rel)) {\n throw new MissingEventTriggerSupport(\n `relation has changed within the transaction: ${stringify(orel)} vs ${stringify(rel)}`,\n );\n }\n return [];\n }\n}\n\nfunction getSchemaDifference(\n a: PublishedSchema,\n b: PublishedSchema,\n): string \| null {\n // Note: ignore indexes since changes need not to halt replication\n if (a.tables.length !== b.tables.length) {\n return `tables created or dropped`;\n }\n for (let i = 0; i < a.tables.length; i++) {\n const at = a.tables[i];\n const bt = b.tables[i];\n const difference = getTableDifference(at, bt);\n if (difference) {\n return difference;\n }\n }\n return null;\n}\n\n// ColumnSpec comparator\nconst byColumnPos = (a: [string, ColumnSpec], b: [string, ColumnSpec]) =>\n a[1].pos < b[1].pos ? -1 : a[1].pos > b[1].pos ? 1 : 0;\n\nfunction getTableDifference(\n a: PublishedTableSpec,\n b: PublishedTableSpec,\n): string \| null {\n if (a.oid !== b.oid \|\| a.schema !== b.schema \|\| a.name !== b.name) {\n return `Table \"${a.name}\" differs from table \"${b.name}\"`;\n }\n if (!deepEqual(a.primaryKey, b.primaryKey)) {\n return `Primary key of table \"${a.name}\" has changed`;\n }\n const acols = Object.entries(a.columns).sort(byColumnPos);\n const bcols = Object.entries(b.columns).sort(byColumnPos);\n if (\n acols.length !== bcols.length \|\|\n acols.some(([aname, acol], i) => {\n const [bname, bcol] = bcols[i];\n return (\n aname !== bname \|\|\n acol.pos !== bcol.pos \|\|\n acol.typeOID !== bcol.typeOID \|\|\n acol.notNull !== bcol.notNull\n );\n })\n ) {\n return `Columns of table \"${a.name}\" have changed`;\n }\n return null;\n}\n\nexport function relationDifferent(a: PublishedTableSpec, b: PostgresRelation) {\n if (a.oid !== b.relationOid \|\| a.schema !== b.schema \|\| a.name !== b.name) {\n return true;\n }\n if (\n // The MessageRelation's `keyColumns` field contains the columns in column\n // declaration order, whereas the PublishedTableSpec's `primaryKey`\n // contains the columns in primary key (i.e. index) order. Do an\n // order-agnostic compare here since it is not possible to detect\n // key-order changes from the MessageRelation message alone.\n b.replicaIdentity === 'default' &&\n !equals(new Set(a.primaryKey), new Set(b.keyColumns))\n ) {\n return true;\n }\n const acols = Object.entries(a.columns).sort(byColumnPos);\n const bcols = b.columns;\n return (\n acols.length !== bcols.length \|\|\n acols.some(([aname, acol], i) => {\n const bcol = bcols[i];\n return aname !== bcol.name \|\| acol.typeOID !== bcol.typeOid;\n })\n );\n}\n\nfunction translateError(e: unknown): Error {\n if (!(e instanceof Error)) {\n return new Error(String(e));\n }\n if (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) {\n return new ShutdownSignal(e);\n }\n return e;\n}\nconst idString = (id: Identifier) => `${id.schema}.${id.name}`;\n\nfunction specsByID(published: PublishedSchema) {\n return [\n // It would have been nice to use a CustomKeyMap here, but we rely on set-utils\n // operations which use plain Sets.\n new Map(published.tables.map(t => [t.oid, t])),\n new Map(published.indexes.map(i => [idString(i), i])),\n ] as const;\n}\n\nfunction columnsByID(\n columns: Record<string, ColumnSpec>,\n): Map<number, ColumnSpec & {name: string}> {\n const colsByID = new Map<number, ColumnSpec & {name: string}>();\n for (const [name, spec] of Object.entries(columns)) {\n // The `pos` field is the `attnum` in `pg_attribute`, which is a stable\n // identifier for the column in this table (i.e. never reused).\n colsByID.set(spec.pos, {...spec, name});\n }\n return colsByID;\n}\n\n// Avoid sending the `columns` from the Postgres MessageRelation message.\n// They are not used downstream and the message can be large.\nfunction withoutColumns(relation: PostgresRelation): MessageRelation {\n const {columns: _, ...rest} = relation;\n return rest;\n}\n\nclass UnsupportedSchemaChangeError extends Error {\n readonly name = 'UnsupportedSchemaChangeError';\n readonly description: string;\n readonly ddlUpdate: DdlUpdateEvent;\n\n constructor(\n description: string,\n ddlUpdate: DdlUpdateEvent,\n options?: ErrorOptions,\n ) {\n super(\n `Replication halted. Resync the replica to recover: ${description}`,\n options,\n );\n this.description = description;\n this.ddlUpdate = ddlUpdate;\n }\n}\n\nclass MissingEventTriggerSupport extends Error {\n readonly name = 'MissingEventTriggerSupport';\n\n constructor(msg: string) {\n super(\n `${msg}. Schema changes cannot be reliably replicated without event trigger support.`,\n );\n }\n}\n\n// TODO(0xcadams): should this be a ProtocolError?\nclass ShutdownSignal extends AbortError {\n readonly name = 'ShutdownSignal';\n\n constructor(cause: unknown) {\n super(\n 'shutdown signal received (e.g. another zero-cache taking over the replication stream)',\n {\n cause,\n },\n );\n }\n}\n"],"names":["v.parse"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0FA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aAC6E;AAC7E,QAAM;AAAA,IACJ;AAAA,IACA,WAAW,MAAM,KAAK,IAAI,MAAM,QAAQ;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,UAAU,IAAI,SAAS,IAAI,aAAa;AAC9C,QAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;AAC3E,UAAQ,MAAA;AAIR,QAAM,KAAK,SAAS,IAAI,WAAW;AACnC,MAAI;AACF,UAAM,kBAAkB,MAAM;AAAA,MAC5B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAGF,UAAM,eAAe,IAAI;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAGF,WAAO,EAAC,mBAAmB,aAAA;AAAA,EAC7B,UAAA;AACE,UAAM,GAAG,IAAA;AAAA,EACX;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EAAC,gBAAgB,cAAc,cAC/B;AAEA,QAAM,kBAAkB,IAAI,KAAK,OAAO,cAAc;AAEtD,QAAM,kBAAkB,MAAM;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEF,MAAI,CAAC,iBAAiB;AACpB,UAAM,IAAI;AAAA,MACR,8CAA8C,cAAc;AAAA,IAAA;AAAA,EAEhE;AAGA,QAAM,YAAY,CAAC,GAAG,MAAM,YAAY,EAAE,KAAA;AAC1C,QAAM,aAAa,gBAAgB,aAChC,OAAO,CAAA,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAA;AACH,MAAI,CAAC,UAAU,WAAW,UAAU,GAAG;AACrC,OAAG,OAAO,8CAA8C,SAAS,GAAG;AACpE,UAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;AACvD,UAAM,IAAI;AAAA,MACR,2BAA2B,SAAS,4CAChB,UAAU;AAAA,IAAA;AAAA,EAElC;AAKA,MAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GAAG;AACxD,UAAM,IAAI;AAAA,MACR,0BAA0B,gBAAgB,YAAY,2CAClB,UAAU;AAAA,IAAA;AAAA,EAElD;AAGA,QAAM,SAAS,MAAM;AAAA,0DACmC,IAAI,UAAU,CAAC;AAAA,IACrE,OAAA;AACF,MAAI,OAAO,WAAW,WAAW,QAAQ;AACvC,UAAM,IAAI;AAAA,MACR,0BAA0B,OAAO,KAAA,CAAM,iDACL,UAAU;AAAA,IAAA;AAAA,EAEhD;AAEA,QAAM,EAAC,SAAQ;AACf,QAAM,SAAS,MAAM;AAAA;AAAA,0BAIG,IAAI;AAC5B,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,IAAI,gBAAgB,oBAAoB,IAAI,aAAa;AAAA,EACjE;AACA,QAAM,CAAC,EAAC,YAAY,UAAA,CAAU,IAAI;AAClC,MAAI,eAAe,QAAQ,cAAc,QAAQ;AAC/C,UAAM,IAAI;AAAA,MACR,oBAAoB,IAAI;AAAA,IAAA;AAAA,EAE5B;AACA,SAAO;AACT;AAMA,MAAM,qBAA6C;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,IACA,aACA,OACA,SACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,eAAe;AACtD,SAAK,eAAe;AACpB,SAAK,SAAS;AACd,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,YAAY,iBAAgD;AAChE,UAAM,KAAK,SAAS,KAAK,KAAK,KAAK,cAAc,CAAA,GAAI,gBAAgB;AACrE,UAAM,EAAC,SAAQ,KAAK;AAEpB,QAAI,UAAU;AACd,QAAI;AACF,OAAC,EAAC,QAAA,IAAW,MAAM,KAAK;AAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,SAAS,MAAM,uBAAuB,IAAI,KAAK,MAAM;AAC3D,WAAK,IAAI,OAAO,+BAA+B,IAAI,EAAE;AACrD,aAAO,MAAM,KAAK,aAAa,IAAI,MAAM,iBAAiB,MAAM;AAAA,IAClE,UAAA;AACE,WAAK,QAAQ,KAAK,MAAM,GAAG,KAAK;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,aACJ,IACA,MACA,iBACA,aACuB;AACvB,UAAM,cAAc,SAAS,eAAe;AAC5C,UAAM,EAAC,UAAU,KAAA,IAAQ,MAAM;AAAA,MAC7B,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,CAAC,GAAG,YAAY,YAAY;AAAA,MAC5B,gBAAgB,WAAW;AAAA,IAAA;AAG7B,UAAM,UAAU,aAAa,OAA4B;AAAA,MACvD,SAAS,MAAM,SAAS,OAAA;AAAA,IAAO,CAChC;AACD,UAAM,QAAQ,IAAI,MAAM,IAAI;AAE5B,UAAM,cAAc,IAAI;AAAA,MACtB,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA,KAAK,SAAS;AAAA,MACd,KAAK;AAAA,IAAA;AAGP,UAAM,iBAAkB;AACtB,UAAI;AACF,yBAAiB,CAAC,KAAK,GAAG,KAAK,UAAU;AACvC,cAAI,IAAI,QAAQ,aAAa;AAC3B,oBAAQ,KAAK,CAAC,UAAU,KAAK,EAAC,WAAW,cAAc,GAAG,EAAA,CAAE,CAAC;AAC7D;AAAA,UACF;AACA,cAAI;AACJ,qBAAW,UAAU,MAAM,YAAY,YAAY,KAAK,GAAG,GAAG;AAC5D,mBAAO,QAAQ,KAAK,MAAM;AAAA,UAC5B;AACA,gBAAM,MAAM;AAAA,QACd;AAAA,MACF,SAAS,GAAG;AACV,gBAAQ,KAAK,eAAe,CAAC,CAAC;AAAA,MAChC;AAAA,IACF,GAAA;AAEA,SAAK,IAAI;AAAA,MACP,8BAA8B,IAAI,SAAS,eAAe,qBACxD,KAAK,SAAS,OAChB;AAAA,IAAA;AAGF,WAAO;AAAA,MACL;AAAA,MACA,MAAM,EAAC,MAAM,CAAA,WAAU,MAAM,IAAI,OAAO,CAAC,EAAE,SAAS,EAAA;AAAA,IAAC;AAAA,EAEzD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,wCACJ,KACA,YACmC;AACnC,UAAM,iBAAiB,0BAA0B,KAAK,MAAM;AAC5D,UAAM,iBAAiB,sBAAsB,KAAK,MAAM;AAIxD,UAAM,SAAS,MAAM;AAAA;AAAA;AAAA,8BAGK,cAAc,mBAAmB,cAAc;AAAA,+BAC9C,UAAU;AACrC,QAAI,OAAO,WAAW,GAAG;AACvB,YAAM,IAAI;AAAA,QACR,oBAAoB,UAAU;AAAA,MAAA;AAAA,IAIlC;AAEA,UAAM,gBAAgB,GAAG,eAAe,KAAK,MAAM,CAAC;AACpD,UAAM,kBAAkB,IAAI,aAAa,CAAC,kBAAkB,UAAU;AAEtE,UAAM,OAAO,OAAO,OAAO,CAAC,EAAC,IAAA,MAAS,QAAQ,IAAI,EAAE,IAAI,CAAC,EAAC,IAAA,MAAS,GAAG;AACtE,QAAI,KAAK,QAAQ;AACf,WAAK,IAAI,OAAO,uBAAuB,IAAI,eAAe;AAAA,IAC5D;AACA,UAAM,aAAa,OAChB,OAAO,CAAC,EAAC,KAAA,MAAU,SAAS,UAAU,EACtC,IAAI,CAAC,EAAC,KAAA,MAAU,IAAI;AACvB,WAAO;AAAA,MACL,SAAS,WAAW,SAChB,KAAK,sBAAsB,KAAK,UAAU,IAC1C;AAAA,IAAA;AAAA,EAER;AAAA,EAEA,MAAM,sBAAsB,KAAiB,OAAiB;AAC5D,SAAK,IAAI,OAAO,sCAAsC,KAAK,EAAE;AAC7D,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,UAAI;AACF,cAAM;AAAA;AAAA,iCAEmB,IAAI,KAAK,CAAC;AAAA;AAEnC,aAAK,IAAI,OAAO,wBAAwB,KAAK,EAAE;AAC/C;AAAA,MACF,SAAS,GAAG;AAEV,YACE,aAAa,SAAS,iBACtB,EAAE,SAAS,kBACX;AAIA,eAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,IAAI,CAAC;AAAA,QACtD,OAAO;AACL,eAAK,IAAI,OAAO,kBAAkB,KAAK,IAAI,CAAC;AAAA,QAC9C;AACA,cAAM,MAAM,GAAI;AAAA,MAClB;AAAA,IACF;AACA,SAAK,IAAI,OAAO,sCAAsC,KAAK,EAAE;AAAA,EAC/D;AACF;AAGO,MAAM,MAAM;AAAA,EACjB;AAAA,EACA;AAAA,EAEA,YAAY,MAAoB;AAC9B,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,YAAY;AAcV,SAAK,oBAAoB,WAAW,MAAM,KAAK,SAAA,GAAY,GAAI;AAAA,EACjE;AAAA,EAEA,IAAI,WAAwB;AAC1B,SAAK,SAAS,SAAS;AAAA,EACzB;AAAA,EAEA,SAAS,WAAyB;AAChC,iBAAa,KAAK,eAAe;AACjC,SAAK,kBAAkB;AAIvB,UAAM,MAAM,YAAY,gBAAgB,SAAS,IAAI;AACrD,SAAK,MAAM,KAAK,GAAG;AAAA,EACrB;AACF;AASA,MAAM,gCAAgC;AAEtC,MAAM,YAAY;AAAA,EACP;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET;AAAA,EACA;AAAA,EAEA,YACE,IACA,EAAC,OAAO,YACR,aACA,eACA,aACA;AACA,SAAK,MAAM;AAEX,SAAK,eAAe,GAAG,KAAK,IAAI,QAAQ;AACxC,SAAK,eAAe;AACpB,SAAK,iBAAiB;AACtB,SAAK,cAAc,SAAS,IAAI,aAAa;AAAA,MAC3C,CAAC,cAAc,GAAG;AAAA;AAAA,MAClB,YAAY,EAAC,CAAC,kBAAkB,GAAG,8BAAA;AAAA,IAA6B,CACjE;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,KAAa,KAA8C;AAC3E,QAAI,KAAK,QAAQ;AACf,WAAK,UAAU,KAAK,MAAM;AAC1B,aAAO,CAAA;AAAA,IACT;AACA,QAAI;AACF,aAAO,MAAM,KAAK,aAAa,GAAG;AAAA,IACpC,SAAS,KAAK;AACZ,WAAK,SAAS,EAAC,KAAK,KAAK,KAAK,aAAa,EAAA;AAC3C,WAAK,UAAU,KAAK,MAAM;AAE1B,YAAM,UAAU,2CAA2C,WAAW,GAAG,CAAC;AAC1E,YAAM,eAA2B,EAAC,OAAO,QAAA;AACzC,UAAI,eAAe,8BAA8B;AAC/C,qBAAa,SAAS,IAAI;AAC1B,qBAAa,UAAU,IAAI,UAAU;AAAA,MACvC,OAAO;AACL,qBAAa,SAAS,OAAO,GAAG;AAAA,MAClC;AAIA,aAAO;AAAA,QACL,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAC9B,CAAC,WAAW,EAAC,KAAK,kBAAkB,SAAS,cAAa;AAAA,MAAA;AAAA,IAE9D;AAAA,EACF;AAAA,EAEA,UAAU,OAAyB;AACjC,UAAM,EAAC,KAAK,KAAK,KAAK,gBAAe;AACrC,UAAM,MAAM,KAAK,IAAA;AAIjB,QAAI,MAAM,cAAc,KAAQ;AAC9B,WAAK,IAAI;AAAA,QACP,2CAA2C,WAAW,GAAG,CAAC,KAAK;AAAA,UAC7D;AAAA,QAAA,CACD;AAAA,QACD,eAAe,+BACX,IAAI,UAAU;AAAA;AAAA,UAEd,EAAC,GAAG,KAAK,SAAS,OAAA;AAAA;AAAA,MAAS;AAEjC,YAAM,cAAc;AAAA,IACtB;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,aAAa,KAA2C;AAC5D,YAAQ,IAAI,KAAA;AAAA,MACV,KAAK;AACH,eAAO;AAAA,UACL;AAAA,YACE;AAAA,YACA,EAAC,GAAG,KAAK,MAAM,IAAA;AAAA,YACf,EAAC,iBAAiB,cAAc,KAAK,IAAI,SAAS,CAAC,EAAA;AAAA,UAAC;AAAA,QACtD;AAAA,MAGJ,KAAK,UAAU;AACb,YAAI,EAAE,IAAI,OAAO,IAAI,MAAM;AACzB,gBAAM,IAAI;AAAA,YACR,qCAAqC,UAAU,GAAG,CAAC;AAAA,UAAA;AAAA,QAEvD;AACA,eAAO;AAAA,UACL;AAAA,YACE;AAAA,YACA;AAAA,cACE,GAAG;AAAA,cACH,UAAU,eAAe,IAAI,QAAQ;AAAA;AAAA,cAErC,KAAK,KAAK,IAAI,OAAO,IAAI,GAAG;AAAA,YAAA;AAAA,UAC9B;AAAA,QACF;AAAA,MAEJ;AAAA,MAEA,KAAK,UAAU;AACb,eAAO;AAAA,UACL;AAAA,YACE;AAAA,YACA;AAAA,cACE,GAAG;AAAA,cACH,UAAU,eAAe,IAAI,QAAQ;AAAA;AAAA,cAErC,KAAK,IAAI,OAAO,IAAI;AAAA,YAAA;AAAA,UACtB;AAAA,QACF;AAAA,MAEJ;AAAA,MAEA,KAAK;AACH,eAAO,CAAC,CAAC,QAAQ,EAAC,GAAG,KAAK,UAAU,eAAe,IAAI,QAAQ,EAAA,CAAE,CAAC;AAAA,MACpE,KAAK;AACH,eAAO;AAAA,UACL,CAAC,QAAQ,EAAC,GAAG,KAAK,WAAW,IAAI,UAAU,IAAI,cAAc,EAAA,CAAE;AAAA,QAAA;AAAA,MAGnE,KAAK;AACH,YAAI,IAAI,WAAW,KAAK,cAAc;AACpC,eAAK,IAAI,QAAQ,wCAAwC,IAAI,MAAM;AACnE,iBAAO,CAAA;AAAA,QACT;AACA,eAAO,KAAK,qBAAqB,GAAG;AAAA,MAEtC,KAAK;AACH,eAAO;AAAA,UACL,CAAC,UAAU,KAAK,EAAC,WAAW,cAAc,KAAK,IAAI,SAAS,CAAC,EAAA,CAAE;AAAA,QAAA;AAAA,MAGnE,KAAK;AACH,eAAO,KAAK,gBAAgB,GAAG;AAAA,MACjC,KAAK;AACH,eAAO,CAAA;AAAA;AAAA,MACT,KAAK;AAGH,eAAO,CAAA;AAAA,MACT;AAEE,cAAM,IAAI,MAAM,2BAA2B,UAAU,GAAG,CAAC,EAAE;AAAA,IAAA;AAAA,EAEjE;AAAA,EAEA;AAAA,EAEA,qBAAqB,KAAqB;AACxC,UAAM,QAAQ,KAAK,uBAAuB,IAAI,OAAO;AAGrD,iBAAa,KAAK,qBAAqB;AAEvC,QAAI,MAAM,SAAS,YAAY;AAE7B,WAAK,aAAa,MAAM;AACxB,aAAO,CAAA;AAAA,IACT;AAEA,UAAM,UAAU,KAAK;AAAA,MACnB,KAAK,KAAK,YAAY,uCAAuC;AAAA,MAC7D;AAAA,IAAA,EACA,IAAI,CAAA,WAAU,CAAC,QAAQ,MAAM,CAAgB;AAE/C,SAAK,IACF,YAAY,SAAS,MAAM,QAAQ,KAAK,EACxC,OAAO,GAAG,QAAQ,MAAM,qBAAqB,OAAO;AAEvD,UAAM,oBAAoB;AAAA,MACxB,MAAM;AAAA,IAAA;AAER,QAAI,mBAAmB;AACrB,WAAK,wBAAwB,WAAW,YAAY;AAClD,YAAI;AACF,gBAAM,kBAAkB,MAAM,KAAK,KAAK,KAAK,WAAW;AAAA,QAC1D,SAAS,KAAK;AACZ,eAAK,IAAI,OAAO,oCAAoC,GAAG;AAAA,QACzD;AAAA,MACF,GAAG,6BAA6B;AAAA,IAClC;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA6BA,mBACE,WACA,QACc;AACd,QAAI;AACF,YAAM,CAAC,SAAS,OAAO,IAAI,UAAU,SAAS;AAC9C,YAAM,CAAC,SAAS,OAAO,IAAI,UAAU,OAAO,MAAM;AAClD,YAAM,UAAwB,CAAA;AAG9B,iBAAW,SAAS,QAAQ,UAAU;AACpC,iBAAS,KAAK,KAAK,OAAO,OAAO,OAAO,OAAO;AAAA,MACjD;AAEA,YAAM,CAAC,YAAY,UAAU,IAAI,qBAAqB,SAAS,OAAO;AACtE,iBAAW,MAAM,YAAY;AAC3B,cAAM,EAAC,QAAQ,KAAA,IAAQ,KAAK,QAAQ,IAAI,EAAE,CAAC;AAC3C,gBAAQ,KAAK,EAAC,KAAK,cAAc,IAAI,EAAC,QAAQ,KAAA,GAAM;AAAA,MACtD;AAGA,YAAM,CAAC,YAAY,UAAU,IAAI,qBAAqB,SAAS,OAAO;AACtE,iBAAW,MAAM,YAAY;AAC3B,cAAM,EAAC,QAAQ,KAAA,IAAQ,KAAK,QAAQ,IAAI,EAAE,CAAC;AAC3C,gBAAQ,KAAK,EAAC,KAAK,cAAc,IAAI,EAAC,QAAQ,KAAA,GAAM;AAAA,MACtD;AAEA,YAAM,SAAS,aAAa,SAAS,OAAO;AAC5C,iBAAW,MAAM,QAAQ;AACvB,gBAAQ;AAAA,UACN,GAAG,KAAK;AAAA,YACN,KAAK,QAAQ,IAAI,EAAE,CAAC;AAAA,YACpB,KAAK,QAAQ,IAAI,EAAE,CAAC;AAAA,UAAA;AAAA,QACtB;AAAA,MAEJ;AAEA,iBAAW,MAAM,YAAY;AAC3B,cAAM,OAAO,KAAK,QAAQ,IAAI,EAAE,CAAC;AACjC,gBAAQ,KAAK,EAAC,KAAK,gBAAgB,MAAK;AAAA,MAC1C;AAIA,iBAAW,MAAM,YAAY;AAC3B,cAAM,OAAO,KAAK,QAAQ,IAAI,EAAE,CAAC;AACjC,gBAAQ,KAAK,EAAC,KAAK,gBAAgB,MAAK;AAAA,MAC1C;AACA,aAAO;AAAA,IACT,SAAS,GAAG;AACV,YAAM,IAAI,6BAA6B,OAAO,CAAC,GAAG,QAAQ,EAAC,OAAO,GAAE;AAAA,IACtE;AAAA,EACF;AAAA,EAEA,iBAAiB,UAAqB,UAAmC;AACvE,UAAM,UAAwB,CAAA;AAC9B,QACE,SAAS,WAAW,SAAS,UAC7B,SAAS,SAAS,SAAS,MAC3B;AACA,cAAQ,KAAK;AAAA,QACX,KAAK;AAAA,QACL,KAAK,EAAC,QAAQ,SAAS,QAAQ,MAAM,SAAS,KAAA;AAAA,QAC9C,KAAK,EAAC,QAAQ,SAAS,QAAQ,MAAM,SAAS,KAAA;AAAA,MAAI,CACnD;AAAA,IACH;AACA,UAAM,QAAQ,EAAC,QAAQ,SAAS,QAAQ,MAAM,SAAS,KAAA;AACvD,UAAM,aAAa,YAAY,SAAS,OAAO;AAC/C,UAAM,aAAa,YAAY,SAAS,OAAO;AAG/C,UAAM,CAAC,SAAS,KAAK,IAAI,qBAAqB,YAAY,UAAU;AACpE,eAAW,MAAM,SAAS;AACxB,YAAM,EAAC,MAAM,OAAA,IAAU,KAAK,WAAW,IAAI,EAAE,CAAC;AAC9C,cAAQ,KAAK,EAAC,KAAK,eAAe,OAAO,QAAO;AAAA,IAClD;AAGA,UAAM,OAAO,aAAa,YAAY,UAAU;AAChD,eAAW,MAAM,MAAM;AACrB,YAAM,EAAC,MAAM,SAAS,GAAG,QAAA,IAAW,KAAK,WAAW,IAAI,EAAE,CAAC;AAC3D,YAAM,EAAC,MAAM,SAAS,GAAG,QAAA,IAAW,KAAK,WAAW,IAAI,EAAE,CAAC;AAK3D,UACE,YAAY,WACZ,QAAQ,aAAa,QAAQ,YAC7B,QAAQ,YAAY,QAAQ,SAC5B;AACA,gBAAQ,KAAK;AAAA,UACX,KAAK;AAAA,UACL;AAAA,UACA,KAAK,EAAC,MAAM,SAAS,MAAM,QAAA;AAAA,UAC3B,KAAK,EAAC,MAAM,SAAS,MAAM,QAAA;AAAA,QAAO,CACnC;AAAA,MACH;AAAA,IACF;AAGA,eAAW,MAAM,OAAO;AACtB,YAAM,EAAC,MAAM,GAAG,KAAA,IAAQ,KAAK,WAAW,IAAI,EAAE,CAAC;AAC/C,YAAM,SAAS,EAAC,MAAM,KAAA;AAEtB,8BAAwB,MAAM,MAAM,MAAM;AAC1C,cAAQ,KAAK,EAAC,KAAK,cAAc,OAAO,QAAO;AAAA,IACjD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAuB,SAAqB;AAC1C,UAAM,MACJ,mBAAmB,SACf,QAAQ,SAAS,OAAO,IACxB,IAAI,cAAc,OAAO,OAAO;AACtC,UAAM,OAAO,KAAK,MAAM,GAAG;AAC3B,WAAOA,MAAQ,MAAM,wBAAwB,aAAa;AAAA,EAC5D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAoBA,MAAM,gBAAgB,KAAoD;AACxE,UAAM,EAAC,cAAc,aAAA,IAAgB,KAAK;AAC1C,QAAI,cAAc;AAChB,aAAO,CAAA;AAAA,IACT;AACA,UAAM,gBAAgB,MAAM;AAAA,MAC1B,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,UAAM,aAAa,oBAAoB,KAAK,gBAAgB,aAAa;AACzE,QAAI,eAAe,MAAM;AACvB,YAAM,IAAI,2BAA2B,UAAU;AAAA,IACjD;AAKA,UAAM,OAAO,KAAK,eAAe,OAAO;AAAA,MACtC,CAAA,MAAK,EAAE,QAAQ,IAAI;AAAA,IAAA;AAErB,QAAI,CAAC,MAAM;AAET,YAAM,IAAI;AAAA,QACR,kCAAkC,UAAU,GAAG,CAAC;AAAA,MAAA;AAAA,IAEpD;AACA,QAAI,kBAAkB,MAAM,GAAG,GAAG;AAChC,YAAM,IAAI;AAAA,QACR,gDAAgD,UAAU,IAAI,CAAC,OAAO,UAAU,GAAG,CAAC;AAAA,MAAA;AAAA,IAExF;AACA,WAAO,CAAA;AAAA,EACT;AACF;AAEA,SAAS,oBACP,GACA,GACe;AAEf,MAAI,EAAE,OAAO,WAAW,EAAE,OAAO,QAAQ;AACvC,WAAO;AAAA,EACT;AACA,WAAS,IAAI,GAAG,IAAI,EAAE,OAAO,QAAQ,KAAK;AACxC,UAAM,KAAK,EAAE,OAAO,CAAC;AACrB,UAAM,KAAK,EAAE,OAAO,CAAC;AACrB,UAAM,aAAa,mBAAmB,IAAI,EAAE;AAC5C,QAAI,YAAY;AACd,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAGA,MAAM,cAAc,CAAC,GAAyB,MAC5C,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,IAAI;AAEvD,SAAS,mBACP,GACA,GACe;AACf,MAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM;AACjE,WAAO,UAAU,EAAE,IAAI,yBAAyB,EAAE,IAAI;AAAA,EACxD;AACA,MAAI,CAAC,UAAU,EAAE,YAAY,EAAE,UAAU,GAAG;AAC1C,WAAO,yBAAyB,EAAE,IAAI;AAAA,EACxC;AACA,QAAM,QAAQ,OAAO,QAAQ,EAAE,OAAO,EAAE,KAAK,WAAW;AACxD,QAAM,QAAQ,OAAO,QAAQ,EAAE,OAAO,EAAE,KAAK,WAAW;AACxD,MACE,MAAM,WAAW,MAAM,UACvB,MAAM,KAAK,CAAC,CAAC,OAAO,IAAI,GAAG,MAAM;AAC/B,UAAM,CAAC,OAAO,IAAI,IAAI,MAAM,CAAC;AAC7B,WACE,UAAU,SACV,KAAK,QAAQ,KAAK,OAClB,KAAK,YAAY,KAAK,WACtB,KAAK,YAAY,KAAK;AAAA,EAE1B,CAAC,GACD;AACA,WAAO,qBAAqB,EAAE,IAAI;AAAA,EACpC;AACA,SAAO;AACT;AAEO,SAAS,kBAAkB,GAAuB,GAAqB;AAC5E,MAAI,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM;AACzE,WAAO;AAAA,EACT;AACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAME,EAAE,oBAAoB,aACtB,CAAC,OAAO,IAAI,IAAI,EAAE,UAAU,GAAG,IAAI,IAAI,EAAE,UAAU,CAAC;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,OAAO,QAAQ,EAAE,OAAO,EAAE,KAAK,WAAW;AACxD,QAAM,QAAQ,EAAE;AAChB,SACE,MAAM,WAAW,MAAM,UACvB,MAAM,KAAK,CAAC,CAAC,OAAO,IAAI,GAAG,MAAM;AAC/B,UAAM,OAAO,MAAM,CAAC;AACpB,WAAO,UAAU,KAAK,QAAQ,KAAK,YAAY,KAAK;AAAA,EACtD,CAAC;AAEL;AAEA,SAAS,eAAe,GAAmB;AACzC,MAAI,EAAE,aAAa,QAAQ;AACzB,WAAO,IAAI,MAAM,OAAO,CAAC,CAAC;AAAA,EAC5B;AACA,MAAI,aAAa,SAAS,iBAAiB,EAAE,SAAS,mBAAmB;AACvE,WAAO,IAAI,eAAe,CAAC;AAAA,EAC7B;AACA,SAAO;AACT;AACA,MAAM,WAAW,CAAC,OAAmB,GAAG,GAAG,MAAM,IAAI,GAAG,IAAI;AAE5D,SAAS,UAAU,WAA4B;AAC7C,SAAO;AAAA;AAAA;AAAA,IAGL,IAAI,IAAI,UAAU,OAAO,IAAI,CAAA,MAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAAA,IAC7C,IAAI,IAAI,UAAU,QAAQ,IAAI,CAAA,MAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAAA,EAAA;AAExD;AAEA,SAAS,YACP,SAC0C;AAC1C,QAAM,+BAAe,IAAA;AACrB,aAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AAGlD,aAAS,IAAI,KAAK,KAAK,EAAC,GAAG,MAAM,MAAK;AAAA,EACxC;AACA,SAAO;AACT;AAIA,SAAS,eAAe,UAA6C;AACnE,QAAM,EAAC,SAAS,GAAG,GAAG,SAAQ;AAC9B,SAAO;AACT;AAEA,MAAM,qCAAqC,MAAM;AAAA,EACtC,OAAO;AAAA,EACP;AAAA,EACA;AAAA,EAET,YACE,aACA,WACA,SACA;AACA;AAAA,MACE,sDAAsD,WAAW;AAAA,MACjE;AAAA,IAAA;AAEF,SAAK,cAAc;AACnB,SAAK,YAAY;AAAA,EACnB;AACF;AAEA,MAAM,mCAAmC,MAAM;AAAA,EACpC,OAAO;AAAA,EAEhB,YAAY,KAAa;AACvB;AAAA,MACE,GAAG,GAAG;AAAA,IAAA;AAAA,EAEV;AACF;AAGA,MAAM,uBAAuB,WAAW;AAAA,EAC7B,OAAO;AAAA,EAEhB,YAAY,OAAgB;AAC1B;AAAA,MACE;AAAA,MACA;AAAA,QACE;AAAA,MAAA;AAAA,IACF;AAAA,EAEJ;AACF;"}
1	+ {"version":3,"file":"change-source.js","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"sourcesContent":["import {\n PG_ADMIN_SHUTDOWN,\n PG_OBJECT_IN_USE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport postgres from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {promiseVoid} from '../../../../../shared/src/resolved-promises.ts';\nimport {\n equals,\n intersection,\n symmetricDifferences,\n} from '../../../../../shared/src/set-utils.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport * as v from '../../../../../shared/src/valita.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {mapPostgresToLiteColumn} from '../../../db/pg-to-lite.ts';\nimport type {\n ColumnSpec,\n PublishedTableSpec,\n TableSpec,\n} from '../../../db/specs.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport {\n oneAfter,\n versionFromLexi,\n versionToLexi,\n type LexiVersion,\n} from '../../../types/lexi-version.ts';\nimport {pgClient, type PostgresDB} from '../../../types/pg.ts';\nimport {\n upstreamSchema,\n type ShardConfig,\n type ShardID,\n} from '../../../types/shards.ts';\nimport type {Sink} from '../../../types/streams.ts';\nimport {Subscription, type PendingResult} from '../../../types/subscription.ts';\nimport type {\n ChangeSource,\n ChangeStream,\n} from '../../change-streamer/change-streamer-service.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n getSubscriptionState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {JSONObject} from '../protocol/current.ts';\nimport type {\n DataChange,\n Identifier,\n MessageRelation,\n} from '../protocol/current/data.ts';\nimport type {\n ChangeStreamData,\n ChangeStreamMessage,\n Data,\n} from '../protocol/current/downstream.ts';\nimport {type InitialSyncOptions} from './initial-sync.ts';\nimport type {\n Message,\n MessageMessage,\n MessageRelation as PostgresRelation,\n} from './logical-replication/pgoutput.types.ts';\nimport {subscribe} from './logical-replication/stream.ts';\nimport {fromBigInt, toLexiVersion, type LSN} from './lsn.ts';\nimport {replicationEventSchema, type DdlUpdateEvent} from './schema/ddl.ts';\nimport {updateShardSchema} from './schema/init.ts';\nimport {getPublicationInfo, type PublishedSchema} from './schema/published.ts';\nimport {\n dropShard,\n getInternalShardConfig,\n getReplicaAtVersion,\n internalPublicationPrefix,\n legacyReplicationSlot,\n replicaIdentitiesForTablesWithoutPrimaryKeys,\n replicationSlotExpression,\n type InternalShardConfig,\n type Replica,\n} from './schema/shard.ts';\nimport {validate} from './schema/validation.ts';\nimport {initSyncSchema} from './sync-schema.ts';\n\n/*\n Initializes a Postgres change source, including the initial sync of the\n * replica, before streaming changes from the corresponding logical replication\n * stream.\n /\nexport async function initializePostgresChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n syncOptions: InitialSyncOptions,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initSyncSchema(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n shard,\n replicaDbFile,\n upstreamURI,\n syncOptions,\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n // Check that upstream is properly setup, and throw an AutoReset to re-run\n // initial sync if not.\n const db = pgClient(lc, upstreamURI);\n try {\n const upstreamReplica = await checkAndUpdateUpstream(\n lc,\n db,\n shard,\n subscriptionState,\n );\n\n const changeSource = new PostgresChangeSource(\n lc,\n upstreamURI,\n shard,\n upstreamReplica,\n );\n\n return {subscriptionState, changeSource};\n } finally {\n await db.end();\n }\n}\n\nasync function checkAndUpdateUpstream(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardConfig,\n {replicaVersion, publications: subscribed}: SubscriptionState,\n) {\n // Perform any shard schema updates\n await updateShardSchema(lc, sql, shard, replicaVersion);\n\n const upstreamReplica = await getReplicaAtVersion(\n lc,\n sql,\n shard,\n replicaVersion,\n );\n if (!upstreamReplica) {\n throw new AutoResetSignal(\n `No replication slot for replica at version ${replicaVersion}`,\n );\n }\n\n // Verify that the publications match what is being replicated.\n const requested = [...shard.publications].sort();\n const replicated = upstreamReplica.publications\n .filter(p => !p.startsWith(internalPublicationPrefix(shard)))\n .sort();\n if (!deepEqual(requested, replicated)) {\n lc.warn?.(`Dropping shard to change publications to: [${requested}]`);\n await sql.unsafe(dropShard(shard.appID, shard.shardNum));\n throw new AutoResetSignal(\n `Requested publications [${requested}] do not match configured ` +\n `publications: [${replicated}]`,\n );\n }\n\n // Sanity check: The subscription state on the replica should have the\n // same publications. This should be guaranteed by the equivalence of the\n // replicaVersion, but it doesn't hurt to verify.\n if (!deepEqual(upstreamReplica.publications, subscribed)) {\n throw new AutoResetSignal(\n `Upstream publications [${upstreamReplica.publications}] do not ` +\n `match subscribed publications [${subscribed}]`,\n );\n }\n\n // Verify that the publications exist.\n const exists = await sql`\n SELECT pubname FROM pg_publication WHERE pubname IN ${sql(subscribed)};\n `.values();\n if (exists.length !== subscribed.length) {\n throw new AutoResetSignal(\n `Upstream publications [${exists.flat()}] do not contain ` +\n `all subscribed publications [${subscribed}]`,\n );\n }\n\n const {slot} = upstreamReplica;\n const result = await sql<\n {restartLSN: LSN \| null; walStatus: string \| null}[]\n > /sql/ `\n SELECT restart_lsn as \"restartLSN\", wal_status as \"walStatus\" FROM pg_replication_slots\n WHERE slot_name = ${slot}`;\n if (result.length === 0) {\n throw new AutoResetSignal(`replication slot ${slot} is missing`);\n }\n const [{restartLSN, walStatus}] = result;\n if (restartLSN === null \|\| walStatus === 'lost') {\n throw new AutoResetSignal(\n `replication slot ${slot} has been invalidated for exceeding the max_slot_wal_keep_size`,\n );\n }\n return upstreamReplica;\n}\n\n/\n Postgres implementation of a {@link ChangeSource} backed by a logical\n * replication stream.\n /\nclass PostgresChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replica: Replica;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replica: Replica,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replica = replica;\n }\n\n async startStream(clientWatermark: string): Promise<ChangeStream> {\n const db = pgClient(this.#lc, this.#upstreamUri, {}, 'json-as-string');\n const {slot} = this.#replica;\n\n let cleanup = promiseVoid;\n try {\n ({cleanup} = await this.#stopExistingReplicationSlotSubscribers(\n db,\n slot,\n ));\n const config = await getInternalShardConfig(db, this.#shard);\n this.#lc.info?.(`starting replication stream@${slot}`);\n return await this.#startStream(db, slot, clientWatermark, config);\n } finally {\n void cleanup.then(() => db.end());\n }\n }\n\n async #startStream(\n db: PostgresDB,\n slot: string,\n clientWatermark: string,\n shardConfig: InternalShardConfig,\n ): Promise<ChangeStream> {\n const clientStart = oneAfter(clientWatermark);\n const {messages, acks} = await subscribe(\n this.#lc,\n db,\n slot,\n [...shardConfig.publications],\n versionFromLexi(clientStart),\n );\n\n const changes = Subscription.create<ChangeStreamMessage>({\n cleanup: () => messages.cancel(),\n });\n const acker = new Acker(acks);\n\n const changeMaker = new ChangeMaker(\n this.#lc,\n this.#shard,\n shardConfig,\n this.#replica.initialSchema,\n this.#upstreamUri,\n );\n\n void (async function () {\n try {\n for await (const [lsn, msg] of messages) {\n if (msg.tag === 'keepalive') {\n changes.push(['status', msg, {watermark: versionToLexi(lsn)}]);\n continue;\n }\n let last: PendingResult \| undefined;\n for (const change of await changeMaker.makeChanges(lsn, msg)) {\n last = changes.push(change);\n }\n await last?.result; // Allow the change-streamer to push back.\n }\n } catch (e) {\n changes.fail(translateError(e));\n }\n })();\n\n this.#lc.info?.(\n `started replication stream@${slot} from ${clientWatermark} (replicaVersion: ${\n this.#replica.version\n })`,\n );\n\n return {\n changes,\n acks: {push: status => acker.ack(status[2].watermark)},\n };\n }\n\n /\n Stops replication slots associated with this shard, and returns\n * a `cleanup` task that drops any slot other than the specified\n * `slotToKeep`.\n \n Note that replication slots created after `slotToKeep` (as indicated by\n * the timestamp suffix) are preserved, as those are newly syncing replicas\n * that will soon take over the slot.\n /\n async #stopExistingReplicationSlotSubscribers(\n sql: PostgresDB,\n slotToKeep: string,\n ): Promise<{cleanup: Promise<void>}> {\n const slotExpression = replicationSlotExpression(this.#shard);\n const legacySlotName = legacyReplicationSlot(this.#shard);\n\n // Note: `slot_name <= slotToKeep` uses a string compare of the millisecond\n // timestamp, which works until it exceeds 13 digits (sometime in 2286).\n const result = await sql<\n {slot: string; pid: string \| null; terminated: boolean \| null}[]\n >`\n SELECT slot_name as slot, pg_terminate_backend(active_pid) as terminated, active_pid as pid\n FROM pg_replication_slots \n WHERE (slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName})\n AND slot_name <= ${slotToKeep}`;\n this.#lc.info?.(`terminated replication slots: ${JSON.stringify(result)}`);\n if (result.length === 0) {\n const shardSlots = await sql<\n {\n slot: string;\n active: boolean;\n pid: string \| null;\n }[]\n >`\n SELECT slot_name as slot, active, active_pid as pid\n FROM pg_replication_slots\n WHERE slot_name LIKE ${slotExpression} OR slot_name = ${legacySlotName}\n ORDER BY slot_name`;\n this.#lc.warn?.(\n `slot ${slotToKeep} not found while cleaning subscribers; shard slots at time of failure: ${JSON.stringify(\n shardSlots,\n )}`,\n );\n throw new AbortError(\n `replication slot ${slotToKeep} is missing. A different ` +\n `replication-manager should now be running on a new ` +\n `replication slot.`,\n );\n }\n // Clear the state of the older replicas.\n const replicasTable = `${upstreamSchema(this.#shard)}.replicas`;\n const replicasBefore = await sql<{slot: string; version: string}[]>`\n SELECT slot, version FROM ${sql(replicasTable)} ORDER BY slot`;\n this.#lc.info?.(\n `replicas before cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(\n replicasBefore,\n )}`,\n );\n await sql<{slot: string; version: string}[]>`\n DELETE FROM ${sql(replicasTable)}\n WHERE slot < ${slotToKeep}`;\n const replicasAfter = await sql<{slot: string; version: string}[]>`\n SELECT slot, version FROM ${sql(replicasTable)} ORDER BY slot`;\n this.#lc.info?.(\n `replicas after cleanup (slotToKeep=${slotToKeep}): ${JSON.stringify(\n replicasAfter,\n )}`,\n );\n\n const pids = result.filter(({pid}) => pid !== null).map(({pid}) => pid);\n if (pids.length) {\n this.#lc.info?.(`signaled subscriber ${pids} to shut down`);\n }\n const otherSlots = result\n .filter(({slot}) => slot !== slotToKeep)\n .map(({slot}) => slot);\n return {\n cleanup: otherSlots.length\n ? this.#dropReplicationSlots(sql, otherSlots)\n : promiseVoid,\n };\n }\n\n async #dropReplicationSlots(sql: PostgresDB, slots: string[]) {\n this.#lc.info?.(`dropping other replication slot(s) ${slots}`);\n for (let i = 0; i < 5; i++) {\n try {\n await sql`\n SELECT pg_drop_replication_slot(slot_name) FROM pg_replication_slots\n WHERE slot_name IN ${sql(slots)}\n `;\n this.#lc.info?.(`successfully dropped ${slots}`);\n return;\n } catch (e) {\n // error: replication slot \"zero_slot_change_source_test_id\" is active for PID 268\n if (\n e instanceof postgres.PostgresError &&\n e.code === PG_OBJECT_IN_USE\n ) {\n // The freeing up of the replication slot is not transactional;\n // sometimes it takes time for Postgres to consider the slot\n // inactive.\n this.#lc.debug?.(`attempt ${i + 1}: ${String(e)}`, e);\n } else {\n this.#lc.warn?.(`error dropping ${slots}`, e);\n }\n await sleep(1000);\n }\n }\n this.#lc.warn?.(`maximum attempts exceeded dropping ${slots}`);\n }\n}\n\n// Exported for testing.\nexport class Acker {\n #acks: Sink<bigint>;\n #keepaliveTimer: NodeJS.Timeout \| undefined;\n\n constructor(acks: Sink<bigint>) {\n this.#acks = acks;\n }\n\n keepalive() {\n // Sets a timeout to send a standby status update in response to\n // a primary keepalive message.\n //\n // https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-PRIMARY-KEEPALIVE-MESSAGE\n //\n // A primary keepalive message is streamed to the change-streamer as a\n // 'status' message, which in turn responds with an ack. However, in the\n // event that the change-streamer is backed up processing preceding\n // changes, this timeout will fire to send a status update that does not\n // change the confirmed flush position. This timeout must be shorter than\n // the `wal_sender_timeout`, which defaults to 60 seconds.\n //\n // https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT\n this.#keepaliveTimer ??= setTimeout(() => this.#sendAck(), 1000);\n }\n\n ack(watermark: LexiVersion) {\n this.#sendAck(watermark);\n }\n\n #sendAck(watermark?: LexiVersion) {\n clearTimeout(this.#keepaliveTimer);\n this.#keepaliveTimer = undefined;\n\n // Note: Sending '0/0' means \"keep alive but do not update confirmed_flush_lsn\"\n // https://github.com/postgres/postgres/blob/3edc67d337c2e498dad1cd200e460f7c63e512e6/src/backend/replication/walsender.c#L2457\n const lsn = watermark ? versionFromLexi(watermark) : 0n;\n this.#acks.push(lsn);\n }\n}\n\ntype ReplicationError = {\n lsn: bigint;\n msg: Message;\n err: unknown;\n lastLogTime: number;\n};\n\nconst SET_REPLICA_IDENTITY_DELAY_MS = 500;\n\nclass ChangeMaker {\n readonly #lc: LogContext;\n readonly #shardPrefix: string;\n readonly #shardConfig: InternalShardConfig;\n readonly #initialSchema: PublishedSchema;\n readonly #upstreamDB: PostgresDB;\n\n #replicaIdentityTimer: NodeJS.Timeout \| undefined;\n #error: ReplicationError \| undefined;\n\n constructor(\n lc: LogContext,\n {appID, shardNum}: ShardID,\n shardConfig: InternalShardConfig,\n initialSchema: PublishedSchema,\n upstreamURI: string,\n ) {\n this.#lc = lc;\n // Note: This matches the prefix used in pg_logical_emit_message() in pg/schema/ddl.ts.\n this.#shardPrefix = `${appID}/${shardNum}`;\n this.#shardConfig = shardConfig;\n this.#initialSchema = initialSchema;\n this.#upstreamDB = pgClient(lc, upstreamURI, {\n ['idle_timeout']: 10, // only used occasionally\n connection: {['application_name']: 'zero-schema-change-detector'},\n });\n }\n\n async makeChanges(lsn: bigint, msg: Message): Promise<ChangeStreamMessage[]> {\n if (this.#error) {\n this.#logError(this.#error);\n return [];\n }\n try {\n return await this.#makeChanges(msg);\n } catch (err) {\n this.#error = {lsn, msg, err, lastLogTime: 0};\n this.#logError(this.#error);\n\n const message = `Unable to continue replication from LSN ${fromBigInt(lsn)}`;\n const errorDetails: JSONObject = {error: message};\n if (err instanceof UnsupportedSchemaChangeError) {\n errorDetails.reason = err.description;\n errorDetails.context = err.ddlUpdate.context;\n } else {\n errorDetails.reason = String(err);\n }\n\n // Rollback the current transaction to avoid dangling transactions in\n // downstream processors (i.e. changeLog, replicator).\n return [\n ['rollback', {tag: 'rollback'}],\n ['control', {tag: 'reset-required', message, errorDetails}],\n ];\n }\n }\n\n #logError(error: ReplicationError) {\n const {lsn, msg, err, lastLogTime} = error;\n const now = Date.now();\n\n // Output an error to logs as replication messages continue to be dropped,\n // at most once a minute.\n if (now - lastLogTime > 60_000) {\n this.#lc.error?.(\n `Unable to continue replication from LSN ${fromBigInt(lsn)}: ${String(\n err,\n )}`,\n err instanceof UnsupportedSchemaChangeError\n ? err.ddlUpdate.context\n : // 'content' can be a large byte Buffer. Exclude it from logging output.\n {...msg, content: undefined},\n );\n error.lastLogTime = now;\n }\n }\n\n // oxlint-disable-next-line require-await\n async #makeChanges(msg: Message): Promise<ChangeStreamData[]> {\n switch (msg.tag) {\n case 'begin':\n return [\n [\n 'begin',\n {...msg, json: 's'},\n {commitWatermark: toLexiVersion(must(msg.commitLsn))},\n ],\n ];\n\n case 'delete': {\n if (!(msg.key ?? msg.old)) {\n throw new Error(\n `Invalid DELETE msg (missing key): ${stringify(msg)}`,\n );\n }\n return [\n [\n 'data',\n {\n ...msg,\n relation: withoutColumns(msg.relation),\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-DELETE\n key: must(msg.old ?? msg.key),\n },\n ],\n ];\n }\n\n case 'update': {\n return [\n [\n 'data',\n {\n ...msg,\n relation: withoutColumns(msg.relation),\n // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-UPDATE\n key: msg.old ?? msg.key,\n },\n ],\n ];\n }\n\n case 'insert':\n return [['data', {...msg, relation: withoutColumns(msg.relation)}]];\n case 'truncate':\n return [\n ['data', {...msg, relations: msg.relations.map(withoutColumns)}],\n ];\n\n case 'message':\n if (msg.prefix !== this.#shardPrefix) {\n this.#lc.debug?.('ignoring message for different shard', msg.prefix);\n return [];\n }\n return this.#handleCustomMessage(msg);\n\n case 'commit':\n return [\n ['commit', msg, {watermark: toLexiVersion(must(msg.commitLsn))}],\n ];\n\n case 'relation':\n return this.#handleRelation(msg);\n case 'type':\n return []; // Nothing need be done for custom types.\n case 'origin':\n // No need to detect replication loops since we are not a\n // PG replication source.\n return [];\n default:\n msg satisfies never;\n throw new Error(`Unexpected message type ${stringify(msg)}`);\n }\n }\n\n #preSchema: PublishedSchema \| undefined;\n\n #handleCustomMessage(msg: MessageMessage) {\n const event = this.#parseReplicationEvent(msg.content);\n // Cancel manual schema adjustment timeouts when an upstream schema change\n // is about to happen, so as to avoid interfering / redundant work.\n clearTimeout(this.#replicaIdentityTimer);\n\n if (event.type === 'ddlStart') {\n // Store the schema in order to diff it with a potential ddlUpdate.\n this.#preSchema = event.schema;\n return [];\n }\n // ddlUpdate\n const changes = this.#makeSchemaChanges(\n must(this.#preSchema, `ddlUpdate received without a ddlStart`),\n event,\n ).map(change => ['data', change] satisfies Data);\n\n this.#lc\n .withContext('query', event.context.query)\n .info?.(`${changes.length} schema change(s)`, changes);\n\n const replicaIdentities = replicaIdentitiesForTablesWithoutPrimaryKeys(\n event.schema,\n );\n if (replicaIdentities) {\n this.#replicaIdentityTimer = setTimeout(async () => {\n try {\n await replicaIdentities.apply(this.#lc, this.#upstreamDB);\n } catch (err) {\n this.#lc.warn?.(`error setting replica identities`, err);\n }\n }, SET_REPLICA_IDENTITY_DELAY_MS);\n }\n\n return changes;\n }\n\n /\n A note on operation order:\n \n Postgres will drop related indexes when columns are dropped,\n * but SQLite will error instead (https://sqlite.org/forum/forumpost/2e62dba69f?t=c&hist).\n * The current workaround is to drop indexes first.\n \n Also note that although it should not be possible to both rename and\n * add/drop tables/columns in a single statement, the operations are\n * ordered to handle that possibility, by always dropping old entities,\n * then modifying kept entities, and then adding new entities.\n \n Thus, the order of replicating DDL updates is:\n * - drop indexes\n * - drop tables\n * - alter tables\n * - drop columns\n * - alter columns\n * - add columns\n * - create tables\n * - create indexes\n \n In the future the replication logic should be improved to handle this\n * behavior in SQLite by dropping dependent indexes manually before dropping\n * columns. This, for example, would be needed to properly support changing\n * the type of a column that's indexed.\n /\n #makeSchemaChanges(\n preSchema: PublishedSchema,\n update: DdlUpdateEvent,\n ): DataChange[] {\n try {\n const [prevTbl, prevIdx] = specsByID(preSchema);\n const [nextTbl, nextIdx] = specsByID(update.schema);\n const changes: DataChange[] = [];\n\n // Validate the new table schemas\n for (const table of nextTbl.values()) {\n validate(this.#lc, table, update.schema.indexes);\n }\n\n const [droppedIdx, createdIdx] = symmetricDifferences(prevIdx, nextIdx);\n for (const id of droppedIdx) {\n const {schema, name} = must(prevIdx.get(id));\n changes.push({tag: 'drop-index', id: {schema, name}});\n }\n\n // DROP\n const [droppedTbl, createdTbl] = symmetricDifferences(prevTbl, nextTbl);\n for (const id of droppedTbl) {\n const {schema, name} = must(prevTbl.get(id));\n changes.push({tag: 'drop-table', id: {schema, name}});\n }\n // ALTER\n const tables = intersection(prevTbl, nextTbl);\n for (const id of tables) {\n changes.push(\n ...this.#getTableChanges(\n must(prevTbl.get(id)),\n must(nextTbl.get(id)),\n ),\n );\n }\n // CREATE\n for (const id of createdTbl) {\n const spec = must(nextTbl.get(id));\n changes.push({tag: 'create-table', spec});\n }\n\n // Add indexes last since they may reference tables / columns that need\n // to be created first.\n for (const id of createdIdx) {\n const spec = must(nextIdx.get(id));\n changes.push({tag: 'create-index', spec});\n }\n return changes;\n } catch (e) {\n throw new UnsupportedSchemaChangeError(String(e), update, {cause: e});\n }\n }\n\n #getTableChanges(oldTable: TableSpec, newTable: TableSpec): DataChange[] {\n const changes: DataChange[] = [];\n if (\n oldTable.schema !== newTable.schema \|\|\n oldTable.name !== newTable.name\n ) {\n changes.push({\n tag: 'rename-table',\n old: {schema: oldTable.schema, name: oldTable.name},\n new: {schema: newTable.schema, name: newTable.name},\n });\n }\n const table = {schema: newTable.schema, name: newTable.name};\n const oldColumns = columnsByID(oldTable.columns);\n const newColumns = columnsByID(newTable.columns);\n\n // DROP\n const [dropped, added] = symmetricDifferences(oldColumns, newColumns);\n for (const id of dropped) {\n const {name: column} = must(oldColumns.get(id));\n changes.push({tag: 'drop-column', table, column});\n }\n\n // ALTER\n const both = intersection(oldColumns, newColumns);\n for (const id of both) {\n const {name: oldName, ...oldSpec} = must(oldColumns.get(id));\n const {name: newName, ...newSpec} = must(newColumns.get(id));\n // The three things that we care about are:\n // 1. name\n // 2. type\n // 3. not-null\n if (\n oldName !== newName \|\|\n oldSpec.dataType !== newSpec.dataType \|\|\n oldSpec.notNull !== newSpec.notNull\n ) {\n changes.push({\n tag: 'update-column',\n table,\n old: {name: oldName, spec: oldSpec},\n new: {name: newName, spec: newSpec},\n });\n }\n }\n\n // ADD\n for (const id of added) {\n const {name, ...spec} = must(newColumns.get(id));\n const column = {name, spec};\n // Validate that the ChangeProcessor will accept the column change.\n mapPostgresToLiteColumn(table.name, column);\n changes.push({tag: 'add-column', table, column});\n }\n return changes;\n }\n\n #parseReplicationEvent(content: Uint8Array) {\n const str =\n content instanceof Buffer\n ? content.toString('utf-8')\n : new TextDecoder().decode(content);\n const json = JSON.parse(str);\n return v.parse(json, replicationEventSchema, 'passthrough');\n }\n\n /\n If `ddlDetection === true`, relation messages are irrelevant,\n * as schema changes are detected by event triggers that\n * emit custom messages.\n \n For degraded-mode replication (`ddlDetection === false`):\n * 1. query the current published schemas on upstream\n * 2. compare that with the InternalShardConfig.initialSchema\n * 3. compare that with the incoming MessageRelation\n * 4. On any discrepancy, throw an UnsupportedSchemaChangeError\n * to halt replication.\n \n Note that schemas queried in step [1] will be post-transaction\n * schemas, which are not necessarily suitable for actually processing\n * the statements in the transaction being replicated. In other words,\n * this mechanism cannot be used to reliably replicate schema changes.\n * However, they serve the purpose determining if schemas have changed.\n */\n async #handleRelation(rel: PostgresRelation): Promise<ChangeStreamData[]> {\n const {publications, ddlDetection} = this.#shardConfig;\n if (ddlDetection) {\n return [];\n }\n const currentSchema = await getPublicationInfo(\n this.#upstreamDB,\n publications,\n );\n const difference = getSchemaDifference(this.#initialSchema, currentSchema);\n if (difference !== null) {\n throw new MissingEventTriggerSupport(difference);\n }\n // Even if the currentSchema is equal to the initialSchema, the\n // MessageRelation itself must be checked to detect transient\n // schema changes within the transaction (e.g. adding and dropping\n // a table, or renaming a column and then renaming it back).\n const orel = this.#initialSchema.tables.find(\n t => t.oid === rel.relationOid,\n );\n if (!orel) {\n // Can happen if a table is created and then dropped in the same transaction.\n throw new MissingEventTriggerSupport(\n `relation not in initialSchema: ${stringify(rel)}`,\n );\n }\n if (relationDifferent(orel, rel)) {\n throw new MissingEventTriggerSupport(\n `relation has changed within the transaction: ${stringify(orel)} vs ${stringify(rel)}`,\n );\n }\n return [];\n }\n}\n\nfunction getSchemaDifference(\n a: PublishedSchema,\n b: PublishedSchema,\n): string \| null {\n // Note: ignore indexes since changes need not to halt replication\n if (a.tables.length !== b.tables.length) {\n return `tables created or dropped`;\n }\n for (let i = 0; i < a.tables.length; i++) {\n const at = a.tables[i];\n const bt = b.tables[i];\n const difference = getTableDifference(at, bt);\n if (difference) {\n return difference;\n }\n }\n return null;\n}\n\n// ColumnSpec comparator\nconst byColumnPos = (a: [string, ColumnSpec], b: [string, ColumnSpec]) =>\n a[1].pos < b[1].pos ? -1 : a[1].pos > b[1].pos ? 1 : 0;\n\nfunction getTableDifference(\n a: PublishedTableSpec,\n b: PublishedTableSpec,\n): string \| null {\n if (a.oid !== b.oid \|\| a.schema !== b.schema \|\| a.name !== b.name) {\n return `Table \"${a.name}\" differs from table \"${b.name}\"`;\n }\n if (!deepEqual(a.primaryKey, b.primaryKey)) {\n return `Primary key of table \"${a.name}\" has changed`;\n }\n const acols = Object.entries(a.columns).sort(byColumnPos);\n const bcols = Object.entries(b.columns).sort(byColumnPos);\n if (\n acols.length !== bcols.length \|\|\n acols.some(([aname, acol], i) => {\n const [bname, bcol] = bcols[i];\n return (\n aname !== bname \|\|\n acol.pos !== bcol.pos \|\|\n acol.typeOID !== bcol.typeOID \|\|\n acol.notNull !== bcol.notNull\n );\n })\n ) {\n return `Columns of table \"${a.name}\" have changed`;\n }\n return null;\n}\n\nexport function relationDifferent(a: PublishedTableSpec, b: PostgresRelation) {\n if (a.oid !== b.relationOid \|\| a.schema !== b.schema \|\| a.name !== b.name) {\n return true;\n }\n if (\n // The MessageRelation's `keyColumns` field contains the columns in column\n // declaration order, whereas the PublishedTableSpec's `primaryKey`\n // contains the columns in primary key (i.e. index) order. Do an\n // order-agnostic compare here since it is not possible to detect\n // key-order changes from the MessageRelation message alone.\n b.replicaIdentity === 'default' &&\n !equals(new Set(a.primaryKey), new Set(b.keyColumns))\n ) {\n return true;\n }\n const acols = Object.entries(a.columns).sort(byColumnPos);\n const bcols = b.columns;\n return (\n acols.length !== bcols.length \|\|\n acols.some(([aname, acol], i) => {\n const bcol = bcols[i];\n return aname !== bcol.name \|\| acol.typeOID !== bcol.typeOid;\n })\n );\n}\n\nfunction translateError(e: unknown): Error {\n if (!(e instanceof Error)) {\n return new Error(String(e));\n }\n if (e instanceof postgres.PostgresError && e.code === PG_ADMIN_SHUTDOWN) {\n return new ShutdownSignal(e);\n }\n return e;\n}\nconst idString = (id: Identifier) => `${id.schema}.${id.name}`;\n\nfunction specsByID(published: PublishedSchema) {\n return [\n // It would have been nice to use a CustomKeyMap here, but we rely on set-utils\n // operations which use plain Sets.\n new Map(published.tables.map(t => [t.oid, t])),\n new Map(published.indexes.map(i => [idString(i), i])),\n ] as const;\n}\n\nfunction columnsByID(\n columns: Record<string, ColumnSpec>,\n): Map<number, ColumnSpec & {name: string}> {\n const colsByID = new Map<number, ColumnSpec & {name: string}>();\n for (const [name, spec] of Object.entries(columns)) {\n // The `pos` field is the `attnum` in `pg_attribute`, which is a stable\n // identifier for the column in this table (i.e. never reused).\n colsByID.set(spec.pos, {...spec, name});\n }\n return colsByID;\n}\n\n// Avoid sending the `columns` from the Postgres MessageRelation message.\n// They are not used downstream and the message can be large.\nfunction withoutColumns(relation: PostgresRelation): MessageRelation {\n const {columns: _, ...rest} = relation;\n return rest;\n}\n\nclass UnsupportedSchemaChangeError extends Error {\n readonly name = 'UnsupportedSchemaChangeError';\n readonly description: string;\n readonly ddlUpdate: DdlUpdateEvent;\n\n constructor(\n description: string,\n ddlUpdate: DdlUpdateEvent,\n options?: ErrorOptions,\n ) {\n super(\n `Replication halted. Resync the replica to recover: ${description}`,\n options,\n );\n this.description = description;\n this.ddlUpdate = ddlUpdate;\n }\n}\n\nclass MissingEventTriggerSupport extends Error {\n readonly name = 'MissingEventTriggerSupport';\n\n constructor(msg: string) {\n super(\n `${msg}. Schema changes cannot be reliably replicated without event trigger support.`,\n );\n }\n}\n\n// TODO(0xcadams): should this be a ProtocolError?\nclass ShutdownSignal extends AbortError {\n readonly name = 'ShutdownSignal';\n\n constructor(cause: unknown) {\n super(\n 'shutdown signal received (e.g. another zero-cache taking over the replication stream)',\n {\n cause,\n },\n );\n }\n}\n"],"names":["v.parse"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0FA,eAAsB,+BACpB,IACA,aACA,OACA,eACA,aAC6E;AAC7E,QAAM;AAAA,IACJ;AAAA,IACA,WAAW,MAAM,KAAK,IAAI,MAAM,QAAQ;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,UAAU,IAAI,SAAS,IAAI,aAAa;AAC9C,QAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;AAC3E,UAAQ,MAAA;AAIR,QAAM,KAAK,SAAS,IAAI,WAAW;AACnC,MAAI;AACF,UAAM,kBAAkB,MAAM;AAAA,MAC5B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAGF,UAAM,eAAe,IAAI;AAAA,MACvB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAGF,WAAO,EAAC,mBAAmB,aAAA;AAAA,EAC7B,UAAA;AACE,UAAM,GAAG,IAAA;AAAA,EACX;AACF;AAEA,eAAe,uBACb,IACA,KACA,OACA,EAAC,gBAAgB,cAAc,cAC/B;AAEA,QAAM,kBAAkB,IAAI,KAAK,OAAO,cAAc;AAEtD,QAAM,kBAAkB,MAAM;AAAA,IAC5B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEF,MAAI,CAAC,iBAAiB;AACpB,UAAM,IAAI;AAAA,MACR,8CAA8C,cAAc;AAAA,IAAA;AAAA,EAEhE;AAGA,QAAM,YAAY,CAAC,GAAG,MAAM,YAAY,EAAE,KAAA;AAC1C,QAAM,aAAa,gBAAgB,aAChC,OAAO,CAAA,MAAK,CAAC,EAAE,WAAW,0BAA0B,KAAK,CAAC,CAAC,EAC3D,KAAA;AACH,MAAI,CAAC,UAAU,WAAW,UAAU,GAAG;AACrC,OAAG,OAAO,8CAA8C,SAAS,GAAG;AACpE,UAAM,IAAI,OAAO,UAAU,MAAM,OAAO,MAAM,QAAQ,CAAC;AACvD,UAAM,IAAI;AAAA,MACR,2BAA2B,SAAS,4CAChB,UAAU;AAAA,IAAA;AAAA,EAElC;AAKA,MAAI,CAAC,UAAU,gBAAgB,cAAc,UAAU,GAAG;AACxD,UAAM,IAAI;AAAA,MACR,0BAA0B,gBAAgB,YAAY,2CAClB,UAAU;AAAA,IAAA;AAAA,EAElD;AAGA,QAAM,SAAS,MAAM;AAAA,0DACmC,IAAI,UAAU,CAAC;AAAA,IACrE,OAAA;AACF,MAAI,OAAO,WAAW,WAAW,QAAQ;AACvC,UAAM,IAAI;AAAA,MACR,0BAA0B,OAAO,KAAA,CAAM,iDACL,UAAU;AAAA,IAAA;AAAA,EAEhD;AAEA,QAAM,EAAC,SAAQ;AACf,QAAM,SAAS,MAAM;AAAA;AAAA,0BAIG,IAAI;AAC5B,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,IAAI,gBAAgB,oBAAoB,IAAI,aAAa;AAAA,EACjE;AACA,QAAM,CAAC,EAAC,YAAY,UAAA,CAAU,IAAI;AAClC,MAAI,eAAe,QAAQ,cAAc,QAAQ;AAC/C,UAAM,IAAI;AAAA,MACR,oBAAoB,IAAI;AAAA,IAAA;AAAA,EAE5B;AACA,SAAO;AACT;AAMA,MAAM,qBAA6C;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,IACA,aACA,OACA,SACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,eAAe;AACtD,SAAK,eAAe;AACpB,SAAK,SAAS;AACd,SAAK,WAAW;AAAA,EAClB;AAAA,EAEA,MAAM,YAAY,iBAAgD;AAChE,UAAM,KAAK,SAAS,KAAK,KAAK,KAAK,cAAc,CAAA,GAAI,gBAAgB;AACrE,UAAM,EAAC,SAAQ,KAAK;AAEpB,QAAI,UAAU;AACd,QAAI;AACF,OAAC,EAAC,QAAA,IAAW,MAAM,KAAK;AAAA,QACtB;AAAA,QACA;AAAA,MAAA;AAEF,YAAM,SAAS,MAAM,uBAAuB,IAAI,KAAK,MAAM;AAC3D,WAAK,IAAI,OAAO,+BAA+B,IAAI,EAAE;AACrD,aAAO,MAAM,KAAK,aAAa,IAAI,MAAM,iBAAiB,MAAM;AAAA,IAClE,UAAA;AACE,WAAK,QAAQ,KAAK,MAAM,GAAG,KAAK;AAAA,IAClC;AAAA,EACF;AAAA,EAEA,MAAM,aACJ,IACA,MACA,iBACA,aACuB;AACvB,UAAM,cAAc,SAAS,eAAe;AAC5C,UAAM,EAAC,UAAU,KAAA,IAAQ,MAAM;AAAA,MAC7B,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,CAAC,GAAG,YAAY,YAAY;AAAA,MAC5B,gBAAgB,WAAW;AAAA,IAAA;AAG7B,UAAM,UAAU,aAAa,OAA4B;AAAA,MACvD,SAAS,MAAM,SAAS,OAAA;AAAA,IAAO,CAChC;AACD,UAAM,QAAQ,IAAI,MAAM,IAAI;AAE5B,UAAM,cAAc,IAAI;AAAA,MACtB,KAAK;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA,KAAK,SAAS;AAAA,MACd,KAAK;AAAA,IAAA;AAGP,UAAM,iBAAkB;AACtB,UAAI;AACF,yBAAiB,CAAC,KAAK,GAAG,KAAK,UAAU;AACvC,cAAI,IAAI,QAAQ,aAAa;AAC3B,oBAAQ,KAAK,CAAC,UAAU,KAAK,EAAC,WAAW,cAAc,GAAG,EAAA,CAAE,CAAC;AAC7D;AAAA,UACF;AACA,cAAI;AACJ,qBAAW,UAAU,MAAM,YAAY,YAAY,KAAK,GAAG,GAAG;AAC5D,mBAAO,QAAQ,KAAK,MAAM;AAAA,UAC5B;AACA,gBAAM,MAAM;AAAA,QACd;AAAA,MACF,SAAS,GAAG;AACV,gBAAQ,KAAK,eAAe,CAAC,CAAC;AAAA,MAChC;AAAA,IACF,GAAA;AAEA,SAAK,IAAI;AAAA,MACP,8BAA8B,IAAI,SAAS,eAAe,qBACxD,KAAK,SAAS,OAChB;AAAA,IAAA;AAGF,WAAO;AAAA,MACL;AAAA,MACA,MAAM,EAAC,MAAM,CAAA,WAAU,MAAM,IAAI,OAAO,CAAC,EAAE,SAAS,EAAA;AAAA,IAAC;AAAA,EAEzD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,wCACJ,KACA,YACmC;AACnC,UAAM,iBAAiB,0BAA0B,KAAK,MAAM;AAC5D,UAAM,iBAAiB,sBAAsB,KAAK,MAAM;AAIxD,UAAM,SAAS,MAAM;AAAA;AAAA;AAAA,8BAKK,cAAc,mBAAmB,cAAc;AAAA,+BAC9C,UAAU;AACrC,SAAK,IAAI,OAAO,iCAAiC,KAAK,UAAU,MAAM,CAAC,EAAE;AACzE,QAAI,OAAO,WAAW,GAAG;AACvB,YAAM,aAAa,MAAM;AAAA;AAAA;AAAA,+BASA,cAAc,mBAAmB,cAAc;AAAA;AAExE,WAAK,IAAI;AAAA,QACP,QAAQ,UAAU,0EAA0E,KAAK;AAAA,UAC/F;AAAA,QAAA,CACD;AAAA,MAAA;AAEH,YAAM,IAAI;AAAA,QACR,oBAAoB,UAAU;AAAA,MAAA;AAAA,IAIlC;AAEA,UAAM,gBAAgB,GAAG,eAAe,KAAK,MAAM,CAAC;AACpD,UAAM,iBAAiB,MAAM;AAAA,kCACC,IAAI,aAAa,CAAC;AAChD,SAAK,IAAI;AAAA,MACP,uCAAuC,UAAU,MAAM,KAAK;AAAA,QAC1D;AAAA,MAAA,CACD;AAAA,IAAA;AAEH,UAAM;AAAA,oBACU,IAAI,aAAa,CAAC;AAAA,uBACf,UAAU;AAC7B,UAAM,gBAAgB,MAAM;AAAA,kCACE,IAAI,aAAa,CAAC;AAChD,SAAK,IAAI;AAAA,MACP,sCAAsC,UAAU,MAAM,KAAK;AAAA,QACzD;AAAA,MAAA,CACD;AAAA,IAAA;AAGH,UAAM,OAAO,OAAO,OAAO,CAAC,EAAC,IAAA,MAAS,QAAQ,IAAI,EAAE,IAAI,CAAC,EAAC,IAAA,MAAS,GAAG;AACtE,QAAI,KAAK,QAAQ;AACf,WAAK,IAAI,OAAO,uBAAuB,IAAI,eAAe;AAAA,IAC5D;AACA,UAAM,aAAa,OAChB,OAAO,CAAC,EAAC,KAAA,MAAU,SAAS,UAAU,EACtC,IAAI,CAAC,EAAC,KAAA,MAAU,IAAI;AACvB,WAAO;AAAA,MACL,SAAS,WAAW,SAChB,KAAK,sBAAsB,KAAK,UAAU,IAC1C;AAAA,IAAA;AAAA,EAER;AAAA,EAEA,MAAM,sBAAsB,KAAiB,OAAiB;AAC5D,SAAK,IAAI,OAAO,sCAAsC,KAAK,EAAE;AAC7D,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,UAAI;AACF,cAAM;AAAA;AAAA,iCAEmB,IAAI,KAAK,CAAC;AAAA;AAEnC,aAAK,IAAI,OAAO,wBAAwB,KAAK,EAAE;AAC/C;AAAA,MACF,SAAS,GAAG;AAEV,YACE,aAAa,SAAS,iBACtB,EAAE,SAAS,kBACX;AAIA,eAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,KAAK,OAAO,CAAC,CAAC,IAAI,CAAC;AAAA,QACtD,OAAO;AACL,eAAK,IAAI,OAAO,kBAAkB,KAAK,IAAI,CAAC;AAAA,QAC9C;AACA,cAAM,MAAM,GAAI;AAAA,MAClB;AAAA,IACF;AACA,SAAK,IAAI,OAAO,sCAAsC,KAAK,EAAE;AAAA,EAC/D;AACF;AAGO,MAAM,MAAM;AAAA,EACjB;AAAA,EACA;AAAA,EAEA,YAAY,MAAoB;AAC9B,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,YAAY;AAcV,SAAK,oBAAoB,WAAW,MAAM,KAAK,SAAA,GAAY,GAAI;AAAA,EACjE;AAAA,EAEA,IAAI,WAAwB;AAC1B,SAAK,SAAS,SAAS;AAAA,EACzB;AAAA,EAEA,SAAS,WAAyB;AAChC,iBAAa,KAAK,eAAe;AACjC,SAAK,kBAAkB;AAIvB,UAAM,MAAM,YAAY,gBAAgB,SAAS,IAAI;AACrD,SAAK,MAAM,KAAK,GAAG;AAAA,EACrB;AACF;AASA,MAAM,gCAAgC;AAEtC,MAAM,YAAY;AAAA,EACP;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET;AAAA,EACA;AAAA,EAEA,YACE,IACA,EAAC,OAAO,YACR,aACA,eACA,aACA;AACA,SAAK,MAAM;AAEX,SAAK,eAAe,GAAG,KAAK,IAAI,QAAQ;AACxC,SAAK,eAAe;AACpB,SAAK,iBAAiB;AACtB,SAAK,cAAc,SAAS,IAAI,aAAa;AAAA,MAC3C,CAAC,cAAc,GAAG;AAAA;AAAA,MAClB,YAAY,EAAC,CAAC,kBAAkB,GAAG,8BAAA;AAAA,IAA6B,CACjE;AAAA,EACH;AAAA,EAEA,MAAM,YAAY,KAAa,KAA8C;AAC3E,QAAI,KAAK,QAAQ;AACf,WAAK,UAAU,KAAK,MAAM;AAC1B,aAAO,CAAA;AAAA,IACT;AACA,QAAI;AACF,aAAO,MAAM,KAAK,aAAa,GAAG;AAAA,IACpC,SAAS,KAAK;AACZ,WAAK,SAAS,EAAC,KAAK,KAAK,KAAK,aAAa,EAAA;AAC3C,WAAK,UAAU,KAAK,MAAM;AAE1B,YAAM,UAAU,2CAA2C,WAAW,GAAG,CAAC;AAC1E,YAAM,eAA2B,EAAC,OAAO,QAAA;AACzC,UAAI,eAAe,8BAA8B;AAC/C,qBAAa,SAAS,IAAI;AAC1B,qBAAa,UAAU,IAAI,UAAU;AAAA,MACvC,OAAO;AACL,qBAAa,SAAS,OAAO,GAAG;AAAA,MAClC;AAIA,aAAO;AAAA,QACL,CAAC,YAAY,EAAC,KAAK,YAAW;AAAA,QAC9B,CAAC,WAAW,EAAC,KAAK,kBAAkB,SAAS,cAAa;AAAA,MAAA;AAAA,IAE9D;AAAA,EACF;AAAA,EAEA,UAAU,OAAyB;AACjC,UAAM,EAAC,KAAK,KAAK,KAAK,gBAAe;AACrC,UAAM,MAAM,KAAK,IAAA;AAIjB,QAAI,MAAM,cAAc,KAAQ;AAC9B,WAAK,IAAI;AAAA,QACP,2CAA2C,WAAW,GAAG,CAAC,KAAK;AAAA,UAC7D;AAAA,QAAA,CACD;AAAA,QACD,eAAe,+BACX,IAAI,UAAU;AAAA;AAAA,UAEd,EAAC,GAAG,KAAK,SAAS,OAAA;AAAA;AAAA,MAAS;AAEjC,YAAM,cAAc;AAAA,IACtB;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,aAAa,KAA2C;AAC5D,YAAQ,IAAI,KAAA;AAAA,MACV,KAAK;AACH,eAAO;AAAA,UACL;AAAA,YACE;AAAA,YACA,EAAC,GAAG,KAAK,MAAM,IAAA;AAAA,YACf,EAAC,iBAAiB,cAAc,KAAK,IAAI,SAAS,CAAC,EAAA;AAAA,UAAC;AAAA,QACtD;AAAA,MAGJ,KAAK,UAAU;AACb,YAAI,EAAE,IAAI,OAAO,IAAI,MAAM;AACzB,gBAAM,IAAI;AAAA,YACR,qCAAqC,UAAU,GAAG,CAAC;AAAA,UAAA;AAAA,QAEvD;AACA,eAAO;AAAA,UACL;AAAA,YACE;AAAA,YACA;AAAA,cACE,GAAG;AAAA,cACH,UAAU,eAAe,IAAI,QAAQ;AAAA;AAAA,cAErC,KAAK,KAAK,IAAI,OAAO,IAAI,GAAG;AAAA,YAAA;AAAA,UAC9B;AAAA,QACF;AAAA,MAEJ;AAAA,MAEA,KAAK,UAAU;AACb,eAAO;AAAA,UACL;AAAA,YACE;AAAA,YACA;AAAA,cACE,GAAG;AAAA,cACH,UAAU,eAAe,IAAI,QAAQ;AAAA;AAAA,cAErC,KAAK,IAAI,OAAO,IAAI;AAAA,YAAA;AAAA,UACtB;AAAA,QACF;AAAA,MAEJ;AAAA,MAEA,KAAK;AACH,eAAO,CAAC,CAAC,QAAQ,EAAC,GAAG,KAAK,UAAU,eAAe,IAAI,QAAQ,EAAA,CAAE,CAAC;AAAA,MACpE,KAAK;AACH,eAAO;AAAA,UACL,CAAC,QAAQ,EAAC,GAAG,KAAK,WAAW,IAAI,UAAU,IAAI,cAAc,EAAA,CAAE;AAAA,QAAA;AAAA,MAGnE,KAAK;AACH,YAAI,IAAI,WAAW,KAAK,cAAc;AACpC,eAAK,IAAI,QAAQ,wCAAwC,IAAI,MAAM;AACnE,iBAAO,CAAA;AAAA,QACT;AACA,eAAO,KAAK,qBAAqB,GAAG;AAAA,MAEtC,KAAK;AACH,eAAO;AAAA,UACL,CAAC,UAAU,KAAK,EAAC,WAAW,cAAc,KAAK,IAAI,SAAS,CAAC,EAAA,CAAE;AAAA,QAAA;AAAA,MAGnE,KAAK;AACH,eAAO,KAAK,gBAAgB,GAAG;AAAA,MACjC,KAAK;AACH,eAAO,CAAA;AAAA;AAAA,MACT,KAAK;AAGH,eAAO,CAAA;AAAA,MACT;AAEE,cAAM,IAAI,MAAM,2BAA2B,UAAU,GAAG,CAAC,EAAE;AAAA,IAAA;AAAA,EAEjE;AAAA,EAEA;AAAA,EAEA,qBAAqB,KAAqB;AACxC,UAAM,QAAQ,KAAK,uBAAuB,IAAI,OAAO;AAGrD,iBAAa,KAAK,qBAAqB;AAEvC,QAAI,MAAM,SAAS,YAAY;AAE7B,WAAK,aAAa,MAAM;AACxB,aAAO,CAAA;AAAA,IACT;AAEA,UAAM,UAAU,KAAK;AAAA,MACnB,KAAK,KAAK,YAAY,uCAAuC;AAAA,MAC7D;AAAA,IAAA,EACA,IAAI,CAAA,WAAU,CAAC,QAAQ,MAAM,CAAgB;AAE/C,SAAK,IACF,YAAY,SAAS,MAAM,QAAQ,KAAK,EACxC,OAAO,GAAG,QAAQ,MAAM,qBAAqB,OAAO;AAEvD,UAAM,oBAAoB;AAAA,MACxB,MAAM;AAAA,IAAA;AAER,QAAI,mBAAmB;AACrB,WAAK,wBAAwB,WAAW,YAAY;AAClD,YAAI;AACF,gBAAM,kBAAkB,MAAM,KAAK,KAAK,KAAK,WAAW;AAAA,QAC1D,SAAS,KAAK;AACZ,eAAK,IAAI,OAAO,oCAAoC,GAAG;AAAA,QACzD;AAAA,MACF,GAAG,6BAA6B;AAAA,IAClC;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EA6BA,mBACE,WACA,QACc;AACd,QAAI;AACF,YAAM,CAAC,SAAS,OAAO,IAAI,UAAU,SAAS;AAC9C,YAAM,CAAC,SAAS,OAAO,IAAI,UAAU,OAAO,MAAM;AAClD,YAAM,UAAwB,CAAA;AAG9B,iBAAW,SAAS,QAAQ,UAAU;AACpC,iBAAS,KAAK,KAAK,OAAO,OAAO,OAAO,OAAO;AAAA,MACjD;AAEA,YAAM,CAAC,YAAY,UAAU,IAAI,qBAAqB,SAAS,OAAO;AACtE,iBAAW,MAAM,YAAY;AAC3B,cAAM,EAAC,QAAQ,KAAA,IAAQ,KAAK,QAAQ,IAAI,EAAE,CAAC;AAC3C,gBAAQ,KAAK,EAAC,KAAK,cAAc,IAAI,EAAC,QAAQ,KAAA,GAAM;AAAA,MACtD;AAGA,YAAM,CAAC,YAAY,UAAU,IAAI,qBAAqB,SAAS,OAAO;AACtE,iBAAW,MAAM,YAAY;AAC3B,cAAM,EAAC,QAAQ,KAAA,IAAQ,KAAK,QAAQ,IAAI,EAAE,CAAC;AAC3C,gBAAQ,KAAK,EAAC,KAAK,cAAc,IAAI,EAAC,QAAQ,KAAA,GAAM;AAAA,MACtD;AAEA,YAAM,SAAS,aAAa,SAAS,OAAO;AAC5C,iBAAW,MAAM,QAAQ;AACvB,gBAAQ;AAAA,UACN,GAAG,KAAK;AAAA,YACN,KAAK,QAAQ,IAAI,EAAE,CAAC;AAAA,YACpB,KAAK,QAAQ,IAAI,EAAE,CAAC;AAAA,UAAA;AAAA,QACtB;AAAA,MAEJ;AAEA,iBAAW,MAAM,YAAY;AAC3B,cAAM,OAAO,KAAK,QAAQ,IAAI,EAAE,CAAC;AACjC,gBAAQ,KAAK,EAAC,KAAK,gBAAgB,MAAK;AAAA,MAC1C;AAIA,iBAAW,MAAM,YAAY;AAC3B,cAAM,OAAO,KAAK,QAAQ,IAAI,EAAE,CAAC;AACjC,gBAAQ,KAAK,EAAC,KAAK,gBAAgB,MAAK;AAAA,MAC1C;AACA,aAAO;AAAA,IACT,SAAS,GAAG;AACV,YAAM,IAAI,6BAA6B,OAAO,CAAC,GAAG,QAAQ,EAAC,OAAO,GAAE;AAAA,IACtE;AAAA,EACF;AAAA,EAEA,iBAAiB,UAAqB,UAAmC;AACvE,UAAM,UAAwB,CAAA;AAC9B,QACE,SAAS,WAAW,SAAS,UAC7B,SAAS,SAAS,SAAS,MAC3B;AACA,cAAQ,KAAK;AAAA,QACX,KAAK;AAAA,QACL,KAAK,EAAC,QAAQ,SAAS,QAAQ,MAAM,SAAS,KAAA;AAAA,QAC9C,KAAK,EAAC,QAAQ,SAAS,QAAQ,MAAM,SAAS,KAAA;AAAA,MAAI,CACnD;AAAA,IACH;AACA,UAAM,QAAQ,EAAC,QAAQ,SAAS,QAAQ,MAAM,SAAS,KAAA;AACvD,UAAM,aAAa,YAAY,SAAS,OAAO;AAC/C,UAAM,aAAa,YAAY,SAAS,OAAO;AAG/C,UAAM,CAAC,SAAS,KAAK,IAAI,qBAAqB,YAAY,UAAU;AACpE,eAAW,MAAM,SAAS;AACxB,YAAM,EAAC,MAAM,OAAA,IAAU,KAAK,WAAW,IAAI,EAAE,CAAC;AAC9C,cAAQ,KAAK,EAAC,KAAK,eAAe,OAAO,QAAO;AAAA,IAClD;AAGA,UAAM,OAAO,aAAa,YAAY,UAAU;AAChD,eAAW,MAAM,MAAM;AACrB,YAAM,EAAC,MAAM,SAAS,GAAG,QAAA,IAAW,KAAK,WAAW,IAAI,EAAE,CAAC;AAC3D,YAAM,EAAC,MAAM,SAAS,GAAG,QAAA,IAAW,KAAK,WAAW,IAAI,EAAE,CAAC;AAK3D,UACE,YAAY,WACZ,QAAQ,aAAa,QAAQ,YAC7B,QAAQ,YAAY,QAAQ,SAC5B;AACA,gBAAQ,KAAK;AAAA,UACX,KAAK;AAAA,UACL;AAAA,UACA,KAAK,EAAC,MAAM,SAAS,MAAM,QAAA;AAAA,UAC3B,KAAK,EAAC,MAAM,SAAS,MAAM,QAAA;AAAA,QAAO,CACnC;AAAA,MACH;AAAA,IACF;AAGA,eAAW,MAAM,OAAO;AACtB,YAAM,EAAC,MAAM,GAAG,KAAA,IAAQ,KAAK,WAAW,IAAI,EAAE,CAAC;AAC/C,YAAM,SAAS,EAAC,MAAM,KAAA;AAEtB,8BAAwB,MAAM,MAAM,MAAM;AAC1C,cAAQ,KAAK,EAAC,KAAK,cAAc,OAAO,QAAO;AAAA,IACjD;AACA,WAAO;AAAA,EACT;AAAA,EAEA,uBAAuB,SAAqB;AAC1C,UAAM,MACJ,mBAAmB,SACf,QAAQ,SAAS,OAAO,IACxB,IAAI,cAAc,OAAO,OAAO;AACtC,UAAM,OAAO,KAAK,MAAM,GAAG;AAC3B,WAAOA,MAAQ,MAAM,wBAAwB,aAAa;AAAA,EAC5D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAoBA,MAAM,gBAAgB,KAAoD;AACxE,UAAM,EAAC,cAAc,aAAA,IAAgB,KAAK;AAC1C,QAAI,cAAc;AAChB,aAAO,CAAA;AAAA,IACT;AACA,UAAM,gBAAgB,MAAM;AAAA,MAC1B,KAAK;AAAA,MACL;AAAA,IAAA;AAEF,UAAM,aAAa,oBAAoB,KAAK,gBAAgB,aAAa;AACzE,QAAI,eAAe,MAAM;AACvB,YAAM,IAAI,2BAA2B,UAAU;AAAA,IACjD;AAKA,UAAM,OAAO,KAAK,eAAe,OAAO;AAAA,MACtC,CAAA,MAAK,EAAE,QAAQ,IAAI;AAAA,IAAA;AAErB,QAAI,CAAC,MAAM;AAET,YAAM,IAAI;AAAA,QACR,kCAAkC,UAAU,GAAG,CAAC;AAAA,MAAA;AAAA,IAEpD;AACA,QAAI,kBAAkB,MAAM,GAAG,GAAG;AAChC,YAAM,IAAI;AAAA,QACR,gDAAgD,UAAU,IAAI,CAAC,OAAO,UAAU,GAAG,CAAC;AAAA,MAAA;AAAA,IAExF;AACA,WAAO,CAAA;AAAA,EACT;AACF;AAEA,SAAS,oBACP,GACA,GACe;AAEf,MAAI,EAAE,OAAO,WAAW,EAAE,OAAO,QAAQ;AACvC,WAAO;AAAA,EACT;AACA,WAAS,IAAI,GAAG,IAAI,EAAE,OAAO,QAAQ,KAAK;AACxC,UAAM,KAAK,EAAE,OAAO,CAAC;AACrB,UAAM,KAAK,EAAE,OAAO,CAAC;AACrB,UAAM,aAAa,mBAAmB,IAAI,EAAE;AAC5C,QAAI,YAAY;AACd,aAAO;AAAA,IACT;AAAA,EACF;AACA,SAAO;AACT;AAGA,MAAM,cAAc,CAAC,GAAyB,MAC5C,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,IAAI;AAEvD,SAAS,mBACP,GACA,GACe;AACf,MAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM;AACjE,WAAO,UAAU,EAAE,IAAI,yBAAyB,EAAE,IAAI;AAAA,EACxD;AACA,MAAI,CAAC,UAAU,EAAE,YAAY,EAAE,UAAU,GAAG;AAC1C,WAAO,yBAAyB,EAAE,IAAI;AAAA,EACxC;AACA,QAAM,QAAQ,OAAO,QAAQ,EAAE,OAAO,EAAE,KAAK,WAAW;AACxD,QAAM,QAAQ,OAAO,QAAQ,EAAE,OAAO,EAAE,KAAK,WAAW;AACxD,MACE,MAAM,WAAW,MAAM,UACvB,MAAM,KAAK,CAAC,CAAC,OAAO,IAAI,GAAG,MAAM;AAC/B,UAAM,CAAC,OAAO,IAAI,IAAI,MAAM,CAAC;AAC7B,WACE,UAAU,SACV,KAAK,QAAQ,KAAK,OAClB,KAAK,YAAY,KAAK,WACtB,KAAK,YAAY,KAAK;AAAA,EAE1B,CAAC,GACD;AACA,WAAO,qBAAqB,EAAE,IAAI;AAAA,EACpC;AACA,SAAO;AACT;AAEO,SAAS,kBAAkB,GAAuB,GAAqB;AAC5E,MAAI,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM;AACzE,WAAO;AAAA,EACT;AACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAME,EAAE,oBAAoB,aACtB,CAAC,OAAO,IAAI,IAAI,EAAE,UAAU,GAAG,IAAI,IAAI,EAAE,UAAU,CAAC;AAAA,IACpD;AACA,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,OAAO,QAAQ,EAAE,OAAO,EAAE,KAAK,WAAW;AACxD,QAAM,QAAQ,EAAE;AAChB,SACE,MAAM,WAAW,MAAM,UACvB,MAAM,KAAK,CAAC,CAAC,OAAO,IAAI,GAAG,MAAM;AAC/B,UAAM,OAAO,MAAM,CAAC;AACpB,WAAO,UAAU,KAAK,QAAQ,KAAK,YAAY,KAAK;AAAA,EACtD,CAAC;AAEL;AAEA,SAAS,eAAe,GAAmB;AACzC,MAAI,EAAE,aAAa,QAAQ;AACzB,WAAO,IAAI,MAAM,OAAO,CAAC,CAAC;AAAA,EAC5B;AACA,MAAI,aAAa,SAAS,iBAAiB,EAAE,SAAS,mBAAmB;AACvE,WAAO,IAAI,eAAe,CAAC;AAAA,EAC7B;AACA,SAAO;AACT;AACA,MAAM,WAAW,CAAC,OAAmB,GAAG,GAAG,MAAM,IAAI,GAAG,IAAI;AAE5D,SAAS,UAAU,WAA4B;AAC7C,SAAO;AAAA;AAAA;AAAA,IAGL,IAAI,IAAI,UAAU,OAAO,IAAI,CAAA,MAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAAA,IAC7C,IAAI,IAAI,UAAU,QAAQ,IAAI,CAAA,MAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;AAAA,EAAA;AAExD;AAEA,SAAS,YACP,SAC0C;AAC1C,QAAM,+BAAe,IAAA;AACrB,aAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,OAAO,GAAG;AAGlD,aAAS,IAAI,KAAK,KAAK,EAAC,GAAG,MAAM,MAAK;AAAA,EACxC;AACA,SAAO;AACT;AAIA,SAAS,eAAe,UAA6C;AACnE,QAAM,EAAC,SAAS,GAAG,GAAG,SAAQ;AAC9B,SAAO;AACT;AAEA,MAAM,qCAAqC,MAAM;AAAA,EACtC,OAAO;AAAA,EACP;AAAA,EACA;AAAA,EAET,YACE,aACA,WACA,SACA;AACA;AAAA,MACE,sDAAsD,WAAW;AAAA,MACjE;AAAA,IAAA;AAEF,SAAK,cAAc;AACnB,SAAK,YAAY;AAAA,EACnB;AACF;AAEA,MAAM,mCAAmC,MAAM;AAAA,EACpC,OAAO;AAAA,EAEhB,YAAY,KAAa;AACvB;AAAA,MACE,GAAG,GAAG;AAAA,IAAA;AAAA,EAEV;AACF;AAGA,MAAM,uBAAuB,WAAW;AAAA,EAC7B,OAAO;AAAA,EAEhB,YAAY,OAAgB;AAC1B;AAAA,MACE;AAAA,MACA;AAAA,QACE;AAAA,MAAA;AAAA,IACF;AAAA,EAEJ;AACF;"}

package/out/zero-cache/src/services/change-source/pg/schema/shard.js CHANGED Viewed

@@ -193,7 +193,7 @@ async function getReplicaAtVersion(lc, sql, shard, replicaVersion) {
   if (result.length === 0) {
     const allReplicas = await sql`
       SELECT * FROM ${schema}.replicas JOIN ${schema}."shardConfig" ON true`;
-    lc.debug?.(`Replica not found in: ${JSON.stringify(allReplicas)}`);
+    lc.info?.(`Replica not found in: ${JSON.stringify(allReplicas)}`);
     return null;
   }
   return parse(result[0], replicaSchema, "passthrough");

package/out/zero-cache/src/services/change-source/pg/schema/shard.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"shard.js","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n getPublicationInfo,\n publishedSchema,\n type PublicationInfo,\n type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\nexport function internalPublicationPrefix({appID}: AppID) {\n return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n const {appID, shardNum} = check(shard);\n return `${appID}_${shardNum}_`;\n}\n\n/*\n An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n /\nexport function replicationSlotExpression(shard: ShardID) {\n // Underscores have a special meaning in LIKE values\n // so they have to be escaped.\n return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string \| number) {\n return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n appID: string,\n shardID: string \| number,\n) {\n return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n const app = id(appSchema(appID));\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${app};\n\n CREATE TABLE IF NOT EXISTS ${app}.\"schemaVersions\" (\n \"minSupportedVersion\" INT4,\n \"maxSupportedVersion\" INT4,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${app}.\"schemaVersions\" (\"lock\", \"minSupportedVersion\", \"maxSupportedVersion\")\n VALUES (true, 1, 1) ON CONFLICT DO NOTHING;\n\n CREATE TABLE IF NOT EXISTS ${app}.permissions (\n \"permissions\" JSONB,\n \"hash\" TEXT,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW.hash = md5(NEW.permissions::text);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE TRIGGER on_set_permissions \n BEFORE INSERT OR UPDATE ON ${app}.permissions\n FOR EACH ROW\n EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"clients\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"lastMutationID\" BIGINT NOT NULL,\n \"userID\" TEXT,\n PRIMARY KEY(\"clientGroupID\", \"clientID\")\n );`;\n}\n\n/\n Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n \n The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n /\nexport function getMutationsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"mutations\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"mutationID\" BIGINT NOT NULL,\n \"result\" JSON NOT NULL,\n PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n shardConfig: ShardConfig,\n metadataPublication: string,\n): string {\n const app = id(appSchema(shardConfig));\n const shard = id(upstreamSchema(shardConfig));\n\n const pubs = [...shardConfig.publications].sort();\n assert(pubs.includes(metadataPublication));\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${shard};\n\n ${getClientsTableDefinition(shard)}\n ${getMutationsTableDefinition(shard)}\n\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n CREATE PUBLICATION ${id(metadataPublication)}\n FOR TABLE ${app}.\"schemaVersions\", ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\" TEXT[] NOT NULL,\n \"ddlDetection\" BOOL NOT NULL,\n\n -- Ensure that there is only a single row in the table.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\",\n \"ddlDetection\" \n ) VALUES (\n ARRAY[${literal(pubs)}], \n false -- set in SAVEPOINT with triggerSetup() statements\n );\n\n CREATE TABLE ${shard}.replicas (\n \"slot\" TEXT PRIMARY KEY,\n \"version\" TEXT NOT NULL,\n \"initialSchema\" JSON NOT NULL\n );\n `;\n}\n\nexport function dropShard(appID: string, shardID: string \| number): string {\n const schema = `${appID}_${shardID}`;\n const metadataPublication = metadataPublicationName(appID, shardID);\n const defaultPublication = defaultPublicationName(appID, shardID);\n\n // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n // so PUBLICATIONs must be dropped explicitly.\n return /sql/ `\n DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n `;\n}\n\nconst internalShardConfigSchema = v.object({\n publications: v.array(v.string()),\n ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n slot: v.string(),\n version: v.string(),\n initialSchema: publishedSchema,\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n const schema = id(upstreamSchema(shard));\n return (\n createEventTriggerStatements(shard) +\n /sql/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n sql: PostgresDB,\n shard: ShardID,\n slot: string,\n replicaVersion: string,\n {tables, indexes}: PublishedSchema,\n) {\n const schema = upstreamSchema(shard);\n const synced: PublishedSchema = {tables, indexes};\n await sql`\n INSERT INTO ${sql(schema)}.replicas (\"slot\", \"version\", \"initialSchema\")\n VALUES (${slot}, ${replicaVersion}, ${synced})`;\n}\n\nexport async function getReplicaAtVersion(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaVersion: string,\n): Promise<Replica \| null> {\n const schema = sql(upstreamSchema(shard));\n const result = await sql`\n SELECT FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n WHERE version = ${replicaVersion};\n `;\n if (result.length === 0) {\n // log out all the replicas and the joined shardConfig\n const allReplicas = await sql`\n SELECT * FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true`;\n lc.debug?.(`Replica not found in: ${JSON.stringify(allReplicas)}`);\n return null;\n }\n return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n sql: PostgresDB,\n shard: ShardID,\n): Promise<InternalShardConfig> {\n const result = await sql`\n SELECT \"publications\", \"ddlDetection\"\n FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n `;\n assert(result.length === 1);\n return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/*\n Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n lc: LogContext,\n sql: PostgresTransaction,\n requested: ShardConfig,\n) {\n const {publications} = requested;\n // Validate requested publications.\n for (const pub of publications) {\n if (pub.startsWith('_')) {\n throw new Error(\n `Publication names starting with \"_\" are reserved for internal use.\\n` +\n `Please use a different name for publication \"${pub}\".`,\n );\n }\n }\n const allPublications: string[] = [];\n\n // Setup application publications.\n if (publications.length) {\n const results = await sql<{pubname: string}[]>`\n SELECT pubname from pg_publication WHERE pubname IN ${sql(\n publications,\n )}`.values();\n\n if (results.length !== publications.length) {\n throw new Error(\n `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n );\n }\n allPublications.push(...publications);\n } else {\n const defaultPublication = defaultPublicationName(\n requested.appID,\n requested.shardNum,\n );\n await sql`\n DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n await sql`\n CREATE PUBLICATION ${sql(defaultPublication)} \n FOR TABLES IN SCHEMA public\n WITH (publish_via_partition_root = true)`;\n allPublications.push(defaultPublication);\n }\n\n const metadataPublication = metadataPublicationName(\n requested.appID,\n requested.shardNum,\n );\n allPublications.push(metadataPublication);\n\n const shard = {...requested, publications: allPublications};\n\n // Setup the global tables and shard tables / publications.\n await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n const pubs = await getPublicationInfo(sql, allPublications);\n await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n lc: LogContext,\n tx: PostgresTransaction,\n shard: ShardConfig,\n) {\n try {\n await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n } catch (e) {\n if (\n !(\n e instanceof postgres.PostgresError &&\n e.code === PG_INSUFFICIENT_PRIVILEGE\n )\n ) {\n throw e;\n }\n // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n lc.warn?.(\n `Unable to create event triggers for schema change detection:\\n\\n` +\n `\"${e.hint ?? e.message}\"\\n\\n` +\n `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n `requiring the replica to be reset (manually or with --auto-reset).`,\n );\n }\n}\n\nexport function validatePublications(\n lc: LogContext,\n published: PublicationInfo,\n) {\n // Verify that all publications export the proper events.\n published.publications.forEach(pub => {\n if (\n !pub.pubinsert \|\|\n !pub.pubtruncate \|\|\n !pub.pubdelete \|\|\n !pub.pubtruncate\n ) {\n // TODO: Make APIError?\n throw new Error(\n `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n );\n }\n });\n\n published.tables.forEach(table => validate(lc, table, published.indexes));\n}\n\ntype ReplicaIdentities = {\n apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n pubs: PublishedSchema,\n): ReplicaIdentities \| undefined {\n const replicaIdentities: {\n schema: string;\n tableName: string;\n indexName: string;\n }[] = [];\n for (const table of pubs.tables) {\n if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n // - UNIQUE\n // - NOT NULL columns\n // - not deferrable (i.e. isImmediate)\n // - not partial (are already filtered out)\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n const {schema, name: tableName} = table;\n for (const {columns, name: indexName} of pubs.indexes.filter(\n idx =>\n idx.schema === schema &&\n idx.tableName === tableName &&\n idx.unique &&\n idx.isImmediate,\n )) {\n if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n continue; // Only indexes with all NOT NULL columns are suitable.\n }\n replicaIdentities.push({schema, tableName, indexName});\n break;\n }\n }\n }\n\n if (replicaIdentities.length === 0) {\n return undefined;\n }\n return {\n apply: async (lc: LogContext, sql: PostgresDB) => {\n for (const {schema, tableName, indexName} of replicaIdentities) {\n lc.info?.(\n `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n );\n await sql`\n ALTER TABLE ${sql(schema)}.${sql(tableName)} \n REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n }\n },\n };\n}\n"],"names":["v.object","v.array","v.string","v.boolean","v.parse"],"mappings":";;;;;;;;;;;;AAoBO,SAAS,0BAA0B,EAAC,SAAe;AACxD,SAAO,IAAI,KAAK;AAClB;AAEO,SAAS,sBAAsB,EAAC,OAAO,YAAoB;AAChE,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAEO,SAAS,sBAAsB,OAAgB;AACpD,QAAM,EAAC,OAAO,aAAY,MAAM,KAAK;AACrC,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAMO,SAAS,0BAA0B,OAAgB;AAGxD,SAAO,GAAG,sBAAsB,KAAK,CAAC,IAAI,WAAW,KAAK,KAAK;AACjE;AAEO,SAAS,mBAAmB,OAAgB;AACjD,SAAO,sBAAsB,KAAK,IAAI,KAAK,IAAA;AAC7C;AAEA,SAAS,uBAAuB,OAAe,SAA0B;AACvE,SAAO,IAAI,KAAK,WAAW,OAAO;AACpC;AAEO,SAAS,wBACd,OACA,SACA;AACA,SAAO,IAAI,KAAK,aAAa,OAAO;AACtC;AAGA,SAAS,YAAY,OAAsB;AACzC,QAAM,MAAM,GAAG,UAAU,KAAK,CAAC;AAE/B;AAAA;AAAA,IAAe;AAAA,gCACe,GAAG;AAAA;AAAA,+BAEJ,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAUlB,GAAG;AAAA;AAAA;AAAA,+BAGY,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,+BAUH,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iCASD,GAAG;AAAA;AAAA,uBAEb,GAAG;AAAA;AAAA,gBAEV,GAAG;AAAA;AAAA;AAEnB;AAEA,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,QAAM,GAAG,OAAO,YAAY,KAAK,CAAC;AACpC;AAEO,SAAS,0BAA0B,QAAgB;AACxD;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAUO,SAAS,4BAA4B,QAAgB;AAC1D;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAEO,MAAM,qBAAqB;AAE3B,SAAS,WACd,aACA,qBACQ;AACR,QAAM,MAAM,GAAG,UAAU,WAAW,CAAC;AACrC,QAAM,QAAQ,GAAG,eAAe,WAAW,CAAC;AAE5C,QAAM,OAAO,CAAC,GAAG,YAAY,YAAY,EAAE,KAAA;AAC3C,SAAO,KAAK,SAAS,mBAAmB,CAAC;AAEzC;AAAA;AAAA,IAAe;AAAA,gCACe,KAAK;AAAA;AAAA,IAEjC,0BAA0B,KAAK,CAAC;AAAA,IAChC,4BAA4B,KAAK,CAAC;AAAA;AAAA,+BAEP,GAAG,mBAAmB,CAAC;AAAA,uBAC/B,GAAG,mBAAmB,CAAC;AAAA,gBAC9B,GAAG,sBAAsB,GAAG,yBAAyB,KAAK,eAAe,KAAK;AAAA;AAAA,iBAE7E,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAQ7B,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA,cAI9B,QAAQ,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA,iBAIV,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAMtB;AAEO,SAAS,UAAU,OAAe,SAAkC;AACzE,QAAM,SAAS,GAAG,KAAK,IAAI,OAAO;AAClC,QAAM,sBAAsB,wBAAwB,OAAO,OAAO;AAClE,QAAM,qBAAqB,uBAAuB,OAAO,OAAO;AAIhE;AAAA;AAAA,IAAe;AAAA,iCACgB,GAAG,kBAAkB,CAAC;AAAA,iCACtB,GAAG,mBAAmB,CAAC;AAAA,4BAC5B,GAAG,MAAM,CAAC;AAAA;AAAA;AAEtC;AAEA,MAAM,4BAA4BA,OAAS;AAAA,EACzC,cAAcC,MAAQC,QAAU;AAAA,EAChC,cAAcC,QAAE;AAClB,CAAC;AAID,MAAM,gBAAgB,0BAA0B,OAAO;AAAA,EACrD,MAAMD,OAAE;AAAA,EACR,SAASA,OAAE;AAAA,EACX,eAAe;AACjB,CAAC;AAOD,SAAS,aAAa,OAA4B;AAChD,QAAM,SAAS,GAAG,eAAe,KAAK,CAAC;AACvC,SACE,6BAA6B,KAAK;AAAA,EAC1B,UAAU,MAAM;AAE5B;AAGA,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT;AACA,QAAM,SAAS,eAAe,KAAK;AACnC,QAAM,SAA0B,EAAC,QAAQ,QAAA;AACzC,QAAM;AAAA,kBACU,IAAI,MAAM,CAAC;AAAA,gBACb,IAAI,KAAK,cAAc,KAAK,MAAM;AAClD;AAEA,eAAsB,oBACpB,IACA,KACA,OACA,gBACyB;AACzB,QAAM,SAAS,IAAI,eAAe,KAAK,CAAC;AACxC,QAAM,SAAS,MAAM;AAAA,oBACH,MAAM,kBAAkB,MAAM;AAAA,wBAC1B,cAAc;AAAA;AAEpC,MAAI,OAAO,WAAW,GAAG;AAEvB,UAAM,cAAc,MAAM;AAAA,sBACR,MAAM,kBAAkB,MAAM;AAChD,OAAG,QAAQ,yBAAyB,KAAK,UAAU,WAAW,CAAC,EAAE;AACjE,WAAO;AAAA,EACT;AACA,SAAOE,MAAQ,OAAO,CAAC,GAAG,eAAe,aAAa;AACxD;AAEA,eAAsB,uBACpB,KACA,OAC8B;AAC9B,QAAM,SAAS,MAAM;AAAA;AAAA,aAEV,IAAI,eAAe,KAAK,CAAC,CAAC;AAAA;AAErC,SAAO,OAAO,WAAW,CAAC;AAC1B,SAAOA,MAAQ,OAAO,CAAC,GAAG,2BAA2B,aAAa;AACpE;AAMA,eAAsB,0BACpB,IACA,KACA,WACA;AACA,QAAM,EAAC,iBAAgB;AAEvB,aAAW,OAAO,cAAc;AAC9B,QAAI,IAAI,WAAW,GAAG,GAAG;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,+CACkD,GAAG;AAAA,MAAA;AAAA,IAEzD;AAAA,EACF;AACA,QAAM,kBAA4B,CAAA;AAGlC,MAAI,aAAa,QAAQ;AACvB,UAAM,UAAU,MAAM;AAAA,0DACgC;AAAA,MACpD;AAAA,IAAA,CACD,GAAG,OAAA;AAEJ,QAAI,QAAQ,WAAW,aAAa,QAAQ;AAC1C,YAAM,IAAI;AAAA,QACR,gDAAgD,YAAY,cAAc,QAAQ,MAAM;AAAA,MAAA;AAAA,IAE5F;AACA,oBAAgB,KAAK,GAAG,YAAY;AAAA,EACtC,OAAO;AACL,UAAM,qBAAqB;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,IAAA;AAEZ,UAAM;AAAA,mCACyB,IAAI,kBAAkB,CAAC;AACtD,UAAM;AAAA,2BACiB,IAAI,kBAAkB,CAAC;AAAA;AAAA;AAG9C,oBAAgB,KAAK,kBAAkB;AAAA,EACzC;AAEA,QAAM,sBAAsB;AAAA,IAC1B,UAAU;AAAA,IACV,UAAU;AAAA,EAAA;AAEZ,kBAAgB,KAAK,mBAAmB;AAExC,QAAM,QAAQ,EAAC,GAAG,WAAW,cAAc,gBAAA;AAG3C,QAAM,IAAI,OAAO,YAAY,KAAK,IAAI,WAAW,OAAO,mBAAmB,CAAC;AAE5E,QAAM,OAAO,MAAM,mBAAmB,KAAK,eAAe;AAC1D,QAAM,6CAA6C,IAAI,GAAG,MAAM,IAAI,GAAG;AAEvE,QAAM,cAAc,IAAI,KAAK,KAAK;AACpC;AAEA,eAAsB,cACpB,IACA,IACA,OACA;AACA,MAAI;AACF,UAAM,GAAG,UAAU,CAAA,QAAO,IAAI,OAAO,aAAa,KAAK,CAAC,CAAC;AAAA,EAC3D,SAAS,GAAG;AACV,QACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,4BAEb;AACA,YAAM;AAAA,IACR;AAEA,OAAG;AAAA,MACD;AAAA;AAAA,GACM,EAAE,QAAQ,EAAE,OAAO;AAAA;AAAA;AAAA;AAAA,IAAA;AAAA,EAI7B;AACF;AAEO,SAAS,qBACd,IACA,WACA;AAEA,YAAU,aAAa,QAAQ,CAAA,QAAO;AACpC,QACE,CAAC,IAAI,aACL,CAAC,IAAI,eACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL;AAEA,YAAM,IAAI;AAAA,QACR,eAAe,IAAI,OAAO;AAAA,MAAA;AAAA,IAE9B;AAAA,EACF,CAAC;AAED,YAAU,OAAO,QAAQ,CAAA,UAAS,SAAS,IAAI,OAAO,UAAU,OAAO,CAAC;AAC1E;AAMO,SAAS,6CACd,MAC+B;AAC/B,QAAM,oBAIA,CAAA;AACN,aAAW,SAAS,KAAK,QAAQ;AAC/B,QAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAoB,SAAS;AAQlE,YAAM,EAAC,QAAQ,MAAM,UAAA,IAAa;AAClC,iBAAW,EAAC,SAAS,MAAM,UAAA,KAAc,KAAK,QAAQ;AAAA,QACpD,CAAA,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI;AAAA,MAAA,GACL;AACD,YAAI,OAAO,KAAK,OAAO,EAAE,KAAK,CAAA,QAAO,CAAC,MAAM,QAAQ,GAAG,EAAE,OAAO,GAAG;AACjE;AAAA,QACF;AACA,0BAAkB,KAAK,EAAC,QAAQ,WAAW,WAAU;AACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,kBAAkB,WAAW,GAAG;AAClC,WAAO;AAAA,EACT;AACA,SAAO;AAAA,IACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,iBAAW,EAAC,QAAQ,WAAW,UAAA,KAAc,mBAAmB;AAC9D,WAAG;AAAA,UACD,YAAY,SAAS,kCAAkC,SAAS;AAAA,QAAA;AAElE,cAAM;AAAA,sBACQ,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;AAAA,yCACV,IAAI,SAAS,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EAAA;AAEJ;"}
1	+ {"version":3,"file":"shard.js","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/shard.ts"],"sourcesContent":["import {PG_INSUFFICIENT_PRIVILEGE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {literal} from 'pg-format';\nimport postgres from 'postgres';\nimport {assert} from '../../../../../../shared/src/asserts.ts';\nimport * as v from '../../../../../../shared/src/valita.ts';\nimport {Default} from '../../../../db/postgres-replica-identity-enum.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../../../types/pg.ts';\nimport type {AppID, ShardConfig, ShardID} from '../../../../types/shards.ts';\nimport {appSchema, check, upstreamSchema} from '../../../../types/shards.ts';\nimport {id} from '../../../../types/sql.ts';\nimport {createEventTriggerStatements} from './ddl.ts';\nimport {\n getPublicationInfo,\n publishedSchema,\n type PublicationInfo,\n type PublishedSchema,\n} from './published.ts';\nimport {validate} from './validation.ts';\n\nexport function internalPublicationPrefix({appID}: AppID) {\n return `_${appID}_`;\n}\n\nexport function legacyReplicationSlot({appID, shardNum}: ShardID) {\n return `${appID}_${shardNum}`;\n}\n\nexport function replicationSlotPrefix(shard: ShardID) {\n const {appID, shardNum} = check(shard);\n return `${appID}_${shardNum}_`;\n}\n\n/*\n An expression used to match replication slots in the shard\n * in a Postgres `LIKE` operator.\n /\nexport function replicationSlotExpression(shard: ShardID) {\n // Underscores have a special meaning in LIKE values\n // so they have to be escaped.\n return `${replicationSlotPrefix(shard)}%`.replaceAll('_', '\\\\_');\n}\n\nexport function newReplicationSlot(shard: ShardID) {\n return replicationSlotPrefix(shard) + Date.now();\n}\n\nfunction defaultPublicationName(appID: string, shardID: string \| number) {\n return `_${appID}_public_${shardID}`;\n}\n\nexport function metadataPublicationName(\n appID: string,\n shardID: string \| number,\n) {\n return `_${appID}_metadata_${shardID}`;\n}\n\n// The GLOBAL_SETUP must be idempotent as it can be run multiple times for different shards.\nfunction globalSetup(appID: AppID): string {\n const app = id(appSchema(appID));\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${app};\n\n CREATE TABLE IF NOT EXISTS ${app}.\"schemaVersions\" (\n \"minSupportedVersion\" INT4,\n \"maxSupportedVersion\" INT4,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${app}.\"schemaVersions\" (\"lock\", \"minSupportedVersion\", \"maxSupportedVersion\")\n VALUES (true, 1, 1) ON CONFLICT DO NOTHING;\n\n CREATE TABLE IF NOT EXISTS ${app}.permissions (\n \"permissions\" JSONB,\n \"hash\" TEXT,\n\n -- Ensure that there is only a single row in the table.\n -- Application code can be agnostic to this column, and\n -- simply invoke UPDATE statements on the version columns.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n CREATE OR REPLACE FUNCTION ${app}.set_permissions_hash()\n RETURNS TRIGGER AS $$\n BEGIN\n NEW.hash = md5(NEW.permissions::text);\n RETURN NEW;\n END;\n $$ LANGUAGE plpgsql;\n\n CREATE OR REPLACE TRIGGER on_set_permissions \n BEFORE INSERT OR UPDATE ON ${app}.permissions\n FOR EACH ROW\n EXECUTE FUNCTION ${app}.set_permissions_hash();\n\n INSERT INTO ${app}.permissions (permissions) VALUES (NULL) ON CONFLICT DO NOTHING;\n`;\n}\n\nexport async function ensureGlobalTables(db: PostgresDB, appID: AppID) {\n await db.unsafe(globalSetup(appID));\n}\n\nexport function getClientsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"clients\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"lastMutationID\" BIGINT NOT NULL,\n \"userID\" TEXT,\n PRIMARY KEY(\"clientGroupID\", \"clientID\")\n );`;\n}\n\n/\n Tracks the results of mutations.\n * 1. It is an error for the same mutation ID to be used twice.\n * 2. The result is JSONB to allow for arbitrary results.\n \n The tables must be cleaned up as the clients\n * receive the mutation responses and as clients are removed.\n /\nexport function getMutationsTableDefinition(schema: string) {\n return /sql/ `\n CREATE TABLE ${schema}.\"mutations\" (\n \"clientGroupID\" TEXT NOT NULL,\n \"clientID\" TEXT NOT NULL,\n \"mutationID\" BIGINT NOT NULL,\n \"result\" JSON NOT NULL,\n PRIMARY KEY(\"clientGroupID\", \"clientID\", \"mutationID\")\n );`;\n}\n\nexport const SHARD_CONFIG_TABLE = 'shardConfig';\n\nexport function shardSetup(\n shardConfig: ShardConfig,\n metadataPublication: string,\n): string {\n const app = id(appSchema(shardConfig));\n const shard = id(upstreamSchema(shardConfig));\n\n const pubs = [...shardConfig.publications].sort();\n assert(pubs.includes(metadataPublication));\n\n return /sql/ `\n CREATE SCHEMA IF NOT EXISTS ${shard};\n\n ${getClientsTableDefinition(shard)}\n ${getMutationsTableDefinition(shard)}\n\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n CREATE PUBLICATION ${id(metadataPublication)}\n FOR TABLE ${app}.\"schemaVersions\", ${app}.\"permissions\", TABLE ${shard}.\"clients\", ${shard}.\"mutations\";\n\n CREATE TABLE ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\" TEXT[] NOT NULL,\n \"ddlDetection\" BOOL NOT NULL,\n\n -- Ensure that there is only a single row in the table.\n \"lock\" BOOL PRIMARY KEY DEFAULT true CHECK (lock)\n );\n\n INSERT INTO ${shard}.\"${SHARD_CONFIG_TABLE}\" (\n \"publications\",\n \"ddlDetection\" \n ) VALUES (\n ARRAY[${literal(pubs)}], \n false -- set in SAVEPOINT with triggerSetup() statements\n );\n\n CREATE TABLE ${shard}.replicas (\n \"slot\" TEXT PRIMARY KEY,\n \"version\" TEXT NOT NULL,\n \"initialSchema\" JSON NOT NULL\n );\n `;\n}\n\nexport function dropShard(appID: string, shardID: string \| number): string {\n const schema = `${appID}_${shardID}`;\n const metadataPublication = metadataPublicationName(appID, shardID);\n const defaultPublication = defaultPublicationName(appID, shardID);\n\n // DROP SCHEMA ... CASCADE does not drop dependent PUBLICATIONS,\n // so PUBLICATIONs must be dropped explicitly.\n return /sql/ `\n DROP PUBLICATION IF EXISTS ${id(defaultPublication)};\n DROP PUBLICATION IF EXISTS ${id(metadataPublication)};\n DROP SCHEMA IF EXISTS ${id(schema)} CASCADE;\n `;\n}\n\nconst internalShardConfigSchema = v.object({\n publications: v.array(v.string()),\n ddlDetection: v.boolean(),\n});\n\nexport type InternalShardConfig = v.Infer<typeof internalShardConfigSchema>;\n\nconst replicaSchema = internalShardConfigSchema.extend({\n slot: v.string(),\n version: v.string(),\n initialSchema: publishedSchema,\n});\n\nexport type Replica = v.Infer<typeof replicaSchema>;\n\n// triggerSetup is run separately in a sub-transaction (i.e. SAVEPOINT) so\n// that a failure (e.g. due to lack of superuser permissions) can be handled\n// by continuing in a degraded mode (ddlDetection = false).\nfunction triggerSetup(shard: ShardConfig): string {\n const schema = id(upstreamSchema(shard));\n return (\n createEventTriggerStatements(shard) +\n /sql/ `UPDATE ${schema}.\"shardConfig\" SET \"ddlDetection\" = true;`\n );\n}\n\n// Called in initial-sync to store the exact schema that was initially synced.\nexport async function addReplica(\n sql: PostgresDB,\n shard: ShardID,\n slot: string,\n replicaVersion: string,\n {tables, indexes}: PublishedSchema,\n) {\n const schema = upstreamSchema(shard);\n const synced: PublishedSchema = {tables, indexes};\n await sql`\n INSERT INTO ${sql(schema)}.replicas (\"slot\", \"version\", \"initialSchema\")\n VALUES (${slot}, ${replicaVersion}, ${synced})`;\n}\n\nexport async function getReplicaAtVersion(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n replicaVersion: string,\n): Promise<Replica \| null> {\n const schema = sql(upstreamSchema(shard));\n const result = await sql`\n SELECT FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true\n WHERE version = ${replicaVersion};\n `;\n if (result.length === 0) {\n // log out all the replicas and the joined shardConfig\n const allReplicas = await sql`\n SELECT * FROM ${schema}.replicas JOIN ${schema}.\"shardConfig\" ON true`;\n lc.info?.(`Replica not found in: ${JSON.stringify(allReplicas)}`);\n return null;\n }\n return v.parse(result[0], replicaSchema, 'passthrough');\n}\n\nexport async function getInternalShardConfig(\n sql: PostgresDB,\n shard: ShardID,\n): Promise<InternalShardConfig> {\n const result = await sql`\n SELECT \"publications\", \"ddlDetection\"\n FROM ${sql(upstreamSchema(shard))}.\"shardConfig\";\n `;\n assert(result.length === 1);\n return v.parse(result[0], internalShardConfigSchema, 'passthrough');\n}\n\n/*\n Sets up and returns all publications (including internal ones) for\n * the given shard.\n */\nexport async function setupTablesAndReplication(\n lc: LogContext,\n sql: PostgresTransaction,\n requested: ShardConfig,\n) {\n const {publications} = requested;\n // Validate requested publications.\n for (const pub of publications) {\n if (pub.startsWith('_')) {\n throw new Error(\n `Publication names starting with \"_\" are reserved for internal use.\\n` +\n `Please use a different name for publication \"${pub}\".`,\n );\n }\n }\n const allPublications: string[] = [];\n\n // Setup application publications.\n if (publications.length) {\n const results = await sql<{pubname: string}[]>`\n SELECT pubname from pg_publication WHERE pubname IN ${sql(\n publications,\n )}`.values();\n\n if (results.length !== publications.length) {\n throw new Error(\n `Unknown or invalid publications. Specified: [${publications}]. Found: [${results.flat()}]`,\n );\n }\n allPublications.push(...publications);\n } else {\n const defaultPublication = defaultPublicationName(\n requested.appID,\n requested.shardNum,\n );\n await sql`\n DROP PUBLICATION IF EXISTS ${sql(defaultPublication)}`;\n await sql`\n CREATE PUBLICATION ${sql(defaultPublication)} \n FOR TABLES IN SCHEMA public\n WITH (publish_via_partition_root = true)`;\n allPublications.push(defaultPublication);\n }\n\n const metadataPublication = metadataPublicationName(\n requested.appID,\n requested.shardNum,\n );\n allPublications.push(metadataPublication);\n\n const shard = {...requested, publications: allPublications};\n\n // Setup the global tables and shard tables / publications.\n await sql.unsafe(globalSetup(shard) + shardSetup(shard, metadataPublication));\n\n const pubs = await getPublicationInfo(sql, allPublications);\n await replicaIdentitiesForTablesWithoutPrimaryKeys(pubs)?.apply(lc, sql);\n\n await setupTriggers(lc, sql, shard);\n}\n\nexport async function setupTriggers(\n lc: LogContext,\n tx: PostgresTransaction,\n shard: ShardConfig,\n) {\n try {\n await tx.savepoint(sub => sub.unsafe(triggerSetup(shard)));\n } catch (e) {\n if (\n !(\n e instanceof postgres.PostgresError &&\n e.code === PG_INSUFFICIENT_PRIVILEGE\n )\n ) {\n throw e;\n }\n // If triggerSetup() fails, replication continues in ddlDetection=false mode.\n lc.warn?.(\n `Unable to create event triggers for schema change detection:\\n\\n` +\n `\"${e.hint ?? e.message}\"\\n\\n` +\n `Proceeding in degraded mode: schema changes will halt replication,\\n` +\n `requiring the replica to be reset (manually or with --auto-reset).`,\n );\n }\n}\n\nexport function validatePublications(\n lc: LogContext,\n published: PublicationInfo,\n) {\n // Verify that all publications export the proper events.\n published.publications.forEach(pub => {\n if (\n !pub.pubinsert \|\|\n !pub.pubtruncate \|\|\n !pub.pubdelete \|\|\n !pub.pubtruncate\n ) {\n // TODO: Make APIError?\n throw new Error(\n `PUBLICATION ${pub.pubname} must publish insert, update, delete, and truncate`,\n );\n }\n });\n\n published.tables.forEach(table => validate(lc, table, published.indexes));\n}\n\ntype ReplicaIdentities = {\n apply(lc: LogContext, db: PostgresDB): Promise<void>;\n};\n\nexport function replicaIdentitiesForTablesWithoutPrimaryKeys(\n pubs: PublishedSchema,\n): ReplicaIdentities \| undefined {\n const replicaIdentities: {\n schema: string;\n tableName: string;\n indexName: string;\n }[] = [];\n for (const table of pubs.tables) {\n if (!table.primaryKey?.length && table.replicaIdentity === Default) {\n // Look for an index that can serve as the REPLICA IDENTITY USING INDEX. It must be:\n // - UNIQUE\n // - NOT NULL columns\n // - not deferrable (i.e. isImmediate)\n // - not partial (are already filtered out)\n //\n // https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY\n const {schema, name: tableName} = table;\n for (const {columns, name: indexName} of pubs.indexes.filter(\n idx =>\n idx.schema === schema &&\n idx.tableName === tableName &&\n idx.unique &&\n idx.isImmediate,\n )) {\n if (Object.keys(columns).some(col => !table.columns[col].notNull)) {\n continue; // Only indexes with all NOT NULL columns are suitable.\n }\n replicaIdentities.push({schema, tableName, indexName});\n break;\n }\n }\n }\n\n if (replicaIdentities.length === 0) {\n return undefined;\n }\n return {\n apply: async (lc: LogContext, sql: PostgresDB) => {\n for (const {schema, tableName, indexName} of replicaIdentities) {\n lc.info?.(\n `setting \"${indexName}\" as the REPLICA IDENTITY for \"${tableName}\"`,\n );\n await sql`\n ALTER TABLE ${sql(schema)}.${sql(tableName)} \n REPLICA IDENTITY USING INDEX ${sql(indexName)}`;\n }\n },\n };\n}\n"],"names":["v.object","v.array","v.string","v.boolean","v.parse"],"mappings":";;;;;;;;;;;;AAoBO,SAAS,0BAA0B,EAAC,SAAe;AACxD,SAAO,IAAI,KAAK;AAClB;AAEO,SAAS,sBAAsB,EAAC,OAAO,YAAoB;AAChE,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAEO,SAAS,sBAAsB,OAAgB;AACpD,QAAM,EAAC,OAAO,aAAY,MAAM,KAAK;AACrC,SAAO,GAAG,KAAK,IAAI,QAAQ;AAC7B;AAMO,SAAS,0BAA0B,OAAgB;AAGxD,SAAO,GAAG,sBAAsB,KAAK,CAAC,IAAI,WAAW,KAAK,KAAK;AACjE;AAEO,SAAS,mBAAmB,OAAgB;AACjD,SAAO,sBAAsB,KAAK,IAAI,KAAK,IAAA;AAC7C;AAEA,SAAS,uBAAuB,OAAe,SAA0B;AACvE,SAAO,IAAI,KAAK,WAAW,OAAO;AACpC;AAEO,SAAS,wBACd,OACA,SACA;AACA,SAAO,IAAI,KAAK,aAAa,OAAO;AACtC;AAGA,SAAS,YAAY,OAAsB;AACzC,QAAM,MAAM,GAAG,UAAU,KAAK,CAAC;AAE/B;AAAA;AAAA,IAAe;AAAA,gCACe,GAAG;AAAA;AAAA,+BAEJ,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAUlB,GAAG;AAAA;AAAA;AAAA,+BAGY,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,+BAUH,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,iCASD,GAAG;AAAA;AAAA,uBAEb,GAAG;AAAA;AAAA,gBAEV,GAAG;AAAA;AAAA;AAEnB;AAEA,eAAsB,mBAAmB,IAAgB,OAAc;AACrE,QAAM,GAAG,OAAO,YAAY,KAAK,CAAC;AACpC;AAEO,SAAS,0BAA0B,QAAgB;AACxD;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAUO,SAAS,4BAA4B,QAAgB;AAC1D;AAAA;AAAA,IAAe;AAAA,iBACA,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOvB;AAEO,MAAM,qBAAqB;AAE3B,SAAS,WACd,aACA,qBACQ;AACR,QAAM,MAAM,GAAG,UAAU,WAAW,CAAC;AACrC,QAAM,QAAQ,GAAG,eAAe,WAAW,CAAC;AAE5C,QAAM,OAAO,CAAC,GAAG,YAAY,YAAY,EAAE,KAAA;AAC3C,SAAO,KAAK,SAAS,mBAAmB,CAAC;AAEzC;AAAA;AAAA,IAAe;AAAA,gCACe,KAAK;AAAA;AAAA,IAEjC,0BAA0B,KAAK,CAAC;AAAA,IAChC,4BAA4B,KAAK,CAAC;AAAA;AAAA,+BAEP,GAAG,mBAAmB,CAAC;AAAA,uBAC/B,GAAG,mBAAmB,CAAC;AAAA,gBAC9B,GAAG,sBAAsB,GAAG,yBAAyB,KAAK,eAAe,KAAK;AAAA;AAAA,iBAE7E,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,gBAQ7B,KAAK,KAAK,kBAAkB;AAAA;AAAA;AAAA;AAAA,cAI9B,QAAQ,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA,iBAIV,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAMtB;AAEO,SAAS,UAAU,OAAe,SAAkC;AACzE,QAAM,SAAS,GAAG,KAAK,IAAI,OAAO;AAClC,QAAM,sBAAsB,wBAAwB,OAAO,OAAO;AAClE,QAAM,qBAAqB,uBAAuB,OAAO,OAAO;AAIhE;AAAA;AAAA,IAAe;AAAA,iCACgB,GAAG,kBAAkB,CAAC;AAAA,iCACtB,GAAG,mBAAmB,CAAC;AAAA,4BAC5B,GAAG,MAAM,CAAC;AAAA;AAAA;AAEtC;AAEA,MAAM,4BAA4BA,OAAS;AAAA,EACzC,cAAcC,MAAQC,QAAU;AAAA,EAChC,cAAcC,QAAE;AAClB,CAAC;AAID,MAAM,gBAAgB,0BAA0B,OAAO;AAAA,EACrD,MAAMD,OAAE;AAAA,EACR,SAASA,OAAE;AAAA,EACX,eAAe;AACjB,CAAC;AAOD,SAAS,aAAa,OAA4B;AAChD,QAAM,SAAS,GAAG,eAAe,KAAK,CAAC;AACvC,SACE,6BAA6B,KAAK;AAAA,EAC1B,UAAU,MAAM;AAE5B;AAGA,eAAsB,WACpB,KACA,OACA,MACA,gBACA,EAAC,QAAQ,WACT;AACA,QAAM,SAAS,eAAe,KAAK;AACnC,QAAM,SAA0B,EAAC,QAAQ,QAAA;AACzC,QAAM;AAAA,kBACU,IAAI,MAAM,CAAC;AAAA,gBACb,IAAI,KAAK,cAAc,KAAK,MAAM;AAClD;AAEA,eAAsB,oBACpB,IACA,KACA,OACA,gBACyB;AACzB,QAAM,SAAS,IAAI,eAAe,KAAK,CAAC;AACxC,QAAM,SAAS,MAAM;AAAA,oBACH,MAAM,kBAAkB,MAAM;AAAA,wBAC1B,cAAc;AAAA;AAEpC,MAAI,OAAO,WAAW,GAAG;AAEvB,UAAM,cAAc,MAAM;AAAA,sBACR,MAAM,kBAAkB,MAAM;AAChD,OAAG,OAAO,yBAAyB,KAAK,UAAU,WAAW,CAAC,EAAE;AAChE,WAAO;AAAA,EACT;AACA,SAAOE,MAAQ,OAAO,CAAC,GAAG,eAAe,aAAa;AACxD;AAEA,eAAsB,uBACpB,KACA,OAC8B;AAC9B,QAAM,SAAS,MAAM;AAAA;AAAA,aAEV,IAAI,eAAe,KAAK,CAAC,CAAC;AAAA;AAErC,SAAO,OAAO,WAAW,CAAC;AAC1B,SAAOA,MAAQ,OAAO,CAAC,GAAG,2BAA2B,aAAa;AACpE;AAMA,eAAsB,0BACpB,IACA,KACA,WACA;AACA,QAAM,EAAC,iBAAgB;AAEvB,aAAW,OAAO,cAAc;AAC9B,QAAI,IAAI,WAAW,GAAG,GAAG;AACvB,YAAM,IAAI;AAAA,QACR;AAAA,+CACkD,GAAG;AAAA,MAAA;AAAA,IAEzD;AAAA,EACF;AACA,QAAM,kBAA4B,CAAA;AAGlC,MAAI,aAAa,QAAQ;AACvB,UAAM,UAAU,MAAM;AAAA,0DACgC;AAAA,MACpD;AAAA,IAAA,CACD,GAAG,OAAA;AAEJ,QAAI,QAAQ,WAAW,aAAa,QAAQ;AAC1C,YAAM,IAAI;AAAA,QACR,gDAAgD,YAAY,cAAc,QAAQ,MAAM;AAAA,MAAA;AAAA,IAE5F;AACA,oBAAgB,KAAK,GAAG,YAAY;AAAA,EACtC,OAAO;AACL,UAAM,qBAAqB;AAAA,MACzB,UAAU;AAAA,MACV,UAAU;AAAA,IAAA;AAEZ,UAAM;AAAA,mCACyB,IAAI,kBAAkB,CAAC;AACtD,UAAM;AAAA,2BACiB,IAAI,kBAAkB,CAAC;AAAA;AAAA;AAG9C,oBAAgB,KAAK,kBAAkB;AAAA,EACzC;AAEA,QAAM,sBAAsB;AAAA,IAC1B,UAAU;AAAA,IACV,UAAU;AAAA,EAAA;AAEZ,kBAAgB,KAAK,mBAAmB;AAExC,QAAM,QAAQ,EAAC,GAAG,WAAW,cAAc,gBAAA;AAG3C,QAAM,IAAI,OAAO,YAAY,KAAK,IAAI,WAAW,OAAO,mBAAmB,CAAC;AAE5E,QAAM,OAAO,MAAM,mBAAmB,KAAK,eAAe;AAC1D,QAAM,6CAA6C,IAAI,GAAG,MAAM,IAAI,GAAG;AAEvE,QAAM,cAAc,IAAI,KAAK,KAAK;AACpC;AAEA,eAAsB,cACpB,IACA,IACA,OACA;AACA,MAAI;AACF,UAAM,GAAG,UAAU,CAAA,QAAO,IAAI,OAAO,aAAa,KAAK,CAAC,CAAC;AAAA,EAC3D,SAAS,GAAG;AACV,QACE,EACE,aAAa,SAAS,iBACtB,EAAE,SAAS,4BAEb;AACA,YAAM;AAAA,IACR;AAEA,OAAG;AAAA,MACD;AAAA;AAAA,GACM,EAAE,QAAQ,EAAE,OAAO;AAAA;AAAA;AAAA;AAAA,IAAA;AAAA,EAI7B;AACF;AAEO,SAAS,qBACd,IACA,WACA;AAEA,YAAU,aAAa,QAAQ,CAAA,QAAO;AACpC,QACE,CAAC,IAAI,aACL,CAAC,IAAI,eACL,CAAC,IAAI,aACL,CAAC,IAAI,aACL;AAEA,YAAM,IAAI;AAAA,QACR,eAAe,IAAI,OAAO;AAAA,MAAA;AAAA,IAE9B;AAAA,EACF,CAAC;AAED,YAAU,OAAO,QAAQ,CAAA,UAAS,SAAS,IAAI,OAAO,UAAU,OAAO,CAAC;AAC1E;AAMO,SAAS,6CACd,MAC+B;AAC/B,QAAM,oBAIA,CAAA;AACN,aAAW,SAAS,KAAK,QAAQ;AAC/B,QAAI,CAAC,MAAM,YAAY,UAAU,MAAM,oBAAoB,SAAS;AAQlE,YAAM,EAAC,QAAQ,MAAM,UAAA,IAAa;AAClC,iBAAW,EAAC,SAAS,MAAM,UAAA,KAAc,KAAK,QAAQ;AAAA,QACpD,CAAA,QACE,IAAI,WAAW,UACf,IAAI,cAAc,aAClB,IAAI,UACJ,IAAI;AAAA,MAAA,GACL;AACD,YAAI,OAAO,KAAK,OAAO,EAAE,KAAK,CAAA,QAAO,CAAC,MAAM,QAAQ,GAAG,EAAE,OAAO,GAAG;AACjE;AAAA,QACF;AACA,0BAAkB,KAAK,EAAC,QAAQ,WAAW,WAAU;AACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,kBAAkB,WAAW,GAAG;AAClC,WAAO;AAAA,EACT;AACA,SAAO;AAAA,IACL,OAAO,OAAO,IAAgB,QAAoB;AAChD,iBAAW,EAAC,QAAQ,WAAW,UAAA,KAAc,mBAAmB;AAC9D,WAAG;AAAA,UACD,YAAY,SAAS,kCAAkC,SAAS;AAAA,QAAA;AAElE,cAAM;AAAA,sBACQ,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC;AAAA,yCACV,IAAI,SAAS,CAAC;AAAA,MACjD;AAAA,IACF;AAAA,EAAA;AAEJ;"}

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js CHANGED Viewed

@@ -214,6 +214,11 @@ class ChangeStreamerImpl {
   }
   async getChangeLogState() {
     const minWatermark = await this.#storer.getMinWatermarkForCatchup();
+    if (!minWatermark) {
+      this.#lc.warn?.(
+        `Unexpected empty changeLog. Resync if "Local replica watermark" errors arise`
+      );
+    }
     return {
       replicaVersion: this.#replicaVersion,
       minWatermark: minWatermark ?? this.#replicaVersion

package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Sink, Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n type ChangeStreamMessage,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeSourceUpstream} from '../change-source/protocol/current/upstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n autoReset,\n setTimeoutFn,\n );\n}\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\nexport type ChangeStream = {\n changes: Source<ChangeStreamMessage>;\n\n /\n A Sink to push the {@link StatusMessage}s that reflect Commits\n * that have been successfully stored by the {@link Storer}, or\n * downstream {@link StatusMessage}s henceforth.\n /\n acks: Sink<ChangeSourceUpstream>;\n};\n\n/* Encapsulates an upstream-specific implementation of a stream of Changes. /\nexport interface ChangeSource {\n /\n Starts a stream of changes starting after the specific watermark,\n * with a corresponding sink for upstream acknowledgements.\n /\n startStream(afterWatermark: string): Promise<ChangeStream>;\n}\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n autoReset: boolean,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n );\n this.#forwarder = new Forwarder();\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n // The storer will, in turn, detect changes to ownership and stop\n // the change-streamer appropriately.\n this.#storer\n .run()\n .then(() => this.stop())\n .catch(e => this.stop(e));\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n try {\n const startAfter = await this.#storer.getLastWatermarkToStartStream();\n const stream = await this.#source.startStream(startAfter);\n this.#stream = stream;\n this.#state.resetBackoff();\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n this.#storer.status(change); // storer acks once it gets through its queue\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n this.#storer.store([watermark, change]);\n this.#forwarder.forward([watermark, change]);\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n }\n\n await this.#state.backoff(this.#lc, err);\n }\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n\n if (initial) {\n this.scheduleCleanup(watermark);\n }\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);\n this.#initialWatermarks.delete(earliestInitial);\n }\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;AA6CA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AA4KA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,IAAA;AAEtB,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAGnB,SAAK,QACF,IAAA,EACA,KAAK,MAAM,KAAK,KAAA,CAAM,EACtB,MAAM,CAAA,MAAK,KAAK,KAAK,CAAC,CAAC;AAE1B,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI;AACF,cAAM,aAAa,MAAM,KAAK,QAAQ,8BAAA;AACtC,cAAM,SAAS,MAAM,KAAK,QAAQ,YAAY,UAAU;AACxD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,mBAAK,QAAQ,OAAO,MAAM;AAC1B;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,eAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACtC,eAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AAE3C,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,aAAK,WAAW,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,WAAA,CAAW,CAAC,CAAC;AAAA,MACtE;AAEA,YAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,IACzC;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA,EAEA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,aAAK,IAAI,OAAO,UAAU,OAAO,mBAAmB,eAAe,EAAE;AACrE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}
1	+ {"version":3,"file":"change-streamer-service.js","sources":["../../../../../../zero-cache/src/services/change-streamer/change-streamer-service.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {unreachable} from '../../../../shared/src/asserts.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {\n min,\n type AtLeastOne,\n type LexiVersion,\n} from '../../types/lexi-version.ts';\nimport type {PostgresDB} from '../../types/pg.ts';\nimport type {ShardID} from '../../types/shards.ts';\nimport type {Sink, Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n type ChangeStreamControl,\n type ChangeStreamData,\n type ChangeStreamMessage,\n} from '../change-source/protocol/current/downstream.ts';\nimport type {ChangeSourceUpstream} from '../change-source/protocol/current/upstream.ts';\nimport {publishReplicationError} from '../replicator/replication-status.ts';\nimport type {SubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n DEFAULT_MAX_RETRY_DELAY_MS,\n RunningState,\n UnrecoverableError,\n} from '../running-state.ts';\nimport {\n type ChangeStreamerService,\n type Downstream,\n type SubscriberContext,\n} from './change-streamer.ts';\nimport * as ErrorType from './error-type-enum.ts';\nimport {Forwarder} from './forwarder.ts';\nimport {initChangeStreamerSchema} from './schema/init.ts';\nimport {\n AutoResetSignal,\n ensureReplicationConfig,\n markResetRequired,\n} from './schema/tables.ts';\nimport {Storer} from './storer.ts';\nimport {Subscriber} from './subscriber.ts';\n\n/*\n Performs initialization and schema migrations to initialize a ChangeStreamerImpl.\n /\nexport async function initializeStreamer(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n changeSource: ChangeSource,\n subscriptionState: SubscriptionState,\n autoReset: boolean,\n setTimeoutFn = setTimeout,\n): Promise<ChangeStreamerService> {\n // Make sure the ChangeLog DB is set up.\n await initChangeStreamerSchema(lc, changeDB, shard);\n await ensureReplicationConfig(\n lc,\n changeDB,\n subscriptionState,\n shard,\n autoReset,\n );\n\n const {replicaVersion} = subscriptionState;\n return new ChangeStreamerImpl(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n changeSource,\n autoReset,\n setTimeoutFn,\n );\n}\n\n/\n Internally all Downstream messages (not just commits) are given a watermark.\n * These are used for internal ordering for:\n * 1. Replaying new changes in the Storer\n * 2. Filtering old changes in the Subscriber\n \n However, only the watermark for `Commit` messages are exposed to\n * subscribers, as that is the only semantically correct watermark to\n * use for tracking a position in a replication stream.\n /\nexport type WatermarkedChange = [watermark: string, ChangeStreamData];\n\nexport type ChangeStream = {\n changes: Source<ChangeStreamMessage>;\n\n /\n A Sink to push the {@link StatusMessage}s that reflect Commits\n * that have been successfully stored by the {@link Storer}, or\n * downstream {@link StatusMessage}s henceforth.\n /\n acks: Sink<ChangeSourceUpstream>;\n};\n\n/* Encapsulates an upstream-specific implementation of a stream of Changes. /\nexport interface ChangeSource {\n /\n Starts a stream of changes starting after the specific watermark,\n * with a corresponding sink for upstream acknowledgements.\n /\n startStream(afterWatermark: string): Promise<ChangeStream>;\n}\n\n/\n Upstream-agnostic dispatch of messages in a {@link ChangeStreamMessage} to a\n * {@link Forwarder} and {@link Storer} to execute the forward-store-ack\n * procedure described in {@link ChangeStreamer}.\n \n ### Subscriber Catchup\n \n Connecting clients first need to be \"caught up\" to the current watermark\n * (from stored change log entries) before new entries are forwarded to\n * them. This is non-trivial because the replication stream may be in the\n * middle of a pending streamed Transaction for which some entries have\n * already been forwarded but are not yet committed to the store.\n \n \n * ```\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * \| Historic changes in storage \| Pending (streamed) tx \| Next tx\n * ------------------------------- - - - - - - - - - - - - - - - - - - -\n * Replication stream\n * > > > > > > > > >\n * ^ ---> required catchup ---> ^\n * Subscriber watermark Subscription begins\n * ```\n \n Preemptively buffering the changes of every pending transaction\n * would be wasteful and consume too much memory for large transactions.\n \n Instead, the streamer synchronously dispatches changes and subscriptions\n * to the {@link Forwarder} and the {@link Storer} such that the two\n * components are aligned as to where in the stream the subscription started.\n * The two components then coordinate catchup and handoff via the\n * {@link Subscriber} object with the following algorithm:\n \n * If the streamer is in the middle of a pending Transaction, the\n * Subscriber is \"queued\" on both the Forwarder and the Storer. In this\n * state, new changes are not forwarded to the Subscriber, and catchup\n * is not yet executed.\n * * Once the commit message for the pending Transaction is processed\n * by the Storer, it begins catchup on the Subscriber (with a READONLY\n * snapshot so that it does not block subsequent storage operations).\n * This catchup is thus guaranteed to load the change log entries of\n * that last Transaction.\n * * When the Forwarder processes that same commit message, it moves the\n * Subscriber from the \"queued\" to the \"active\" set of clients such that\n * the Subscriber begins receiving new changes, starting from the next\n * Transaction.\n * * The Subscriber does not forward those changes, however, if its catchup\n * is not complete. Until then, it buffers the changes in memory.\n * * Once catchup is complete, the buffered changes are immediately sent\n * and the Subscriber henceforth forwards changes as they are received.\n \n In the (common) case where the streamer is not in the middle of a pending\n * transaction when a subscription begins, the Storer begins catchup\n * immediately and the Forwarder directly adds the Subscriber to its active\n * set. However, the Subscriber still buffers any forwarded messages until\n * its catchup is complete.\n \n ### Watermarks and ordering\n \n The ChangeStreamerService depends on its {@link ChangeSource} to send\n * changes in contiguous [`begin`, `data` ..., `data`, `commit`] sequences\n * in commit order. This follows Postgres's Logical Replication Protocol\n * Message Flow:\n \n https://www.postgresql.org/docs/16/protocol-logical-replication.html#PROTOCOL-LOGICAL-MESSAGES-FLOW\n \n > The logical replication protocol sends individual transactions one by one.\n * > This means that all messages between a pair of Begin and Commit messages belong to the same transaction.\n \n In order to correctly replay (new) and filter (old) messages to subscribers\n * at different points in the replication stream, these changes must be assigned\n * watermarks such that they preserve the order in which they were received\n * from the ChangeSource.\n \n A previous implementation incorrectly derived these watermarks from the Postgres\n * Log Sequence Numbers (LSN) of each message. However, LSNs from concurrent,\n * non-conflicting transactions can overlap, which can result in a `begin` message\n * with an earlier LSN arriving after a `commit` message. For example, the\n * changes for these transactions:\n \n ```\n * LSN: 1 2 3 4 5 6 7 8 9 10\n * tx1: begin data data data commit\n * tx2: begin data data data commit\n * ```\n \n will arrive as:\n \n ```\n * begin1, data2, data4, data6, commit8, begin3, data5, data7, data9, commit10\n * ```\n \n Thus, LSN of non-commit messages are not suitable for tracking the sorting\n * order of the replication stream.\n \n Instead, the ChangeStreamer uses the following algorithm for deterministic\n * catchup and filtering of changes:\n \n * A `commit` message is assigned to a watermark corresponding to its LSN.\n * These are guaranteed to be in commit order by definition.\n \n * `begin` and `data` messages are assigned to the watermark of the\n * preceding `commit` (the previous transaction, or the replication\n * slot's starting LSN) plus 1. This guarantees that they will be sorted\n * after the previously commit transaction even if their LSNs came before it.\n * This is referred to as the `preCommitWatermark`.\n \n * In the ChangeLog DB, messages have a secondary sort column `pos`, which is\n * the position of the message within its transaction, with the `begin` message\n * starting at `0`. This guarantees that `begin` and `data` messages will be\n * fetched in the original ChangeSource order during catchup.\n \n `begin` and `data` messages share the same watermark, but this is sufficient for\n * Subscriber filtering because subscribers only know about the `commit` watermarks\n * exposed in the `Downstream` `Commit` message. The Subscriber object thus compares\n * the internal watermarks of the incoming messages against the commit watermark of\n * the caller, updating the watermark at every `Commit` message that is forwarded.\n \n ### Cleanup\n \n As mentioned in the {@link ChangeStreamer} documentation: \"the ChangeStreamer\n * uses a combination of [the \"initial\", i.e. backup-derived watermark and] ACK\n * responses from connected subscribers to determine the watermark up\n * to which it is safe to purge old change log entries.\"\n \n More concretely:\n \n * The `initial`, backup-derived watermark is the earliest to which cleanup\n * should ever happen.\n \n * However, it is possible for the replica backup to be ahead of a connected\n * subscriber; and if a network error causes that subscriber to retry from its\n * last watermark, the change streamer must support it.\n \n Thus, before cleaning up to an `initial` backup-derived watermark, the change\n * streamer first confirms that all connected subscribers have also passed\n * that watermark.\n /\nclass ChangeStreamerImpl implements ChangeStreamerService {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #shard: ShardID;\n readonly #changeDB: PostgresDB;\n readonly #replicaVersion: string;\n readonly #source: ChangeSource;\n readonly #storer: Storer;\n readonly #forwarder: Forwarder;\n\n readonly #autoReset: boolean;\n readonly #state: RunningState;\n readonly #initialWatermarks = new Set<string>();\n\n // Starting the (Postgres) ChangeStream results in killing the previous\n // Postgres subscriber, potentially creating a gap in which the old\n // change-streamer has shut down and the new change-streamer has not yet\n // been recognized as \"healthy\" (and thus does not get any requests).\n //\n // To minimize this gap, delay starting the ChangeStream until the first\n // request from a `serving` replicator, indicating that higher level\n // load-balancing / routing logic has begun routing requests to this task.\n readonly #serving = resolver();\n\n readonly #txCounter = getOrCreateCounter(\n 'replication',\n 'transactions',\n 'Count of replicated transactions',\n );\n\n #stream: ChangeStream \| undefined;\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n taskID: string,\n discoveryAddress: string,\n discoveryProtocol: string,\n changeDB: PostgresDB,\n replicaVersion: string,\n source: ChangeSource,\n autoReset: boolean,\n setTimeoutFn = setTimeout,\n ) {\n this.id = `change-streamer`;\n this.#lc = lc.withContext('component', 'change-streamer');\n this.#shard = shard;\n this.#changeDB = changeDB;\n this.#replicaVersion = replicaVersion;\n this.#source = source;\n this.#storer = new Storer(\n lc,\n shard,\n taskID,\n discoveryAddress,\n discoveryProtocol,\n changeDB,\n replicaVersion,\n consumed => this.#stream?.acks.push(['status', consumed[1], consumed[2]]),\n err => this.stop(err),\n );\n this.#forwarder = new Forwarder();\n this.#autoReset = autoReset;\n this.#state = new RunningState(this.id, undefined, setTimeoutFn);\n }\n\n async run() {\n this.#lc.info?.('starting change stream');\n\n // Once this change-streamer acquires \"ownership\" of the change DB,\n // it is safe to start the storer.\n await this.#storer.assumeOwnership();\n // The storer will, in turn, detect changes to ownership and stop\n // the change-streamer appropriately.\n this.#storer\n .run()\n .then(() => this.stop())\n .catch(e => this.stop(e));\n\n while (this.#state.shouldRun()) {\n let err: unknown;\n let watermark: string \| null = null;\n try {\n const startAfter = await this.#storer.getLastWatermarkToStartStream();\n const stream = await this.#source.startStream(startAfter);\n this.#stream = stream;\n this.#state.resetBackoff();\n watermark = null;\n\n for await (const change of stream.changes) {\n const [type, msg] = change;\n switch (type) {\n case 'status':\n this.#storer.status(change); // storer acks once it gets through its queue\n continue;\n case 'control':\n await this.#handleControlMessage(msg);\n continue; // control messages are not stored/forwarded\n case 'begin':\n watermark = change[2].commitWatermark;\n break;\n case 'commit':\n if (watermark !== change[2].watermark) {\n throw new UnrecoverableError(\n `commit watermark ${change[2].watermark} does not match 'begin' watermark ${watermark}`,\n );\n }\n this.#txCounter.add(1);\n break;\n default:\n if (watermark === null) {\n throw new UnrecoverableError(\n `${type} change (${msg.tag}) received before 'begin' message`,\n );\n }\n break;\n }\n\n this.#storer.store([watermark, change]);\n this.#forwarder.forward([watermark, change]);\n\n if (type === 'commit' \|\| type === 'rollback') {\n watermark = null;\n }\n\n // Allow the storer to exert back pressure.\n const readyForMore = this.#storer.readyForMore();\n if (readyForMore) {\n await readyForMore;\n }\n }\n } catch (e) {\n err = e;\n } finally {\n this.#stream?.changes.cancel();\n this.#stream = undefined;\n }\n\n // When the change stream is interrupted, abort any pending transaction.\n if (watermark) {\n this.#lc.warn?.(`aborting interrupted transaction ${watermark}`);\n this.#storer.abort();\n this.#forwarder.forward([watermark, ['rollback', {tag: 'rollback'}]]);\n }\n\n await this.#state.backoff(this.#lc, err);\n }\n this.#lc.info?.('ChangeStreamer stopped');\n }\n\n async #handleControlMessage(msg: ChangeStreamControl[1]) {\n this.#lc.info?.('received control message', msg);\n const {tag} = msg;\n\n switch (tag) {\n case 'reset-required':\n await markResetRequired(this.#changeDB, this.#shard);\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n msg.message ?? 'Resync required',\n msg.errorDetails,\n );\n if (this.#autoReset) {\n this.#lc.warn?.('shutting down for auto-reset');\n await this.stop(new AutoResetSignal());\n }\n break;\n default:\n unreachable(tag);\n }\n }\n\n subscribe(ctx: SubscriberContext): Promise<Source<Downstream>> {\n const {protocolVersion, id, mode, replicaVersion, watermark, initial} = ctx;\n if (mode === 'serving') {\n this.#serving.resolve();\n }\n const downstream = Subscription.create<Downstream>({\n cleanup: () => this.#forwarder.remove(subscriber),\n });\n const subscriber = new Subscriber(\n protocolVersion,\n id,\n watermark,\n downstream,\n );\n if (replicaVersion !== this.#replicaVersion) {\n this.#lc.warn?.(\n `rejecting subscriber at replica version ${replicaVersion}`,\n );\n subscriber.close(\n ErrorType.WrongReplicaVersion,\n `current replica version is ${\n this.#replicaVersion\n } (requested ${replicaVersion})`,\n );\n } else {\n this.#lc.debug?.(`adding subscriber ${subscriber.id}`);\n\n this.#forwarder.add(subscriber);\n this.#storer.catchup(subscriber, mode);\n\n if (initial) {\n this.scheduleCleanup(watermark);\n }\n }\n return Promise.resolve(downstream);\n }\n\n scheduleCleanup(watermark: string) {\n const origSize = this.#initialWatermarks.size;\n this.#initialWatermarks.add(watermark);\n\n if (origSize === 0) {\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n\n async getChangeLogState(): Promise<{\n replicaVersion: string;\n minWatermark: string;\n }> {\n const minWatermark = await this.#storer.getMinWatermarkForCatchup();\n if (!minWatermark) {\n this.#lc.warn?.(\n `Unexpected empty changeLog. Resync if \"Local replica watermark\" errors arise`,\n );\n }\n return {\n replicaVersion: this.#replicaVersion,\n minWatermark: minWatermark ?? this.#replicaVersion,\n };\n }\n\n async #purgeOldChanges(): Promise<void> {\n const initial = [...this.#initialWatermarks];\n if (initial.length === 0) {\n this.#lc.warn?.('No initial watermarks to check for cleanup'); // Not expected.\n return;\n }\n const current = [...this.#forwarder.getAcks()];\n if (current.length === 0) {\n // Also not expected, but possible (e.g. subscriber connects, then disconnects).\n // Bail to be safe.\n this.#lc.warn?.('No subscribers to confirm cleanup');\n return;\n }\n try {\n const earliestInitial = min(...(initial as AtLeastOne<LexiVersion>));\n const earliestCurrent = min(...(current as AtLeastOne<LexiVersion>));\n if (earliestCurrent < earliestInitial) {\n this.#lc.info?.(\n `At least one client is behind backup (${earliestCurrent} < ${earliestInitial})`,\n );\n } else {\n const deleted = await this.#storer.purgeRecordsBefore(earliestInitial);\n this.#lc.info?.(`Purged ${deleted} changes before ${earliestInitial}`);\n this.#initialWatermarks.delete(earliestInitial);\n }\n } finally {\n if (this.#initialWatermarks.size) {\n // If there are unpurged watermarks to check, schedule the next purge.\n this.#state.setTimeout(() => this.#purgeOldChanges(), CLEANUP_DELAY_MS);\n }\n }\n }\n\n async stop(err?: unknown) {\n this.#state.stop(this.#lc, err);\n this.#stream?.changes.cancel();\n await this.#storer.stop();\n }\n}\n\n// The delay between receiving an initial, backup-based watermark\n// and performing a check of whether to purge records before it.\n// This delay should be long enough to handle situations like the following:\n//\n// 1. `litestream restore` downloads a backup for the `replication-manager`\n// 2. `replication-manager` starts up and runs this `change-streamer`\n// 3. `zero-cache`s that are running on a different replica connect to this\n// `change-streamer` after exponential backoff retries.\n//\n// It is possible for a `zero-cache`[3] to be behind the backup restored [1].\n// This cleanup delay (30 seconds) is thus set to be a value comfortably\n// longer than the max delay for exponential backoff (10 seconds) in\n// `services/running-state.ts`. This allows the `zero-cache` [3] to reconnect\n// so that the `change-streamer` can track its progress and know when it has\n// surpassed the initial watermark of the backup [1].\nconst CLEANUP_DELAY_MS = DEFAULT_MAX_RETRY_DELAY_MS 3;\n"],"names":["ErrorType.WrongReplicaVersion"],"mappings":";;;;;;;;;;;;;;;AA6CA,eAAsB,mBACpB,IACA,OACA,QACA,kBACA,mBACA,UACA,cACA,mBACA,WACA,eAAe,YACiB;AAEhC,QAAM,yBAAyB,IAAI,UAAU,KAAK;AAClD,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,EAAC,mBAAkB;AACzB,SAAO,IAAI;AAAA,IACT;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAEJ;AA4KA,MAAM,mBAAoD;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,EACA,yCAAyB,IAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUzB,WAAW,SAAA;AAAA,EAEX,aAAa;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAAA,EAGF;AAAA,EAEA,YACE,IACA,OACA,QACA,kBACA,mBACA,UACA,gBACA,QACA,WACA,eAAe,YACf;AACA,SAAK,KAAK;AACV,SAAK,MAAM,GAAG,YAAY,aAAa,iBAAiB;AACxD,SAAK,SAAS;AACd,SAAK,YAAY;AACjB,SAAK,kBAAkB;AACvB,SAAK,UAAU;AACf,SAAK,UAAU,IAAI;AAAA,MACjB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,CAAA,aAAY,KAAK,SAAS,KAAK,KAAK,CAAC,UAAU,SAAS,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;AAAA,MACxE,CAAA,QAAO,KAAK,KAAK,GAAG;AAAA,IAAA;AAEtB,SAAK,aAAa,IAAI,UAAA;AACtB,SAAK,aAAa;AAClB,SAAK,SAAS,IAAI,aAAa,KAAK,IAAI,QAAW,YAAY;AAAA,EACjE;AAAA,EAEA,MAAM,MAAM;AACV,SAAK,IAAI,OAAO,wBAAwB;AAIxC,UAAM,KAAK,QAAQ,gBAAA;AAGnB,SAAK,QACF,IAAA,EACA,KAAK,MAAM,KAAK,KAAA,CAAM,EACtB,MAAM,CAAA,MAAK,KAAK,KAAK,CAAC,CAAC;AAE1B,WAAO,KAAK,OAAO,aAAa;AAC9B,UAAI;AACJ,UAAI,YAA2B;AAC/B,UAAI;AACF,cAAM,aAAa,MAAM,KAAK,QAAQ,8BAAA;AACtC,cAAM,SAAS,MAAM,KAAK,QAAQ,YAAY,UAAU;AACxD,aAAK,UAAU;AACf,aAAK,OAAO,aAAA;AACZ,oBAAY;AAEZ,yBAAiB,UAAU,OAAO,SAAS;AACzC,gBAAM,CAAC,MAAM,GAAG,IAAI;AACpB,kBAAQ,MAAA;AAAA,YACN,KAAK;AACH,mBAAK,QAAQ,OAAO,MAAM;AAC1B;AAAA,YACF,KAAK;AACH,oBAAM,KAAK,sBAAsB,GAAG;AACpC;AAAA;AAAA,YACF,KAAK;AACH,0BAAY,OAAO,CAAC,EAAE;AACtB;AAAA,YACF,KAAK;AACH,kBAAI,cAAc,OAAO,CAAC,EAAE,WAAW;AACrC,sBAAM,IAAI;AAAA,kBACR,oBAAoB,OAAO,CAAC,EAAE,SAAS,qCAAqC,SAAS;AAAA,gBAAA;AAAA,cAEzF;AACA,mBAAK,WAAW,IAAI,CAAC;AACrB;AAAA,YACF;AACE,kBAAI,cAAc,MAAM;AACtB,sBAAM,IAAI;AAAA,kBACR,GAAG,IAAI,YAAY,IAAI,GAAG;AAAA,gBAAA;AAAA,cAE9B;AACA;AAAA,UAAA;AAGJ,eAAK,QAAQ,MAAM,CAAC,WAAW,MAAM,CAAC;AACtC,eAAK,WAAW,QAAQ,CAAC,WAAW,MAAM,CAAC;AAE3C,cAAI,SAAS,YAAY,SAAS,YAAY;AAC5C,wBAAY;AAAA,UACd;AAGA,gBAAM,eAAe,KAAK,QAAQ,aAAA;AAClC,cAAI,cAAc;AAChB,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR,UAAA;AACE,aAAK,SAAS,QAAQ,OAAA;AACtB,aAAK,UAAU;AAAA,MACjB;AAGA,UAAI,WAAW;AACb,aAAK,IAAI,OAAO,oCAAoC,SAAS,EAAE;AAC/D,aAAK,QAAQ,MAAA;AACb,aAAK,WAAW,QAAQ,CAAC,WAAW,CAAC,YAAY,EAAC,KAAK,WAAA,CAAW,CAAC,CAAC;AAAA,MACtE;AAEA,YAAM,KAAK,OAAO,QAAQ,KAAK,KAAK,GAAG;AAAA,IACzC;AACA,SAAK,IAAI,OAAO,wBAAwB;AAAA,EAC1C;AAAA,EAEA,MAAM,sBAAsB,KAA6B;AACvD,SAAK,IAAI,OAAO,4BAA4B,GAAG;AAC/C,UAAM,EAAC,QAAO;AAEd,YAAQ,KAAA;AAAA,MACN,KAAK;AACH,cAAM,kBAAkB,KAAK,WAAW,KAAK,MAAM;AACnD,cAAM;AAAA,UACJ,KAAK;AAAA,UACL;AAAA,UACA,IAAI,WAAW;AAAA,UACf,IAAI;AAAA,QAAA;AAEN,YAAI,KAAK,YAAY;AACnB,eAAK,IAAI,OAAO,8BAA8B;AAC9C,gBAAM,KAAK,KAAK,IAAI,iBAAiB;AAAA,QACvC;AACA;AAAA,MACF;AACE,oBAAe;AAAA,IAAA;AAAA,EAErB;AAAA,EAEA,UAAU,KAAqD;AAC7D,UAAM,EAAC,iBAAiB,IAAI,MAAM,gBAAgB,WAAW,YAAW;AACxE,QAAI,SAAS,WAAW;AACtB,WAAK,SAAS,QAAA;AAAA,IAChB;AACA,UAAM,aAAa,aAAa,OAAmB;AAAA,MACjD,SAAS,MAAM,KAAK,WAAW,OAAO,UAAU;AAAA,IAAA,CACjD;AACD,UAAM,aAAa,IAAI;AAAA,MACrB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAEF,QAAI,mBAAmB,KAAK,iBAAiB;AAC3C,WAAK,IAAI;AAAA,QACP,2CAA2C,cAAc;AAAA,MAAA;AAE3D,iBAAW;AAAA,QACTA;AAAAA,QACA,8BACE,KAAK,eACP,eAAe,cAAc;AAAA,MAAA;AAAA,IAEjC,OAAO;AACL,WAAK,IAAI,QAAQ,qBAAqB,WAAW,EAAE,EAAE;AAErD,WAAK,WAAW,IAAI,UAAU;AAC9B,WAAK,QAAQ,QAAQ,YAAY,IAAI;AAErC,UAAI,SAAS;AACX,aAAK,gBAAgB,SAAS;AAAA,MAChC;AAAA,IACF;AACA,WAAO,QAAQ,QAAQ,UAAU;AAAA,EACnC;AAAA,EAEA,gBAAgB,WAAmB;AACjC,UAAM,WAAW,KAAK,mBAAmB;AACzC,SAAK,mBAAmB,IAAI,SAAS;AAErC,QAAI,aAAa,GAAG;AAClB,WAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,MAAM,oBAGH;AACD,UAAM,eAAe,MAAM,KAAK,QAAQ,0BAAA;AACxC,QAAI,CAAC,cAAc;AACjB,WAAK,IAAI;AAAA,QACP;AAAA,MAAA;AAAA,IAEJ;AACA,WAAO;AAAA,MACL,gBAAgB,KAAK;AAAA,MACrB,cAAc,gBAAgB,KAAK;AAAA,IAAA;AAAA,EAEvC;AAAA,EAEA,MAAM,mBAAkC;AACtC,UAAM,UAAU,CAAC,GAAG,KAAK,kBAAkB;AAC3C,QAAI,QAAQ,WAAW,GAAG;AACxB,WAAK,IAAI,OAAO,4CAA4C;AAC5D;AAAA,IACF;AACA,UAAM,UAAU,CAAC,GAAG,KAAK,WAAW,SAAS;AAC7C,QAAI,QAAQ,WAAW,GAAG;AAGxB,WAAK,IAAI,OAAO,mCAAmC;AACnD;AAAA,IACF;AACA,QAAI;AACF,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,YAAM,kBAAkB,IAAI,GAAI,OAAmC;AACnE,UAAI,kBAAkB,iBAAiB;AACrC,aAAK,IAAI;AAAA,UACP,yCAAyC,eAAe,MAAM,eAAe;AAAA,QAAA;AAAA,MAEjF,OAAO;AACL,cAAM,UAAU,MAAM,KAAK,QAAQ,mBAAmB,eAAe;AACrE,aAAK,IAAI,OAAO,UAAU,OAAO,mBAAmB,eAAe,EAAE;AACrE,aAAK,mBAAmB,OAAO,eAAe;AAAA,MAChD;AAAA,IACF,UAAA;AACE,UAAI,KAAK,mBAAmB,MAAM;AAEhC,aAAK,OAAO,WAAW,MAAM,KAAK,iBAAA,GAAoB,gBAAgB;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,KAAK,KAAe;AACxB,SAAK,OAAO,KAAK,KAAK,KAAK,GAAG;AAC9B,SAAK,SAAS,QAAQ,OAAA;AACtB,UAAM,KAAK,QAAQ,KAAA;AAAA,EACrB;AACF;AAiBA,MAAM,mBAAmB,6BAA6B;"}

package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAEpE,OAAO,EAA0B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,8CAA8C,CAAC;AACzE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;~~AAiBF~~;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAsBF,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,iBAAiB,EACpC,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,~~iBAiFnB~~;AAED,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC"}
1	+ {"version":3,"file":"tables.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAEpE,OAAO,EAA0B,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AAC9E,OAAO,EAAY,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,8CAA8C,CAAC;AACzE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,8CAA8C,CAAC;AAOpF,eAAO,MAAM,SAAS,QAAQ,CAAC;AAM/B,MAAM,MAAM,cAAc,GAAG;IAG3B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAmBF;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,UASzD;AAED,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAIxB;AAED;;;;;GAKG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;CACjC,CAAC;AAsBF,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,KAAK,EAAE,OAAO,iBAIf;AAED,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,iBAKtE;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,UAAU,EACd,iBAAiB,EAAE,iBAAiB,EACpC,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,OAAO,iBAiGnB;AAED,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,qBAAqB;CACnC"}

package/out/zero-cache/src/services/change-streamer/schema/tables.js CHANGED Viewed

@@ -104,10 +104,18 @@ async function ensureReplicationConfig(lc, db, subscriptionState, shard, autoRes
       }
     }
     if (results.length === 0 || stmts.length > 0) {
+      const watermark2 = replicaConfig.replicaVersion;
+      const initialTx = [
+        { watermark: watermark2, pos: 0, change: { tag: "begin" } },
+        { watermark: watermark2, pos: 1, change: { tag: "commit" } }
+      ];
       stmts.push(
         sql`INSERT INTO ${sql(schema2)}."replicationConfig" ${sql(replicaConfig)}`,
         sql`INSERT INTO ${sql(schema2)}."replicationState"  ${sql(replicationState)}
-              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`
+              ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,
+        ...initialTx.map(
+          (change) => sql`INSERT INTO ${sql(schema2)}."changeLog" ${sql(change)}`
+        )
       );
       return Promise.all(stmts);
     }

package/out/zero-cache/src/services/change-streamer/schema/tables.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"tables.js","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {disableStatementTimeout, type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {Change} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n return /sql/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n // A strictly monotonically increasing, lexicographically sortable\n // value that uniquely identifies a position in the change stream.\n watermark: string;\n change: Change;\n};\n\nfunction createChangeLogTable(shard: ShardID) {\n // Note: The \"change\" column used to be JSONB, but that was problematic in that\n // it does not handle the NULL unicode character.\n // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n return /sql/ `\n CREATE TABLE ${schema(shard)}.\"changeLog\" (\n watermark TEXT,\n pos INT8,\n change JSON NOT NULL,\n precommit TEXT, -- Only exists on commit entries. Purely for debugging.\n PRIMARY KEY (watermark, pos)\n );\n`;\n}\n\n/*\n Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n /\nexport type ReplicationState = {\n lastWatermark: string;\n owner: string \| null;\n ownerAddress: string \| null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n return /sql/ `\n CREATE TABLE ${schema(shard)}.\"replicationState\" (\n \"lastWatermark\" TEXT NOT NULL,\n \"owner\" TEXT,\n \"ownerAddress\" TEXT,\n \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n shard: ShardID,\n sql: PostgresDB,\n): Promise<string \| null> {\n const result = await sql<{ownerAddress: string \| null}[]> /sql/ `\n SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n return result[0].ownerAddress;\n}\n\n/\n This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n /\nexport type ReplicationConfig = {\n replicaVersion: string;\n publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n return /sql/ `\n CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n \"replicaVersion\" TEXT NOT NULL,\n \"publications\" TEXT[] NOT NULL,\n \"resetRequired\" BOOL,\n \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n`;\n}\n\nfunction createTables(shard: ShardID) {\n return (\n createSchema(shard) +\n createChangeLogTable(shard) +\n createReplicationStateTable(shard) +\n createReplicationConfigTable(shard)\n );\n}\n\nexport async function setupCDCTables(\n lc: LogContext,\n db: postgres.TransactionSql,\n shard: ShardID,\n) {\n lc.info?.(`Setting up CDC tables`);\n await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n const schema = cdcSchema(shard);\n await sql`\n UPDATE ${sql(schema)}.\"replicationConfig\"\n SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n lc: LogContext,\n db: PostgresDB,\n subscriptionState: SubscriptionState,\n shard: ShardID,\n autoReset: boolean,\n) {\n const {publications, replicaVersion, watermark} = subscriptionState;\n const replicaConfig = {publications, replicaVersion};\n const replicationState: ReplicationState = {\n lastWatermark: replicaVersion,\n owner: null,\n ownerAddress: null,\n };\n const schema = cdcSchema(shard);\n\n await db.begin(async sql => {\n disableStatementTimeout(sql);\n\n const stmts: PendingQuery<Row[]>[] = [];\n const results = await sql<\n {\n replicaVersion: string;\n publications: string[];\n resetRequired: boolean \| null;\n }[]\n > /sql*/ `\n SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n FROM ${sql(schema)}.\"replicationConfig\"`;\n\n if (results.length) {\n const {replicaVersion, publications} = results[0];\n if (\n replicaVersion !== replicaConfig.replicaVersion \|\|\n !equals(new Set(publications), new Set(replicaConfig.publications))\n ) {\n if (replicaConfig.replicaVersion !== watermark) {\n throw new AutoResetSignal(\n `Cannot reset change db@${replicaVersion} to ` +\n `service replica@${replicaConfig.replicaVersion} ` +\n `from watermark ${watermark}`,\n );\n }\n lc.info?.(\n `Data in cdc tables @${replicaVersion} is incompatible ` +\n `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n );\n // Note: The replicationState table is explicitly not TRUNCATE'd.\n // Any existing row must be overwritten by an UPDATE or\n // INSERT ... ON CONFLICT clause in order to correctly abort\n // any pending transaction by a concurrently running\n // change-streamer. Deleting the existing row and creating\n // a new one, on the other hand, may not properly trigger the\n // SERIALIZATION failure necessary to abort the pending tx.\n stmts.push(\n sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n );\n }\n }\n // Initialize (or re-initialize TRUNCATED) tables\n if (results.length === 0 \|\| stmts.length > 0) {\n stmts.push(\n sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)} \n ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n );\n return Promise.all(stmts);\n }\n\n const {resetRequired} = results[0];\n if (resetRequired) {\n if (autoReset) {\n throw new AutoResetSignal('reset required by replication stream');\n }\n lc.error?.(\n '\\n\\n\\n' +\n 'Reset required but --auto-reset is not enabled.\\n' +\n 'This can happen for upstream databases that do not support event triggers.\\n' +\n 'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n '\\n\\n\\n',\n );\n }\n\n return [];\n });\n}\n\nexport class AutoResetSignal extends AbortError {\n readonly name = 'AutoResetSignal';\n}\n"],"names":["schema","replicaVersion","publications"],"mappings":";;;;;;AAYA,SAAS,OAAO,OAAgB;AAC9B,SAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;AACpC;AAAA;AAAA,IAAe,+BAA+B,OAAO,KAAK,CAAC;AAAA;AAC7D;AASA,SAAS,qBAAqB,OAAgB;AAI5C;AAAA;AAAA,IAAe;AAAA,iBACA,OAAO,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQ9B;AAYO,SAAS,4BAA4B,OAAgB;AAC1D;AAAA;AAAA,IAAe;AAAA,iBACA,OAAO,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAO9B;AAEA,eAAsB,8BACpB,OACA,KACwB;AACxB,QAAM,SAAS,MAAM;AAAA,iCACU,IAAI,UAAU,KAAK,CAAC,CAAC;AACpD,SAAO,OAAO,CAAC,EAAE;AACnB;AAaA,SAAS,6BAA6B,OAAgB;AACpD;AAAA;AAAA,IAAe;AAAA,iBACA,OAAO,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAO9B;AAEA,SAAS,aAAa,OAAgB;AACpC,SACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK;AAEtC;AAEA,eAAsB,eACpB,IACA,IACA,OACA;AACA,KAAG,OAAO,uBAAuB;AACjC,QAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;AACvE,QAAMA,UAAS,UAAU,KAAK;AAC9B,QAAM;AAAA,WACG,IAAIA,OAAM,CAAC;AAAA;AAEtB;AAEA,eAAsB,wBACpB,IACA,IACA,mBACA,OACA,WACA;AACA,QAAM,EAAC,cAAc,gBAAgB,UAAA,IAAa;AAClD,QAAM,gBAAgB,EAAC,cAAc,eAAA;AACrC,QAAM,mBAAqC;AAAA,IACzC,eAAe;AAAA,IACf,OAAO;AAAA,IACP,cAAc;AAAA,EAAA;AAEhB,QAAMA,UAAS,UAAU,KAAK;AAE9B,QAAM,GAAG,MAAM,OAAM,QAAO;AAC1B,4BAAwB,GAAG;AAE3B,UAAM,QAA+B,CAAA;AACrC,UAAM,UAAU,MAAM;AAAA;AAAA,aAQb,IAAIA,OAAM,CAAC;AAEpB,QAAI,QAAQ,QAAQ;AAClB,YAAM,EAAC,gBAAAC,iBAAgB,cAAAC,cAAAA,IAAgB,QAAQ,CAAC;AAChD,UACED,oBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAIC,aAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;AACA,YAAI,cAAc,mBAAmB,WAAW;AAC9C,gBAAM,IAAI;AAAA,YACR,0BAA0BD,eAAc,uBACnB,cAAc,cAAc,mBAC7B,SAAS;AAAA,UAAA;AAAA,QAEjC;AACA,WAAG;AAAA,UACD,uBAAuBA,eAAc,kCAClB,cAAc,cAAc;AAAA,QAAA;AASjD,cAAM;AAAA,UACJ,qBAAqB,IAAID,OAAM,CAAC;AAAA,UAChC,qBAAqB,IAAIA,OAAM,CAAC;AAAA,QAAA;AAAA,MAEpC;AAAA,IACF;AAEA,QAAI,QAAQ,WAAW,KAAK,MAAM,SAAS,GAAG;AAC5C,YAAM;AAAA,QACJ,kBAAkB,IAAIA,OAAM,CAAC,wBAAwB,IAAI,aAAa,CAAC;AAAA,QACvE,kBAAkB,IAAIA,OAAM,CAAC,wBAAwB,IAAI,gBAAgB,CAAC;AAAA,iDACjC,IAAI,gBAAgB,CAAC;AAAA,MAAA;AAEhE,aAAO,QAAQ,IAAI,KAAK;AAAA,IAC1B;AAEA,UAAM,EAAC,cAAA,IAAiB,QAAQ,CAAC;AACjC,QAAI,eAAe;AACjB,UAAI,WAAW;AACb,cAAM,IAAI,gBAAgB,sCAAsC;AAAA,MAClE;AACA,SAAG;AAAA,QACD;AAAA,MAAA;AAAA,IAMJ;AAEA,WAAO,CAAA;AAAA,EACT,CAAC;AACH;AAEO,MAAM,wBAAwB,WAAW;AAAA,EACrC,OAAO;AAClB;"}
1	+ {"version":3,"file":"tables.js","sources":["../../../../../../../zero-cache/src/services/change-streamer/schema/tables.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {ident} from 'pg-format';\nimport type postgres from 'postgres';\nimport {type PendingQuery, type Row} from 'postgres';\nimport {AbortError} from '../../../../../shared/src/abort-error.ts';\nimport {equals} from '../../../../../shared/src/set-utils.ts';\nimport {disableStatementTimeout, type PostgresDB} from '../../../types/pg.ts';\nimport {cdcSchema, type ShardID} from '../../../types/shards.ts';\nimport type {Change} from '../../change-source/protocol/current/data.ts';\nimport type {SubscriptionState} from '../../replicator/schema/replication-state.ts';\n\n// For readability in the sql statements.\nfunction schema(shard: ShardID) {\n return ident(cdcSchema(shard));\n}\n\nexport const PG_SCHEMA = 'cdc';\n\nfunction createSchema(shard: ShardID) {\n return /sql/ `CREATE SCHEMA IF NOT EXISTS ${schema(shard)};`;\n}\n\nexport type ChangeLogEntry = {\n // A strictly monotonically increasing, lexicographically sortable\n // value that uniquely identifies a position in the change stream.\n watermark: string;\n change: Change;\n};\n\ntype FullChangeLogEntry = ChangeLogEntry & {pos: number};\n\nfunction createChangeLogTable(shard: ShardID) {\n // Note: The \"change\" column used to be JSONB, but that was problematic in that\n // it does not handle the NULL unicode character.\n // https://vladimir.varank.in/notes/2021/01/you-dont-insert-unicode-null-character-as-postgres-jsonb/\n return /sql/ `\n CREATE TABLE ${schema(shard)}.\"changeLog\" (\n watermark TEXT,\n pos INT8,\n change JSON NOT NULL,\n precommit TEXT, -- Only exists on commit entries. Purely for debugging.\n PRIMARY KEY (watermark, pos)\n );\n`;\n}\n\n/*\n Tracks the watermark from which to resume the change stream and the\n * current owner (task ID) acting as the single writer to the changeLog.\n /\nexport type ReplicationState = {\n lastWatermark: string;\n owner: string \| null;\n ownerAddress: string \| null;\n};\n\nexport function createReplicationStateTable(shard: ShardID) {\n return /sql/ `\n CREATE TABLE ${schema(shard)}.\"replicationState\" (\n \"lastWatermark\" TEXT NOT NULL,\n \"owner\" TEXT,\n \"ownerAddress\" TEXT,\n \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n`;\n}\n\nexport async function discoverChangeStreamerAddress(\n shard: ShardID,\n sql: PostgresDB,\n): Promise<string \| null> {\n const result = await sql<{ownerAddress: string \| null}[]> /sql/ `\n SELECT \"ownerAddress\" FROM ${sql(cdcSchema(shard))}.\"replicationState\"`;\n return result[0].ownerAddress;\n}\n\n/\n This mirrors the analogously named table in the SQLite replica\n * (`services/replicator/schema/replication-state.ts`), and is used\n * to detect when the replica has been reset and is no longer compatible\n * with the current ChangeLog.\n /\nexport type ReplicationConfig = {\n replicaVersion: string;\n publications: readonly string[];\n};\n\nfunction createReplicationConfigTable(shard: ShardID) {\n return /sql/ `\n CREATE TABLE ${schema(shard)}.\"replicationConfig\" (\n \"replicaVersion\" TEXT NOT NULL,\n \"publications\" TEXT[] NOT NULL,\n \"resetRequired\" BOOL,\n \"lock\" INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n`;\n}\n\nfunction createTables(shard: ShardID) {\n return (\n createSchema(shard) +\n createChangeLogTable(shard) +\n createReplicationStateTable(shard) +\n createReplicationConfigTable(shard)\n );\n}\n\nexport async function setupCDCTables(\n lc: LogContext,\n db: postgres.TransactionSql,\n shard: ShardID,\n) {\n lc.info?.(`Setting up CDC tables`);\n await db.unsafe(createTables(shard));\n}\n\nexport async function markResetRequired(sql: PostgresDB, shard: ShardID) {\n const schema = cdcSchema(shard);\n await sql`\n UPDATE ${sql(schema)}.\"replicationConfig\"\n SET \"resetRequired\" = true`;\n}\n\nexport async function ensureReplicationConfig(\n lc: LogContext,\n db: PostgresDB,\n subscriptionState: SubscriptionState,\n shard: ShardID,\n autoReset: boolean,\n) {\n const {publications, replicaVersion, watermark} = subscriptionState;\n const replicaConfig = {publications, replicaVersion};\n const replicationState: ReplicationState = {\n lastWatermark: replicaVersion,\n owner: null,\n ownerAddress: null,\n };\n const schema = cdcSchema(shard);\n\n await db.begin(async sql => {\n disableStatementTimeout(sql);\n\n const stmts: PendingQuery<Row[]>[] = [];\n const results = await sql<\n {\n replicaVersion: string;\n publications: string[];\n resetRequired: boolean \| null;\n }[]\n > /sql*/ `\n SELECT \"replicaVersion\", \"publications\", \"resetRequired\" \n FROM ${sql(schema)}.\"replicationConfig\"`;\n\n if (results.length) {\n const {replicaVersion, publications} = results[0];\n if (\n replicaVersion !== replicaConfig.replicaVersion \|\|\n !equals(new Set(publications), new Set(replicaConfig.publications))\n ) {\n if (replicaConfig.replicaVersion !== watermark) {\n throw new AutoResetSignal(\n `Cannot reset change db@${replicaVersion} to ` +\n `service replica@${replicaConfig.replicaVersion} ` +\n `from watermark ${watermark}`,\n );\n }\n lc.info?.(\n `Data in cdc tables @${replicaVersion} is incompatible ` +\n `with replica @${replicaConfig.replicaVersion}. Clearing tables.`,\n );\n // Note: The replicationState table is explicitly not TRUNCATE'd.\n // Any existing row must be overwritten by an UPDATE or\n // INSERT ... ON CONFLICT clause in order to correctly abort\n // any pending transaction by a concurrently running\n // change-streamer. Deleting the existing row and creating\n // a new one, on the other hand, may not properly trigger the\n // SERIALIZATION failure necessary to abort the pending tx.\n stmts.push(\n sql`TRUNCATE TABLE ${sql(schema)}.\"changeLog\"`,\n sql`TRUNCATE TABLE ${sql(schema)}.\"replicationConfig\"`,\n );\n }\n }\n // Initialize (or re-initialize TRUNCATED) tables\n if (results.length === 0 \|\| stmts.length > 0) {\n // The storer uses the earliest changeLog entry as the safe watermark\n // from which subscribers can be resumed. These initial entries ensure\n // that subscribers can start from a freshly synced replica, even if\n // new changes have been replicated and not purged from the changeLog.\n //\n // TODO: Replace this with an explicit `firstWatermark` column in the\n // change db.\n const watermark = replicaConfig.replicaVersion;\n const initialTx: FullChangeLogEntry[] = [\n {watermark, pos: 0, change: {tag: 'begin'}},\n {watermark, pos: 1, change: {tag: 'commit'}},\n ];\n\n stmts.push(\n sql`INSERT INTO ${sql(schema)}.\"replicationConfig\" ${sql(replicaConfig)}`,\n sql`INSERT INTO ${sql(schema)}.\"replicationState\" ${sql(replicationState)} \n ON CONFLICT (lock) DO UPDATE SET ${sql(replicationState)}`,\n ...initialTx.map(\n change => sql`INSERT INTO ${sql(schema)}.\"changeLog\" ${sql(change)}`,\n ),\n );\n return Promise.all(stmts);\n }\n\n const {resetRequired} = results[0];\n if (resetRequired) {\n if (autoReset) {\n throw new AutoResetSignal('reset required by replication stream');\n }\n lc.error?.(\n '\\n\\n\\n' +\n 'Reset required but --auto-reset is not enabled.\\n' +\n 'This can happen for upstream databases that do not support event triggers.\\n' +\n 'To correct this, see https://zero.rocicorp.dev/docs/connecting-to-postgres#schema-changes' +\n '\\n\\n\\n',\n );\n }\n\n return [];\n });\n}\n\nexport class AutoResetSignal extends AbortError {\n readonly name = 'AutoResetSignal';\n}\n"],"names":["schema","replicaVersion","publications","watermark"],"mappings":";;;;;;AAYA,SAAS,OAAO,OAAgB;AAC9B,SAAO,MAAM,UAAU,KAAK,CAAC;AAC/B;AAIA,SAAS,aAAa,OAAgB;AACpC;AAAA;AAAA,IAAe,+BAA+B,OAAO,KAAK,CAAC;AAAA;AAC7D;AAWA,SAAS,qBAAqB,OAAgB;AAI5C;AAAA;AAAA,IAAe;AAAA,iBACA,OAAO,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQ9B;AAYO,SAAS,4BAA4B,OAAgB;AAC1D;AAAA;AAAA,IAAe;AAAA,iBACA,OAAO,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAO9B;AAEA,eAAsB,8BACpB,OACA,KACwB;AACxB,QAAM,SAAS,MAAM;AAAA,iCACU,IAAI,UAAU,KAAK,CAAC,CAAC;AACpD,SAAO,OAAO,CAAC,EAAE;AACnB;AAaA,SAAS,6BAA6B,OAAgB;AACpD;AAAA;AAAA,IAAe;AAAA,iBACA,OAAO,KAAK,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAO9B;AAEA,SAAS,aAAa,OAAgB;AACpC,SACE,aAAa,KAAK,IAClB,qBAAqB,KAAK,IAC1B,4BAA4B,KAAK,IACjC,6BAA6B,KAAK;AAEtC;AAEA,eAAsB,eACpB,IACA,IACA,OACA;AACA,KAAG,OAAO,uBAAuB;AACjC,QAAM,GAAG,OAAO,aAAa,KAAK,CAAC;AACrC;AAEA,eAAsB,kBAAkB,KAAiB,OAAgB;AACvE,QAAMA,UAAS,UAAU,KAAK;AAC9B,QAAM;AAAA,WACG,IAAIA,OAAM,CAAC;AAAA;AAEtB;AAEA,eAAsB,wBACpB,IACA,IACA,mBACA,OACA,WACA;AACA,QAAM,EAAC,cAAc,gBAAgB,UAAA,IAAa;AAClD,QAAM,gBAAgB,EAAC,cAAc,eAAA;AACrC,QAAM,mBAAqC;AAAA,IACzC,eAAe;AAAA,IACf,OAAO;AAAA,IACP,cAAc;AAAA,EAAA;AAEhB,QAAMA,UAAS,UAAU,KAAK;AAE9B,QAAM,GAAG,MAAM,OAAM,QAAO;AAC1B,4BAAwB,GAAG;AAE3B,UAAM,QAA+B,CAAA;AACrC,UAAM,UAAU,MAAM;AAAA;AAAA,aAQb,IAAIA,OAAM,CAAC;AAEpB,QAAI,QAAQ,QAAQ;AAClB,YAAM,EAAC,gBAAAC,iBAAgB,cAAAC,cAAAA,IAAgB,QAAQ,CAAC;AAChD,UACED,oBAAmB,cAAc,kBACjC,CAAC,OAAO,IAAI,IAAIC,aAAY,GAAG,IAAI,IAAI,cAAc,YAAY,CAAC,GAClE;AACA,YAAI,cAAc,mBAAmB,WAAW;AAC9C,gBAAM,IAAI;AAAA,YACR,0BAA0BD,eAAc,uBACnB,cAAc,cAAc,mBAC7B,SAAS;AAAA,UAAA;AAAA,QAEjC;AACA,WAAG;AAAA,UACD,uBAAuBA,eAAc,kCAClB,cAAc,cAAc;AAAA,QAAA;AASjD,cAAM;AAAA,UACJ,qBAAqB,IAAID,OAAM,CAAC;AAAA,UAChC,qBAAqB,IAAIA,OAAM,CAAC;AAAA,QAAA;AAAA,MAEpC;AAAA,IACF;AAEA,QAAI,QAAQ,WAAW,KAAK,MAAM,SAAS,GAAG;AAQ5C,YAAMG,aAAY,cAAc;AAChC,YAAM,YAAkC;AAAA,QACtC,EAAC,WAAAA,YAAW,KAAK,GAAG,QAAQ,EAAC,KAAK,UAAO;AAAA,QACzC,EAAC,WAAAA,YAAW,KAAK,GAAG,QAAQ,EAAC,KAAK,SAAA,EAAQ;AAAA,MAAC;AAG7C,YAAM;AAAA,QACJ,kBAAkB,IAAIH,OAAM,CAAC,wBAAwB,IAAI,aAAa,CAAC;AAAA,QACvE,kBAAkB,IAAIA,OAAM,CAAC,wBAAwB,IAAI,gBAAgB,CAAC;AAAA,iDACjC,IAAI,gBAAgB,CAAC;AAAA,QAC9D,GAAG,UAAU;AAAA,UACX,CAAA,WAAU,kBAAkB,IAAIA,OAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC;AAAA,QAAA;AAAA,MACpE;AAEF,aAAO,QAAQ,IAAI,KAAK;AAAA,IAC1B;AAEA,UAAM,EAAC,cAAA,IAAiB,QAAQ,CAAC;AACjC,QAAI,eAAe;AACjB,UAAI,WAAW;AACb,cAAM,IAAI,gBAAgB,sCAAsC;AAAA,MAClE;AACA,SAAG;AAAA,QACD;AAAA,MAAA;AAAA,IAMJ;AAEA,WAAO,CAAA;AAAA,EACT,CAAC;AACH;AAEO,MAAM,wBAAwB,WAAW;AAAA,EACrC,OAAO;AAClB;"}

package/out/zero-cache/src/services/litestream/commands.d.ts CHANGED Viewed

@@ -6,5 +6,5 @@ import type { ZeroConfig } from '../../config/zero-config.ts';
  *          (i.e. not counting failed attempts).
  */
 export declare function restoreReplica(lc: LogContext, config: ZeroConfig): Promise<Date>;
-export declare function startReplicaBackupProcess(config: ZeroConfig): ChildProcess;
+export declare function startReplicaBackupProcess(lc: LogContext, config: ZeroConfig): ChildProcess;
 //# sourceMappingURL=commands.d.ts.map

package/out/zero-cache/src/services/litestream/commands.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AA8JD,wBAAgB,yBAAyB,~~CAAC~~,MAAM,EAAE,UAAU,~~GAAG~~,YAAY,~~CAO1E~~"}
1	+ {"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAE3D,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAOrD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAiB5D;;;GAGG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,OAAO,CAAC,IAAI,CAAC,CAwBf;AA8JD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,UAAU,GACjB,YAAY,CAQd"}

package/out/zero-cache/src/services/litestream/commands.js CHANGED Viewed

@@ -159,8 +159,9 @@ function replicaIsValid(lc, replica, snapshot) {
     db.close();
   }
 }
-function startReplicaBackupProcess(config) {
+function startReplicaBackupProcess(lc, config) {
   const { litestream, env } = getLitestream(config);
+  lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);
   return spawn(litestream, ["replicate"], {
     env,
     stdio: "inherit",

package/out/zero-cache/src/services/litestream/commands.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"commands.js","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/*\n @returns The time at which the last restore started\n * (i.e. not counting failed attempts).\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const {changeStreamer} = config;\n\n for (let i = 0; i < MAX_RETRIES; i++) {\n if (i > 0) {\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n const start = new Date();\n const restored = await tryRestore(lc, config);\n if (restored) {\n return start;\n }\n if (\n changeStreamer.mode === 'dedicated' &&\n changeStreamer.uri === undefined\n ) {\n lc.info?.('no litestream backup found');\n return start;\n }\n }\n throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n // Set the snapshot interval to something smaller than x hours so that\n // the hourly check triggers on the hour, rather than the hour after.\n const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n return {\n litestream: must(executable, `Missing --litestream-executable`),\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n snapshotBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n },\n };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n const {changeStreamer} = config;\n\n const isViewSyncer =\n changeStreamer.mode === 'discover' \|\| changeStreamer.uri !== undefined;\n\n // Fire off a snapshot reservation to the current replication-manager\n // (if there is one).\n const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n let snapshotStatus: SnapshotStatus \| undefined;\n if (isViewSyncer) {\n // The return value is required by view-syncers ...\n snapshotStatus = await firstMessage;\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n } else {\n // but it is also useful to pause change-log cleanup when a new\n // replication-manager is starting up. In this case, the request is\n // best-effort. In particular, there may not be a previous\n // replication-manager running at all.\n void firstMessage.catch(e => lc.debug?.(e));\n }\n\n const {litestream, env} = getLitestream(\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {restoreParallelism: parallelism} = config.litestream;\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: 'inherit', windowsHide: true},\n );\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n await promise;\n if (!existsSync(config.replica.file)) {\n return false;\n }\n if (\n snapshotStatus &&\n !replicaIsValid(lc, config.replica.file, snapshotStatus)\n ) {\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return false;\n }\n return true;\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n snapshot: SnapshotStatus,\n) {\n const db = new Database(lc, replica);\n try {\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== snapshot.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n snapshot,\n );\n return false;\n }\n if (watermark < snapshot.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n snapshot,\n );\n return true;\n } finally {\n db.close();\n }\n}\n\nexport function startReplicaBackupProcess(config: ZeroConfig): ChildProcess {\n const {litestream, env} = getLitestream(config);\n return spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown \| string = '';\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // (i.e. BackupMonitor) returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n if (!isViewSyncer) {\n return reject(err);\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAuBA,MAAM,cAAc;AACpB,MAAM,oBAAoB;AAM1B,eAAsB,eACpB,IACA,QACe;AACf,QAAM,EAAC,mBAAkB;AAEzB,WAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,QAAI,IAAI,GAAG;AACT,SAAG;AAAA,QACD,kCAAkC,oBAAoB,GAAI;AAAA,MAAA;AAE5D,YAAM,MAAM,iBAAiB;AAAA,IAC/B;AACA,UAAM,4BAAY,KAAA;AAClB,UAAM,WAAW,MAAM,WAAW,IAAI,MAAM;AAC5C,QAAI,UAAU;AACZ,aAAO;AAAA,IACT;AACA,QACE,eAAe,SAAS,eACxB,eAAe,QAAQ,QACvB;AACA,SAAG,OAAO,4BAA4B;AACtC,aAAO;AAAA,IACT;AAAA,EACF;AACA,QAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,QACA,kBACA,mBAIA;AACA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,OAAO,OAAO;AAAA,IACrB;AAAA,IACA,yBAAyB,wBAAwB;AAAA;AAAA,IACjD,yBAAyB,yBAAyB;AAAA,IAClD;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA,IACE,OAAO;AAIX,QAAM,gCAAgC,8BAA8B,KAAK;AAEzE,SAAO;AAAA,IACL,YAAY,KAAK,YAAY,iCAAiC;AAAA,IAC9D,KAAK;AAAA,MACH,GAAG,QAAQ;AAAA,MACX,CAAC,mBAAmB,GAAG,OAAO,QAAQ;AAAA,MACtC,CAAC,4BAA4B,GAAG,KAAK,qBAAqB,SAAS;AAAA,MACnE,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,qDAAqD,GAAG;AAAA,QACvD;AAAA,MAAA;AAAA,MAEF,CAAC,2BAA2B,GAAG,oBAAoB;AAAA,MACnD,CAAC,kDAAkD,GAAG;AAAA,QACpD;AAAA,MAAA;AAAA,MAEF,CAAC,uCAAuC,GAAG,OAAO,oBAAoB;AAAA,MACtE,CAAC,gCAAgC,GAAG,OAAO,aAAa;AAAA,MACxD,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,CAAC,mBAAmB,GAAG;AAAA,MACvB,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,GAAI,WAAW,EAAC,CAAC,0BAA0B,GAAG,SAAA,IAAY,CAAA;AAAA,IAAC;AAAA,EAC7D;AAEJ;AAEA,eAAe,WAAW,IAAgB,QAAoB;AAC5D,QAAM,EAAC,mBAAkB;AAEzB,QAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ;AAI/D,QAAM,eAAe,4BAA4B,IAAI,QAAQ,YAAY;AACzE,MAAI;AACJ,MAAI,cAAc;AAEhB,qBAAiB,MAAM;AACvB,OAAG,OAAO,yBAAyB,eAAe,SAAS,EAAE;AAAA,EAC/D,OAAO;AAKL,SAAK,aAAa,MAAM,CAAA,MAAK,GAAG,QAAQ,CAAC,CAAC;AAAA,EAC5C;AAEA,QAAM,EAAC,YAAY,IAAA,IAAO;AAAA,IACxB;AAAA,IACA;AAAA;AAAA,IACA,gBAAgB;AAAA,EAAA;AAElB,QAAM,EAAC,oBAAoB,YAAA,IAAe,OAAO;AACjD,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,OAAO,QAAQ;AAAA,IAAA;AAAA,IAEjB,EAAC,KAAK,OAAO,WAAW,aAAa,KAAA;AAAA,EAAI;AAE3C,QAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,OAAK,GAAG,SAAS,MAAM;AACvB,OAAK,GAAG,SAAS,CAAC,MAAM,WAAW;AACjC,QAAI,QAAQ;AACV,aAAO,0BAA0B,MAAM,EAAE;AAAA,IAC3C,WAAW,SAAS,GAAG;AACrB,aAAO,+BAA+B,IAAI,EAAE;AAAA,IAC9C,OAAO;AACL,cAAA;AAAA,IACF;AAAA,EACF,CAAC;AACD,QAAM;AACN,MAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;AACpC,WAAO;AAAA,EACT;AACA,MACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,cAAc,GACvD;AACA,OAAG,OAAO,6CAA6C;AACvD,iBAAa,OAAO,QAAQ,IAAI;AAChC,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,UACA;AACA,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO;AACnC,MAAI;AACF,UAAM,EAAC,gBAAgB,UAAA,IAAa;AAAA,MAClC,IAAI,gBAAgB,EAAE;AAAA,IAAA;AAExB,QAAI,mBAAmB,SAAS,gBAAgB;AAC9C,SAAG;AAAA,QACD,yBAAyB,cAAc,kDAAkD,SAAS,cAAc;AAAA,QAChH;AAAA,MAAA;AAEF,aAAO;AAAA,IACT;AACA,QAAI,YAAY,SAAS,cAAc;AACrC,SAAG;AAAA,QACD,2BAA2B,SAAS,iDAAiD,SAAS,YAAY;AAAA,MAAA;AAE5G,aAAO;AAAA,IACT;AACA,OAAG;AAAA,MACD,4BAA4B,cAAc,kBAAkB,SAAS;AAAA,MACrE;AAAA,IAAA;AAEF,WAAO;AAAA,EACT,UAAA;AACE,OAAG,MAAA;AAAA,EACL;AACF;AAEO,SAAS,0BAA0B,QAAkC;AAC1E,QAAM,EAAC,YAAY,QAAO,cAAc,MAAM;AAC9C,SAAO,MAAM,YAAY,CAAC,WAAW,GAAG;AAAA,IACtC;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,EAAA,CACd;AACH;AAEA,SAAS,4BACP,IACA,QACA,cACyB;AACzB,QAAM,EAAC,SAAS,QAAQ,SAAS,OAAA,IAAU,SAAA;AAE3C,QAAM,iBAAkB;AACtB,UAAM,QAAQ,IAAI,gBAAA;AAClB,YAAQ,GAAG,UAAU,MAAM,MAAM,OAAO;AACxC,YAAQ,GAAG,WAAW,MAAM,MAAM,OAAO;AAEzC,aAAS,IAAI,KAAK,KAAK;AACrB,UAAI,MAAwB;AAC5B,UAAI;AACF,YAAI,WAAW;AACf,cAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;AAC/C,yBAAiB,OAAO,QAAQ;AAI9B,kBAAQ,IAAI,CAAC,CAAC;AACd,qBAAW;AAAA,QACb;AAGA,YAAI,UAAU;AACZ;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR;AACA,UAAI,CAAC,cAAc;AACjB,eAAO,OAAO,GAAG;AAAA,MACnB;AASA,SAAG;AAAA,QACD,uCAAuC,IAAI,CAAC;AAAA,QAC5C,OAAO,GAAG;AAAA,MAAA;AAEZ,UAAI;AACF,cAAM,MAAM,KAAM,MAAM,MAAM;AAAA,MAChC,SAAS,GAAG;AACV,eAAO,OAAO,CAAC;AAAA,MACjB;AAAA,IACF;AAAA,EACF,GAAA;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;AAClC,mBAAiB,MAAM;AACvB,QAAM,EAAC,QAAQ,QAAQ,eAAA,IAAkB;AACzC,QAAM,UAAU,eAAe,MAAM;AAErC,QAAM,uBAAuB,IAAI;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,EAAA;AAGjB,SAAO,qBAAqB,gBAAgB,MAAM;AACpD;"}
1	+ {"version":3,"file":"commands.js","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync} from 'node:fs';\nimport {must} from '../../../../shared/src/must.ts';\nimport {sleep} from '../../../../shared/src/sleep.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {assertNormalized} from '../../config/normalize.ts';\nimport type {ZeroConfig} from '../../config/zero-config.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {getShardConfig} from '../../types/shards.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {ChangeStreamerHttpClient} from '../change-streamer/change-streamer-http.ts';\nimport type {\n SnapshotMessage,\n SnapshotStatus,\n} from '../change-streamer/snapshot.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\n\n// Retry for up to 3 minutes (60 times with 3 second delay).\n// Beyond that, let the container runner restart the task.\nconst MAX_RETRIES = 60;\nconst RETRY_INTERVAL_MS = 3000;\n\n/*\n @returns The time at which the last restore started\n * (i.e. not counting failed attempts).\n /\nexport async function restoreReplica(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Date> {\n const {changeStreamer} = config;\n\n for (let i = 0; i < MAX_RETRIES; i++) {\n if (i > 0) {\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n const start = new Date();\n const restored = await tryRestore(lc, config);\n if (restored) {\n return start;\n }\n if (\n changeStreamer.mode === 'dedicated' &&\n changeStreamer.uri === undefined\n ) {\n lc.info?.('no litestream backup found');\n return start;\n }\n }\n throw new Error(`max attempts exceeded restoring replica`);\n}\n\nfunction getLitestream(\n config: ZeroConfig,\n logLevelOverride?: LogLevel,\n backupURLOverride?: string,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n backupURL,\n logLevel,\n configPath,\n endpoint,\n port = config.port + 2,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config.litestream;\n\n // Set the snapshot interval to something smaller than x hours so that\n // the hourly check triggers on the hour, rather than the hour after.\n const snapshotBackupIntervalMinutes = snapshotBackupIntervalHours * 60 - 5;\n\n return {\n litestream: must(executable, `Missing --litestream-executable`),\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: config.replica.file,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURLOverride ?? backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_MINUTES']: String(\n snapshotBackupIntervalMinutes,\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: config.log.format,\n ['LITESTREAM_CONFIG']: configPath,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n },\n };\n}\n\nasync function tryRestore(lc: LogContext, config: ZeroConfig) {\n const {changeStreamer} = config;\n\n const isViewSyncer =\n changeStreamer.mode === 'discover' \|\| changeStreamer.uri !== undefined;\n\n // Fire off a snapshot reservation to the current replication-manager\n // (if there is one).\n const firstMessage = reserveAndGetSnapshotStatus(lc, config, isViewSyncer);\n let snapshotStatus: SnapshotStatus \| undefined;\n if (isViewSyncer) {\n // The return value is required by view-syncers ...\n snapshotStatus = await firstMessage;\n lc.info?.(`restoring backup from ${snapshotStatus.backupURL}`);\n } else {\n // but it is also useful to pause change-log cleanup when a new\n // replication-manager is starting up. In this case, the request is\n // best-effort. In particular, there may not be a previous\n // replication-manager running at all.\n void firstMessage.catch(e => lc.debug?.(e));\n }\n\n const {litestream, env} = getLitestream(\n config,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n snapshotStatus?.backupURL,\n );\n const {restoreParallelism: parallelism} = config.litestream;\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n config.replica.file,\n ],\n {env, stdio: 'inherit', windowsHide: true},\n );\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n await promise;\n if (!existsSync(config.replica.file)) {\n return false;\n }\n if (\n snapshotStatus &&\n !replicaIsValid(lc, config.replica.file, snapshotStatus)\n ) {\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(config.replica.file);\n return false;\n }\n return true;\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n snapshot: SnapshotStatus,\n) {\n const db = new Database(lc, replica);\n try {\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (replicaVersion !== snapshot.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match change-streamer replicaVersion ${snapshot.replicaVersion}`,\n snapshot,\n );\n return false;\n }\n if (watermark < snapshot.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than change-streamer minWatermark ${snapshot.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible with change-streamer`,\n snapshot,\n );\n return true;\n } finally {\n db.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: ZeroConfig,\n): ChildProcess {\n const {litestream, env} = getLitestream(config);\n lc.info?.(`starting litestream backup to ${config.litestream.backupURL}`);\n return spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n}\n\nfunction reserveAndGetSnapshotStatus(\n lc: LogContext,\n config: ZeroConfig,\n isViewSyncer: boolean,\n): Promise<SnapshotStatus> {\n const {promise: status, resolve, reject} = resolver<SnapshotStatus>();\n\n void (async function () {\n const abort = new AbortController();\n process.on('SIGINT', () => abort.abort());\n process.on('SIGTERM', () => abort.abort());\n\n for (let i = 0; ; i++) {\n let err: unknown \| string = '';\n try {\n let resolved = false;\n const stream = await reserveSnapshot(lc, config);\n for await (const msg of stream) {\n // Capture the value of the status message that the change-streamer\n // (i.e. BackupMonitor) returns, and hold the connection open to\n // \"reserve\" the snapshot and prevent change log cleanup.\n resolve(msg[1]);\n resolved = true;\n }\n // The change-streamer itself closes the connection when the\n // subscription is started (or the reservation retried).\n if (resolved) {\n break;\n }\n } catch (e) {\n err = e;\n }\n if (!isViewSyncer) {\n return reject(err);\n }\n // Retry in the view-syncer since it cannot proceed until it connects\n // to a (compatible) replication-manager. In particular, a\n // replication-manager that does not support the view-syncer's\n // change-streamer protocol will close the stream with an error; this\n // retry logic essentially delays the startup of a view-syncer until\n // a compatible replication-manager has been rolled out, allowing\n // replication-manager and view-syncer services to be updated in\n // parallel.\n lc.warn?.(\n `Unable to reserve snapshot (attempt ${i + 1}). Retrying in 5 seconds.`,\n String(err),\n );\n try {\n await sleep(5000, abort.signal);\n } catch (e) {\n return reject(e);\n }\n }\n })();\n\n return status;\n}\n\nfunction reserveSnapshot(\n lc: LogContext,\n config: ZeroConfig,\n): Promise<Source<SnapshotMessage>> {\n assertNormalized(config);\n const {taskID, change, changeStreamer} = config;\n const shardID = getShardConfig(config);\n\n const changeStreamerClient = new ChangeStreamerHttpClient(\n lc,\n shardID,\n change.db,\n changeStreamer.uri,\n );\n\n return changeStreamerClient.reserveSnapshot(taskID);\n}\n"],"names":[],"mappings":";;;;;;;;;;;;AAuBA,MAAM,cAAc;AACpB,MAAM,oBAAoB;AAM1B,eAAsB,eACpB,IACA,QACe;AACf,QAAM,EAAC,mBAAkB;AAEzB,WAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,QAAI,IAAI,GAAG;AACT,SAAG;AAAA,QACD,kCAAkC,oBAAoB,GAAI;AAAA,MAAA;AAE5D,YAAM,MAAM,iBAAiB;AAAA,IAC/B;AACA,UAAM,4BAAY,KAAA;AAClB,UAAM,WAAW,MAAM,WAAW,IAAI,MAAM;AAC5C,QAAI,UAAU;AACZ,aAAO;AAAA,IACT;AACA,QACE,eAAe,SAAS,eACxB,eAAe,QAAQ,QACvB;AACA,SAAG,OAAO,4BAA4B;AACtC,aAAO;AAAA,IACT;AAAA,EACF;AACA,QAAM,IAAI,MAAM,yCAAyC;AAC3D;AAEA,SAAS,cACP,QACA,kBACA,mBAIA;AACA,QAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,OAAO,OAAO,OAAO;AAAA,IACrB;AAAA,IACA,yBAAyB,wBAAwB;AAAA;AAAA,IACjD,yBAAyB,yBAAyB;AAAA,IAClD;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA,IACE,OAAO;AAIX,QAAM,gCAAgC,8BAA8B,KAAK;AAEzE,SAAO;AAAA,IACL,YAAY,KAAK,YAAY,iCAAiC;AAAA,IAC9D,KAAK;AAAA,MACH,GAAG,QAAQ;AAAA,MACX,CAAC,mBAAmB,GAAG,OAAO,QAAQ;AAAA,MACtC,CAAC,4BAA4B,GAAG,KAAK,qBAAqB,SAAS;AAAA,MACnE,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,2CAA2C,GAAG;AAAA,QAC7C;AAAA,MAAA;AAAA,MAEF,CAAC,qDAAqD,GAAG;AAAA,QACvD;AAAA,MAAA;AAAA,MAEF,CAAC,2BAA2B,GAAG,oBAAoB;AAAA,MACnD,CAAC,kDAAkD,GAAG;AAAA,QACpD;AAAA,MAAA;AAAA,MAEF,CAAC,uCAAuC,GAAG,OAAO,oBAAoB;AAAA,MACtE,CAAC,gCAAgC,GAAG,OAAO,aAAa;AAAA,MACxD,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,CAAC,mBAAmB,GAAG;AAAA,MACvB,CAAC,iBAAiB,GAAG,OAAO,IAAI;AAAA,MAChC,GAAI,WAAW,EAAC,CAAC,0BAA0B,GAAG,SAAA,IAAY,CAAA;AAAA,IAAC;AAAA,EAC7D;AAEJ;AAEA,eAAe,WAAW,IAAgB,QAAoB;AAC5D,QAAM,EAAC,mBAAkB;AAEzB,QAAM,eACJ,eAAe,SAAS,cAAc,eAAe,QAAQ;AAI/D,QAAM,eAAe,4BAA4B,IAAI,QAAQ,YAAY;AACzE,MAAI;AACJ,MAAI,cAAc;AAEhB,qBAAiB,MAAM;AACvB,OAAG,OAAO,yBAAyB,eAAe,SAAS,EAAE;AAAA,EAC/D,OAAO;AAKL,SAAK,aAAa,MAAM,CAAA,MAAK,GAAG,QAAQ,CAAC,CAAC;AAAA,EAC5C;AAEA,QAAM,EAAC,YAAY,IAAA,IAAO;AAAA,IACxB;AAAA,IACA;AAAA;AAAA,IACA,gBAAgB;AAAA,EAAA;AAElB,QAAM,EAAC,oBAAoB,YAAA,IAAe,OAAO;AACjD,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,MAClB,OAAO,QAAQ;AAAA,IAAA;AAAA,IAEjB,EAAC,KAAK,OAAO,WAAW,aAAa,KAAA;AAAA,EAAI;AAE3C,QAAM,EAAC,SAAS,SAAS,OAAA,IAAU,SAAA;AACnC,OAAK,GAAG,SAAS,MAAM;AACvB,OAAK,GAAG,SAAS,CAAC,MAAM,WAAW;AACjC,QAAI,QAAQ;AACV,aAAO,0BAA0B,MAAM,EAAE;AAAA,IAC3C,WAAW,SAAS,GAAG;AACrB,aAAO,+BAA+B,IAAI,EAAE;AAAA,IAC9C,OAAO;AACL,cAAA;AAAA,IACF;AAAA,EACF,CAAC;AACD,QAAM;AACN,MAAI,CAAC,WAAW,OAAO,QAAQ,IAAI,GAAG;AACpC,WAAO;AAAA,EACT;AACA,MACE,kBACA,CAAC,eAAe,IAAI,OAAO,QAAQ,MAAM,cAAc,GACvD;AACA,OAAG,OAAO,6CAA6C;AACvD,iBAAa,OAAO,QAAQ,IAAI;AAChC,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,eACP,IACA,SACA,UACA;AACA,QAAM,KAAK,IAAI,SAAS,IAAI,OAAO;AACnC,MAAI;AACF,UAAM,EAAC,gBAAgB,UAAA,IAAa;AAAA,MAClC,IAAI,gBAAgB,EAAE;AAAA,IAAA;AAExB,QAAI,mBAAmB,SAAS,gBAAgB;AAC9C,SAAG;AAAA,QACD,yBAAyB,cAAc,kDAAkD,SAAS,cAAc;AAAA,QAChH;AAAA,MAAA;AAEF,aAAO;AAAA,IACT;AACA,QAAI,YAAY,SAAS,cAAc;AACrC,SAAG;AAAA,QACD,2BAA2B,SAAS,iDAAiD,SAAS,YAAY;AAAA,MAAA;AAE5G,aAAO;AAAA,IACT;AACA,OAAG;AAAA,MACD,4BAA4B,cAAc,kBAAkB,SAAS;AAAA,MACrE;AAAA,IAAA;AAEF,WAAO;AAAA,EACT,UAAA;AACE,OAAG,MAAA;AAAA,EACL;AACF;AAEO,SAAS,0BACd,IACA,QACc;AACd,QAAM,EAAC,YAAY,QAAO,cAAc,MAAM;AAC9C,KAAG,OAAO,iCAAiC,OAAO,WAAW,SAAS,EAAE;AACxE,SAAO,MAAM,YAAY,CAAC,WAAW,GAAG;AAAA,IACtC;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,EAAA,CACd;AACH;AAEA,SAAS,4BACP,IACA,QACA,cACyB;AACzB,QAAM,EAAC,SAAS,QAAQ,SAAS,OAAA,IAAU,SAAA;AAE3C,QAAM,iBAAkB;AACtB,UAAM,QAAQ,IAAI,gBAAA;AAClB,YAAQ,GAAG,UAAU,MAAM,MAAM,OAAO;AACxC,YAAQ,GAAG,WAAW,MAAM,MAAM,OAAO;AAEzC,aAAS,IAAI,KAAK,KAAK;AACrB,UAAI,MAAwB;AAC5B,UAAI;AACF,YAAI,WAAW;AACf,cAAM,SAAS,MAAM,gBAAgB,IAAI,MAAM;AAC/C,yBAAiB,OAAO,QAAQ;AAI9B,kBAAQ,IAAI,CAAC,CAAC;AACd,qBAAW;AAAA,QACb;AAGA,YAAI,UAAU;AACZ;AAAA,QACF;AAAA,MACF,SAAS,GAAG;AACV,cAAM;AAAA,MACR;AACA,UAAI,CAAC,cAAc;AACjB,eAAO,OAAO,GAAG;AAAA,MACnB;AASA,SAAG;AAAA,QACD,uCAAuC,IAAI,CAAC;AAAA,QAC5C,OAAO,GAAG;AAAA,MAAA;AAEZ,UAAI;AACF,cAAM,MAAM,KAAM,MAAM,MAAM;AAAA,MAChC,SAAS,GAAG;AACV,eAAO,OAAO,CAAC;AAAA,MACjB;AAAA,IACF;AAAA,EACF,GAAA;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,IACA,QACkC;AAClC,mBAAiB,MAAM;AACvB,QAAM,EAAC,QAAQ,QAAQ,eAAA,IAAkB;AACzC,QAAM,UAAU,eAAe,MAAM;AAErC,QAAM,uBAAuB,IAAI;AAAA,IAC/B;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,EAAA;AAGjB,SAAO,qBAAqB,gBAAgB,MAAM;AACpD;"}

package/out/zero-client/src/client/version.js CHANGED Viewed

@@ -1,4 +1,4 @@
-const version = "0.25.13-canary.8";
+const version = "0.25.13";
 export {
   version
 };

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@rocicorp/zero",
-  "version": "0.25.13-canary.8",
+  "version": "0.25.13",
   "description": "Zero is a web framework for serverless web development.",
   "author": "Rocicorp, Inc.",
   "repository": {