@rocicorp/zero 0.26.0-canary.2 → 0.26.0-canary.3

package/out/zero-cache/src/server/priority-op.js

@@ -0,0 +1,29 @@
+let priorityOpCounter = 0;
+let runningPriorityOpCounter = 0;
+async function runPriorityOp(lc, description, op) {
+  const id = priorityOpCounter++;
+  runningPriorityOpCounter++;
+  const start = Date.now();
+  lc = lc.withContext("priorityOpID", id);
+  try {
+    lc.debug?.(`running priority op ${description}`);
+    const result = await op();
+    lc.debug?.(
+      `finished priority op ${description} in ${Date.now() - start} ms`
+    );
+    return result;
+  } catch (e) {
+    lc.debug?.(`failed priority op ${description} in ${Date.now() - start} ms`);
+    throw e;
+  } finally {
+    runningPriorityOpCounter--;
+  }
+}
+function isPriorityOpRunning() {
+  return runningPriorityOpCounter > 0;
+}
+export {
+  isPriorityOpRunning,
+  runPriorityOp
+};
+//# sourceMappingURL=priority-op.js.map

package/out/zero-cache/src/server/priority-op.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"priority-op.js","sources":["../../../../../zero-cache/src/server/priority-op.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\n\nlet priorityOpCounter = 0;\nlet runningPriorityOpCounter = 0;\n\n/**\n * Run an operation with priority, indicating that IVM should use smaller time\n * slices to allow this operation to proceed more quickly\n */\nexport async function runPriorityOp<T>(\n  lc: LogContext,\n  description: string,\n  op: () => Promise<T>,\n) {\n  const id = priorityOpCounter++;\n  runningPriorityOpCounter++;\n  const start = Date.now();\n  lc = lc.withContext('priorityOpID', id);\n  try {\n    lc.debug?.(`running priority op ${description}`);\n    const result = await op();\n    lc.debug?.(\n      `finished priority op ${description} in ${Date.now() - start} ms`,\n    );\n    return result;\n  } catch (e) {\n    lc.debug?.(`failed priority op ${description} in ${Date.now() - start} ms`);\n    throw e;\n  } finally {\n    runningPriorityOpCounter--;\n  }\n}\n\nexport function isPriorityOpRunning() {\n  return runningPriorityOpCounter > 0;\n}\n"],"names":[],"mappings":"AAEA,IAAI,oBAAoB;AACxB,IAAI,2BAA2B;AAM/B,eAAsB,cACpB,IACA,aACA,IACA;AACA,QAAM,KAAK;AACX;AACA,QAAM,QAAQ,KAAK,IAAA;AACnB,OAAK,GAAG,YAAY,gBAAgB,EAAE;AACtC,MAAI;AACF,OAAG,QAAQ,uBAAuB,WAAW,EAAE;AAC/C,UAAM,SAAS,MAAM,GAAA;AACrB,OAAG;AAAA,MACD,wBAAwB,WAAW,OAAO,KAAK,IAAA,IAAQ,KAAK;AAAA,IAAA;AAE9D,WAAO;AAAA,EACT,SAAS,GAAG;AACV,OAAG,QAAQ,sBAAsB,WAAW,OAAO,KAAK,IAAA,IAAQ,KAAK,KAAK;AAC1E,UAAM;AAAA,EACR,UAAA;AACE;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB;AACpC,SAAO,2BAA2B;AACpC;"}

package/out/zero-cache/src/server/syncer.d.ts

@@ -1,4 +1,3 @@
 import { type Worker } from '../types/processes.ts';
 export default function runWorker(parent: Worker, env: NodeJS.ProcessEnv, ...args: string[]): Promise<void>;
-export declare function isPriorityOpRunning(): boolean;
 //# sourceMappingURL=syncer.d.ts.map

package/out/zero-cache/src/server/syncer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAuBA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AA6B/B,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAoJf;AAwBD,wBAAgB,mBAAmB,YAElC"}
+{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAuBA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AA8B/B,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA8If"}

package/out/zero-cache/src/server/syncer.js

@@ -26,6 +26,7 @@ import { startAnonymousTelemetry } from "./anonymous-otel-start.js";
 import { InspectorDelegate } from "./inspector-delegate.js";
 import { createLogContext } from "./logging.js";
 import { startOtelAuto } from "./otel-start.js";
+import { runPriorityOp, isPriorityOpRunning } from "./priority-op.js";
 function randomID() {
   return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);
 }
@@ -93,16 +94,10 @@ function runWorker(parent, env, ...args) {
       config.taskID,
       id,
       cvrDB,
-      config.upstream.type === "pg" ? upstreamDB : void 0,
       new PipelineDriver(
         logger,
         config.log,
-        new Snapshotter(
-          logger,
-          replicaFile,
-          shard,
-          config.replica.pageCacheSizeKib
-        ),
+        new Snapshotter(logger, replicaFile, shard),
         shard,
         operatorStorage.createClientGroupStorage(id),
         id,
@@ -156,20 +151,7 @@ if (!singleProcessMode()) {
     () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2))
   );
 }
-let priorityOpCounter = 0;
-async function runPriorityOp(op) {
-  priorityOpCounter++;
-  try {
-    return await op();
-  } finally {
-    priorityOpCounter--;
-  }
-}
-function isPriorityOpRunning() {
-  return priorityOpCounter > 0;
-}
 export {
-  runWorker as default,
-  isPriorityOpRunning
+  runWorker as default
 };
 //# sourceMappingURL=syncer.js.map

package/out/zero-cache/src/server/syncer.js.map

@@ -1 +1 @@
-{"version":3,"file":"syncer.js","sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\nexport default function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n\n  startOtelAuto(createLogContext(config, {worker: 'syncer'}, false));\n  const lc = createLogContext(config, {worker: 'syncer'}, true);\n  initEventSink(lc, config);\n\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const {cvr, upstream} = config;\n  assert(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set');\n  assert(upstream.maxConnsPerWorker, 'upstream.maxConnsPerWorker must be set');\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = pgClient(lc, cvr.db, {\n    max: cvr.maxConnsPerWorker,\n    connection: {['application_name']: `zero-sync-worker-${pid}-cvr`},\n  });\n\n  const upstreamDB = pgClient(lc, upstream.db, {\n    max: upstream.maxConnsPerWorker,\n    connection: {['application_name']: `zero-sync-worker-${pid}-upstream`},\n  });\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    warmupConnections(lc, upstreamDB, 'upstream'),\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('component', 'view-syncer')\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    // Create the custom query transformer if configured\n    const customQueryConfig = getCustomQueryConfig(config);\n    const customQueryTransformer =\n      customQueryConfig &&\n      new CustomQueryTransformer(logger, customQueryConfig, shard);\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      config.upstream.type === 'pg' ? upstreamDB : undefined,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(\n          logger,\n          replicaFile,\n          shard,\n          config.replica.pageCacheSizeKib,\n        ),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = (id: string) =>\n    new MutagenService(\n      lc.withContext('component', 'mutagen').withContext('clientGroupID', id),\n      shard,\n      id,\n      upstreamDB,\n      config,\n      writeAuthzStorage,\n    );\n\n  const pusherFactory =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : (id: string) =>\n          new PusherService(\n            config,\n            {\n              ...config.push,\n              ...config.mutate,\n              url: must(\n                config.push.url ?? config.mutate.url,\n                'No push or mutate URL configured',\n              ),\n            },\n            lc.withContext('clientGroupID', id),\n            id,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(() =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n\nlet priorityOpCounter = 0;\n\n/**\n * Run an operation with priority, indicating that IVM should use smaller time\n * slices to allow this operation to proceed more quickly\n */\nasync function runPriorityOp<T>(op: () => Promise<T>) {\n  priorityOpCounter++;\n  try {\n    return await op();\n  } finally {\n    priorityOpCounter--;\n  }\n}\n\nexport function isPriorityOpRunning() {\n  return priorityOpCounter > 0;\n}\n"],"names":["v.parse"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqCA,SAAS,WAAW;AAClB,SAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;AACA,QAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;AAE9D,MAAI,CAAC,aAAa,KAAK;AACrB,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,KAAK,YAAY;AAAA,IACjB,gBAAgB,YAAY,kBAAkB;AAAA,EAAA;AAElD;AAEA,SAAwB,UACtB,QACA,QACG,MACY;AACf,QAAM,SAAS,wBAAwB,EAAC,KAAK,MAAM,KAAK,MAAM,CAAC,GAAE;AAEjE,gBAAc,iBAAiB,QAAQ,EAAC,QAAQ,SAAA,GAAW,KAAK,CAAC;AACjE,QAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,SAAA,GAAW,IAAI;AAC5D,gBAAc,IAAI,MAAM;AAExB,SAAO,KAAK,SAAS,GAAG,+BAA+B;AACvD,QAAM,WAAWA,MAAQ,KAAK,CAAC,GAAG,qBAAqB;AAEvD,QAAM,EAAC,KAAK,SAAA,IAAY;AACxB,SAAO,IAAI,mBAAmB,mCAAmC;AACjE,SAAO,SAAS,mBAAmB,wCAAwC;AAE3E,QAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;AACjE,KAAG,QAAQ,0BAA0B,WAAW,EAAE;AAElD,QAAM,QAAQ,SAAS,IAAI,IAAI,IAAI;AAAA,IACjC,KAAK,IAAI;AAAA,IACT,YAAY,EAAC,CAAC,kBAAkB,GAAG,oBAAoB,GAAG,OAAA;AAAA,EAAM,CACjE;AAED,QAAM,aAAa,SAAS,IAAI,SAAS,IAAI;AAAA,IAC3C,KAAK,SAAS;AAAA,IACd,YAAY,EAAC,CAAC,kBAAkB,GAAG,oBAAoB,GAAG,YAAA;AAAA,EAAW,CACtE;AAED,QAAM,WAAW,QAAQ,WAAW;AAAA,IAClC,kBAAkB,IAAI,OAAO,KAAK;AAAA,IAClC,kBAAkB,IAAI,YAAY,UAAU;AAAA,EAAA,CAC7C;AAED,QAAM,SAAS,OAAO,mBAAmB,OAAA;AACzC,QAAM,kBAAkB,gBAAgB;AAAA,IACtC;AAAA,IACA,KAAK,KAAK,QAAQ,eAAe,WAAA,CAAY,EAAE;AAAA,EAAA;AAEjD,QAAM,oBAAoB,gBAAgB;AAAA,IACxC;AAAA,IACA,KAAK,KAAK,QAAQ,WAAW,WAAA,CAAY,EAAE;AAAA,EAAA;AAG7C,QAAM,QAAQ,WAAW,MAAM;AAE/B,QAAM,oBAAoB,CACxB,IACA,KACA,qBACG;AACH,UAAM,SAAS,GACZ,YAAY,aAAa,aAAa,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,UAAU;AACrC,OAAG;AAAA,MACD,gDAAgD,OAAO,kBAAkB;AAAA,IAAA;AAI3E,UAAM,oBAAoB,qBAAqB,MAAM;AACrD,UAAM,yBACJ,qBACA,IAAI,uBAAuB,QAAQ,mBAAmB,KAAK;AAE7D,UAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;AAEtE,UAAM,oCAAoC,KAAK;AAAA,MAC7C,OAAO,mBAAmB;AAAA,MAC1B;AAAA,IAAA;AAEF,UAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;AAClE,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA,OAAO,SAAS,SAAS,OAAO,aAAa;AAAA,MAC7C,IAAI;AAAA,QACF;AAAA,QACA,OAAO;AAAA,QACP,IAAI;AAAA,UACF;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,QAAQ;AAAA,QAAA;AAAA,QAEjB;AAAA,QACA,gBAAgB,yBAAyB,EAAE;AAAA,QAC3C;AAAA,QACA;AAAA,QACA,MACE,oBAAA,IACI,oCACA;AAAA,QACN,OAAO;AAAA,MAAA;AAAA,MAET;AAAA,MACA;AAAA,MACA,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,QAAM,iBAAiB,CAAC,OACtB,IAAI;AAAA,IACF,GAAG,YAAY,aAAa,SAAS,EAAE,YAAY,iBAAiB,EAAE;AAAA,IACtE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGJ,QAAM,gBACJ,OAAO,KAAK,QAAQ,UAAa,OAAO,OAAO,QAAQ,SACnD,SACA,CAAC,OACC,IAAI;AAAA,IACF;AAAA,IACA;AAAA,MACE,GAAG,OAAO;AAAA,MACV,GAAG,OAAO;AAAA,MACV,KAAK;AAAA,QACH,OAAO,KAAK,OAAO,OAAO,OAAO;AAAA,QACjC;AAAA,MAAA;AAAA,IACF;AAAA,IAEF,GAAG,YAAY,iBAAiB,EAAE;AAAA,IAClC;AAAA,EAAA;AAGV,QAAM,SAAS,IAAI;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,0BAAwB,IAAI,MAAM;AAElC,OAAK,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAA,CAAK,CAAC,CAAC;AAE9D,SAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,qBAAqB;AACxB,OAAK;AAAA,IAAU,MACb,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC;AAAA,EAAA;AAEvE;AAEA,IAAI,oBAAoB;AAMxB,eAAe,cAAiB,IAAsB;AACpD;AACA,MAAI;AACF,WAAO,MAAM,GAAA;AAAA,EACf,UAAA;AACE;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB;AACpC,SAAO,oBAAoB;AAC7B;"}
+{"version":3,"file":"syncer.js","sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n  parentWorker,\n  singleProcessMode,\n  type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n  return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n  config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n  const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n  if (!queryConfig?.url) {\n    return undefined;\n  }\n\n  return {\n    url: queryConfig.url,\n    forwardCookies: queryConfig.forwardCookies ?? false,\n  };\n}\n\nexport default function runWorker(\n  parent: Worker,\n  env: NodeJS.ProcessEnv,\n  ...args: string[]\n): Promise<void> {\n  const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n\n  startOtelAuto(createLogContext(config, {worker: 'syncer'}, false));\n  const lc = createLogContext(config, {worker: 'syncer'}, true);\n  initEventSink(lc, config);\n\n  assert(args.length > 0, `replicator mode not specified`);\n  const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n  const {cvr, upstream} = config;\n  assert(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set');\n  assert(upstream.maxConnsPerWorker, 'upstream.maxConnsPerWorker must be set');\n\n  const replicaFile = replicaFileName(config.replica.file, fileMode);\n  lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n  const cvrDB = pgClient(lc, cvr.db, {\n    max: cvr.maxConnsPerWorker,\n    connection: {['application_name']: `zero-sync-worker-${pid}-cvr`},\n  });\n\n  const upstreamDB = pgClient(lc, upstream.db, {\n    max: upstream.maxConnsPerWorker,\n    connection: {['application_name']: `zero-sync-worker-${pid}-upstream`},\n  });\n\n  const dbWarmup = Promise.allSettled([\n    warmupConnections(lc, cvrDB, 'cvr'),\n    warmupConnections(lc, upstreamDB, 'upstream'),\n  ]);\n\n  const tmpDir = config.storageDBTmpDir ?? tmpdir();\n  const operatorStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `sync-worker-${randomUUID()}`),\n  );\n  const writeAuthzStorage = DatabaseStorage.create(\n    lc,\n    path.join(tmpDir, `mutagen-${randomUUID()}`),\n  );\n\n  const shard = getShardID(config);\n\n  const viewSyncerFactory = (\n    id: string,\n    sub: Subscription<ReplicaState>,\n    drainCoordinator: DrainCoordinator,\n  ) => {\n    const logger = lc\n      .withContext('component', 'view-syncer')\n      .withContext('clientGroupID', id)\n      .withContext('instance', randomID());\n    lc.debug?.(\n      `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n    );\n\n    // Create the custom query transformer if configured\n    const customQueryConfig = getCustomQueryConfig(config);\n    const customQueryTransformer =\n      customQueryConfig &&\n      new CustomQueryTransformer(logger, customQueryConfig, shard);\n\n    const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n    const priorityOpRunningYieldThresholdMs = Math.max(\n      config.yieldThresholdMs / 4,\n      2,\n    );\n    const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n    return new ViewSyncerService(\n      config,\n      logger,\n      shard,\n      config.taskID,\n      id,\n      cvrDB,\n      new PipelineDriver(\n        logger,\n        config.log,\n        new Snapshotter(logger, replicaFile, shard),\n        shard,\n        operatorStorage.createClientGroupStorage(id),\n        id,\n        inspectorDelegate,\n        () =>\n          isPriorityOpRunning()\n            ? priorityOpRunningYieldThresholdMs\n            : normalYieldThresholdMs,\n        config.enableQueryPlanner,\n      ),\n      sub,\n      drainCoordinator,\n      config.log.slowHydrateThreshold,\n      inspectorDelegate,\n      customQueryTransformer,\n      runPriorityOp,\n    );\n  };\n\n  const mutagenFactory = (id: string) =>\n    new MutagenService(\n      lc.withContext('component', 'mutagen').withContext('clientGroupID', id),\n      shard,\n      id,\n      upstreamDB,\n      config,\n      writeAuthzStorage,\n    );\n\n  const pusherFactory =\n    config.push.url === undefined && config.mutate.url === undefined\n      ? undefined\n      : (id: string) =>\n          new PusherService(\n            config,\n            {\n              ...config.push,\n              ...config.mutate,\n              url: must(\n                config.push.url ?? config.mutate.url,\n                'No push or mutate URL configured',\n              ),\n            },\n            lc.withContext('clientGroupID', id),\n            id,\n          );\n\n  const syncer = new Syncer(\n    lc,\n    config,\n    viewSyncerFactory,\n    mutagenFactory,\n    pusherFactory,\n    parent,\n  );\n\n  startAnonymousTelemetry(lc, config);\n\n  void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n  return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n  void exitAfter(() =>\n    runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n  );\n}\n"],"names":["v.parse"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsCA,SAAS,WAAW;AAClB,SAAO,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE;AACxD;AAEA,SAAS,qBACP,QACA;AACA,QAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;AAE9D,MAAI,CAAC,aAAa,KAAK;AACrB,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,KAAK,YAAY;AAAA,IACjB,gBAAgB,YAAY,kBAAkB;AAAA,EAAA;AAElD;AAEA,SAAwB,UACtB,QACA,QACG,MACY;AACf,QAAM,SAAS,wBAAwB,EAAC,KAAK,MAAM,KAAK,MAAM,CAAC,GAAE;AAEjE,gBAAc,iBAAiB,QAAQ,EAAC,QAAQ,SAAA,GAAW,KAAK,CAAC;AACjE,QAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,SAAA,GAAW,IAAI;AAC5D,gBAAc,IAAI,MAAM;AAExB,SAAO,KAAK,SAAS,GAAG,+BAA+B;AACvD,QAAM,WAAWA,MAAQ,KAAK,CAAC,GAAG,qBAAqB;AAEvD,QAAM,EAAC,KAAK,SAAA,IAAY;AACxB,SAAO,IAAI,mBAAmB,mCAAmC;AACjE,SAAO,SAAS,mBAAmB,wCAAwC;AAE3E,QAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,QAAQ;AACjE,KAAG,QAAQ,0BAA0B,WAAW,EAAE;AAElD,QAAM,QAAQ,SAAS,IAAI,IAAI,IAAI;AAAA,IACjC,KAAK,IAAI;AAAA,IACT,YAAY,EAAC,CAAC,kBAAkB,GAAG,oBAAoB,GAAG,OAAA;AAAA,EAAM,CACjE;AAED,QAAM,aAAa,SAAS,IAAI,SAAS,IAAI;AAAA,IAC3C,KAAK,SAAS;AAAA,IACd,YAAY,EAAC,CAAC,kBAAkB,GAAG,oBAAoB,GAAG,YAAA;AAAA,EAAW,CACtE;AAED,QAAM,WAAW,QAAQ,WAAW;AAAA,IAClC,kBAAkB,IAAI,OAAO,KAAK;AAAA,IAClC,kBAAkB,IAAI,YAAY,UAAU;AAAA,EAAA,CAC7C;AAED,QAAM,SAAS,OAAO,mBAAmB,OAAA;AACzC,QAAM,kBAAkB,gBAAgB;AAAA,IACtC;AAAA,IACA,KAAK,KAAK,QAAQ,eAAe,WAAA,CAAY,EAAE;AAAA,EAAA;AAEjD,QAAM,oBAAoB,gBAAgB;AAAA,IACxC;AAAA,IACA,KAAK,KAAK,QAAQ,WAAW,WAAA,CAAY,EAAE;AAAA,EAAA;AAG7C,QAAM,QAAQ,WAAW,MAAM;AAE/B,QAAM,oBAAoB,CACxB,IACA,KACA,qBACG;AACH,UAAM,SAAS,GACZ,YAAY,aAAa,aAAa,EACtC,YAAY,iBAAiB,EAAE,EAC/B,YAAY,YAAY,UAAU;AACrC,OAAG;AAAA,MACD,gDAAgD,OAAO,kBAAkB;AAAA,IAAA;AAI3E,UAAM,oBAAoB,qBAAqB,MAAM;AACrD,UAAM,yBACJ,qBACA,IAAI,uBAAuB,QAAQ,mBAAmB,KAAK;AAE7D,UAAM,oBAAoB,IAAI,kBAAkB,sBAAsB;AAEtE,UAAM,oCAAoC,KAAK;AAAA,MAC7C,OAAO,mBAAmB;AAAA,MAC1B;AAAA,IAAA;AAEF,UAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,CAAC;AAClE,WAAO,IAAI;AAAA,MACT;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA,IAAI;AAAA,QACF;AAAA,QACA,OAAO;AAAA,QACP,IAAI,YAAY,QAAQ,aAAa,KAAK;AAAA,QAC1C;AAAA,QACA,gBAAgB,yBAAyB,EAAE;AAAA,QAC3C;AAAA,QACA;AAAA,QACA,MACE,oBAAA,IACI,oCACA;AAAA,QACN,OAAO;AAAA,MAAA;AAAA,MAET;AAAA,MACA;AAAA,MACA,OAAO,IAAI;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAEA,QAAM,iBAAiB,CAAC,OACtB,IAAI;AAAA,IACF,GAAG,YAAY,aAAa,SAAS,EAAE,YAAY,iBAAiB,EAAE;AAAA,IACtE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGJ,QAAM,gBACJ,OAAO,KAAK,QAAQ,UAAa,OAAO,OAAO,QAAQ,SACnD,SACA,CAAC,OACC,IAAI;AAAA,IACF;AAAA,IACA;AAAA,MACE,GAAG,OAAO;AAAA,MACV,GAAG,OAAO;AAAA,MACV,KAAK;AAAA,QACH,OAAO,KAAK,OAAO,OAAO,OAAO;AAAA,QACjC;AAAA,MAAA;AAAA,IACF;AAAA,IAEF,GAAG,YAAY,iBAAiB,EAAE;AAAA,IAClC;AAAA,EAAA;AAGV,QAAM,SAAS,IAAI;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,0BAAwB,IAAI,MAAM;AAElC,OAAK,SAAS,KAAK,MAAM,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAA,CAAK,CAAC,CAAC;AAE9D,SAAO,eAAe,IAAI,QAAQ,MAAM;AAC1C;AAGA,IAAI,CAAC,qBAAqB;AACxB,OAAK;AAAA,IAAU,MACb,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC;AAAA,EAAA;AAEvE;"}

package/out/zero-cache/src/services/analyze.js

@@ -64,7 +64,7 @@ async function analyzeQuery(lc, config, clientSchema, ast, syncedRows = true, ve
     computeZqlSpecs(lc, db, tableSpecs, fullTables);
     const planDebugger = joinPlans ? new AccumulatorDebugger() : void 0;
     const costModel = joinPlans ? createSQLiteCostModel(db, tableSpecs) : void 0;
-    const timer = await new TimeSliceTimer().start();
+    const timer = await new TimeSliceTimer(lc).start();
     const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;
     const yieldProcess = () => timer.yieldProcess();
     const result = await runAst(

package/out/zero-cache/src/services/analyze.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze.js","sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer, type TokenData} from './view-syncer/view-syncer.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  authData?: TokenData,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(lc, db, tableSpecs, fullTables);\n\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const costModel = joinPlans\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const timer = await new TimeSliceTimer().start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      authData,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      host: {\n        debug: new Debug(),\n        getSource(tableName: string) {\n          let source = tables.get(tableName);\n          if (source) {\n            return source;\n          }\n\n          const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n          const {primaryKey} = tableSpec.tableSpec;\n\n          source = new TableSource(\n            lc,\n            config.log,\n            db,\n            tableName,\n            tableSpec.zqlSpec,\n            primaryKey,\n            shouldYield,\n          );\n          tables.set(tableName, source);\n          return source;\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  result.sqlitePlans = explainQueries(result.readRowCountsByQuery ?? {}, db);\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqBA,MAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,UACA,YAAY,OACiB;AAC7B;AAAA;AAAA,UAAM,KAAK,oBAAI,SAAS,IAAI,OAAO,QAAQ,IAAI;AAC/C,UAAM,iCAAiB,IAAA;AACvB,UAAM,iCAAiB,IAAA;AACvB,UAAM,6BAAa,IAAA;AAEnB,oBAAgB,IAAI,IAAI,YAAY,UAAU;AAE9C,UAAM,eAAe,YAAY,IAAI,oBAAA,IAAwB;AAC7D,UAAM,YAAY,YACd,sBAAsB,IAAI,UAAU,IACpC;AACJ,UAAM,QAAQ,MAAM,IAAI,eAAA,EAAiB,MAAA;AACzC,UAAM,cAAc,MAAM,MAAM,WAAA,IAAe;AAC/C,UAAM,eAAe,MAAM,MAAM,aAAA;AACjC,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,QACE,kBAAkB,gBAAgB;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,QAGA;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM;AAAA,UACJ,OAAO,IAAI,MAAA;AAAA,UACX,UAAU,WAAmB;AAC3B,gBAAI,SAAS,OAAO,IAAI,SAAS;AACjC,gBAAI,QAAQ;AACV,qBAAO;AAAA,YACT;AAEA,kBAAM,YAAY,iBAAiB,YAAY,SAAS;AACxD,kBAAM,EAAC,eAAc,UAAU;AAE/B,qBAAS,IAAI;AAAA,cACX;AAAA,cACA,OAAO;AAAA,cACP;AAAA,cACA;AAAA,cACA,UAAU;AAAA,cACV;AAAA,cACA;AAAA,YAAA;AAEF,mBAAO,IAAI,WAAW,MAAM;AAC5B,mBAAO;AAAA,UACT;AAAA,UACA,gBAAgB;AACd,mBAAO,IAAI,cAAA;AAAA,UACb;AAAA,UACA,qBAAqB,CAAA,UAAS;AAAA,UAC9B,eAAe,CAAA,UAAS;AAAA,UACxB,UAAU;AAAA,UAAC;AAAA,UACX,qBAAqB,CAAA,UAAS;AAAA,QAAA;AAAA,MAChC;AAAA,MAEF;AAAA,IAAA;AAGF,WAAO,cAAc,eAAe,OAAO,wBAAwB,CAAA,GAAI,EAAE;AAEzE,QAAI,cAAc;AAChB,aAAO,YAAY,yBAAyB,aAAa,MAAM;AAAA,IACjE;AAEA,WAAO;AAAA,WAtEP;AAAA;AAAA;AAAA;AAAA;AAuEF;"}
+{"version":3,"file":"analyze.js","sources":["../../../../../zero-cache/src/services/analyze.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {AnalyzeQueryResult} from '../../../zero-protocol/src/analyze-query-result.ts';\nimport type {AST} from '../../../zero-protocol/src/ast.ts';\nimport type {PermissionsConfig} from '../../../zero-schema/src/compiled-permissions.ts';\nimport {Debug} from '../../../zql/src/builder/debug-delegate.ts';\nimport {MemoryStorage} from '../../../zql/src/ivm/memory-storage.ts';\nimport {\n  AccumulatorDebugger,\n  serializePlanDebugEvents,\n} from '../../../zql/src/planner/planner-debug.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport {explainQueries} from '../../../zqlite/src/explain-queries.ts';\nimport {createSQLiteCostModel} from '../../../zqlite/src/sqlite-cost-model.ts';\nimport {TableSource} from '../../../zqlite/src/table-source.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {computeZqlSpecs, mustGetTableSpec} from '../db/lite-tables.ts';\nimport type {LiteAndZqlSpec, LiteTableSpec} from '../db/specs.ts';\nimport {runAst} from './run-ast.ts';\nimport {TimeSliceTimer, type TokenData} from './view-syncer/view-syncer.ts';\nimport type {ClientSchema} from '../../../zero-protocol/src/client-schema.ts';\n\nconst TIME_SLICE_LAP_THRESHOLD_MS = 200;\n\nexport async function analyzeQuery(\n  lc: LogContext,\n  config: NormalizedZeroConfig,\n  clientSchema: ClientSchema,\n  ast: AST,\n  syncedRows = true,\n  vendedRows = false,\n  permissions?: PermissionsConfig,\n  authData?: TokenData,\n  joinPlans = false,\n): Promise<AnalyzeQueryResult> {\n  using db = new Database(lc, config.replica.file);\n  const fullTables = new Map<string, LiteTableSpec>();\n  const tableSpecs = new Map<string, LiteAndZqlSpec>();\n  const tables = new Map<string, TableSource>();\n\n  computeZqlSpecs(lc, db, tableSpecs, fullTables);\n\n  const planDebugger = joinPlans ? new AccumulatorDebugger() : undefined;\n  const costModel = joinPlans\n    ? createSQLiteCostModel(db, tableSpecs)\n    : undefined;\n  const timer = await new TimeSliceTimer(lc).start();\n  const shouldYield = () => timer.elapsedLap() > TIME_SLICE_LAP_THRESHOLD_MS;\n  const yieldProcess = () => timer.yieldProcess();\n  const result = await runAst(\n    lc,\n    clientSchema,\n    ast,\n    true,\n    {\n      applyPermissions: permissions !== undefined,\n      syncedRows,\n      vendedRows,\n      authData,\n      db,\n      tableSpecs,\n      permissions,\n      costModel,\n      planDebugger,\n      host: {\n        debug: new Debug(),\n        getSource(tableName: string) {\n          let source = tables.get(tableName);\n          if (source) {\n            return source;\n          }\n\n          const tableSpec = mustGetTableSpec(tableSpecs, tableName);\n          const {primaryKey} = tableSpec.tableSpec;\n\n          source = new TableSource(\n            lc,\n            config.log,\n            db,\n            tableName,\n            tableSpec.zqlSpec,\n            primaryKey,\n            shouldYield,\n          );\n          tables.set(tableName, source);\n          return source;\n        },\n        createStorage() {\n          return new MemoryStorage();\n        },\n        decorateSourceInput: input => input,\n        decorateInput: input => input,\n        addEdge() {},\n        decorateFilterInput: input => input,\n      },\n    },\n    yieldProcess,\n  );\n\n  result.sqlitePlans = explainQueries(result.readRowCountsByQuery ?? {}, db);\n\n  if (planDebugger) {\n    result.joinPlans = serializePlanDebugEvents(planDebugger.events);\n  }\n\n  return result;\n}\n"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqBA,MAAM,8BAA8B;AAEpC,eAAsB,aACpB,IACA,QACA,cACA,KACA,aAAa,MACb,aAAa,OACb,aACA,UACA,YAAY,OACiB;AAC7B;AAAA;AAAA,UAAM,KAAK,oBAAI,SAAS,IAAI,OAAO,QAAQ,IAAI;AAC/C,UAAM,iCAAiB,IAAA;AACvB,UAAM,iCAAiB,IAAA;AACvB,UAAM,6BAAa,IAAA;AAEnB,oBAAgB,IAAI,IAAI,YAAY,UAAU;AAE9C,UAAM,eAAe,YAAY,IAAI,oBAAA,IAAwB;AAC7D,UAAM,YAAY,YACd,sBAAsB,IAAI,UAAU,IACpC;AACJ,UAAM,QAAQ,MAAM,IAAI,eAAe,EAAE,EAAE,MAAA;AAC3C,UAAM,cAAc,MAAM,MAAM,WAAA,IAAe;AAC/C,UAAM,eAAe,MAAM,MAAM,aAAA;AACjC,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,QACE,kBAAkB,gBAAgB;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,QAGA;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM;AAAA,UACJ,OAAO,IAAI,MAAA;AAAA,UACX,UAAU,WAAmB;AAC3B,gBAAI,SAAS,OAAO,IAAI,SAAS;AACjC,gBAAI,QAAQ;AACV,qBAAO;AAAA,YACT;AAEA,kBAAM,YAAY,iBAAiB,YAAY,SAAS;AACxD,kBAAM,EAAC,eAAc,UAAU;AAE/B,qBAAS,IAAI;AAAA,cACX;AAAA,cACA,OAAO;AAAA,cACP;AAAA,cACA;AAAA,cACA,UAAU;AAAA,cACV;AAAA,cACA;AAAA,YAAA;AAEF,mBAAO,IAAI,WAAW,MAAM;AAC5B,mBAAO;AAAA,UACT;AAAA,UACA,gBAAgB;AACd,mBAAO,IAAI,cAAA;AAAA,UACb;AAAA,UACA,qBAAqB,CAAA,UAAS;AAAA,UAC9B,eAAe,CAAA,UAAS;AAAA,UACxB,UAAU;AAAA,UAAC;AAAA,UACX,qBAAqB,CAAA,UAAS;AAAA,QAAA;AAAA,MAChC;AAAA,MAEF;AAAA,IAAA;AAGF,WAAO,cAAc,eAAe,OAAO,wBAAwB,CAAA,GAAI,EAAE;AAEzE,QAAI,cAAc;AAChB,aAAO,YAAY,yBAAyB,aAAa,MAAM;AAAA,IACjE;AAEA,WAAO;AAAA,WAtEP;AAAA;AAAA;AAAA;AAAA;AAuEF;"}

package/out/zero-cache/src/services/change-source/custom/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAEnE,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,kDAAkD,CAAC;AAQ1D,OAAO,EAGL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AAKtD;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAgC7E;AAwDD;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,iBA+EpB"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAGzD,OAAO,KAAK,EAAC,WAAW,EAAU,MAAM,0BAA0B,CAAC;AAEnE,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,kDAAkD,CAAC;AAO1D,OAAO,EAGL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AAKtD;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAgC7E;AA2DD;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,iBA8EpB"}

package/out/zero-cache/src/services/change-source/custom/change-source.js

@@ -9,7 +9,6 @@ import { stream } from "../../../types/streams.js";
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import { ChangeProcessor } from "../../replicator/change-processor.js";
 import { ReplicationStatusPublisher } from "../../replicator/replication-status.js";
-import { initChangeLog } from "../../replicator/schema/change-log.js";
 import { getSubscriptionState, initReplicationState } from "../../replicator/schema/replication-state.js";
 import { changeStreamMessageSchema } from "../protocol/current/downstream.js";
 import "../protocol/current/status.js";
@@ -69,7 +68,10 @@ class CustomChangeSource {
       url.searchParams.append("publications", pub);
     }
     if (clientWatermark) {
-      assert(replicaVersion.length);
+      assert(
+        replicaVersion.length,
+        "replicaVersion is required when clientWatermark is set"
+      );
       url.searchParams.set("lastWatermark", clientWatermark);
       url.searchParams.set("replicaVersion", replicaVersion);
     }
@@ -118,7 +120,6 @@ async function initialSync(lc, shard, tx, upstreamURI) {
             5e3
           );
           initReplicationState(tx, [...publications].sort(), commitWatermark);
-          initChangeLog(tx);
           processor.processMessage(lc, change);
           break;
         }

package/out/zero-cache/src/services/change-source/custom/change-source.js.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.js","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../../change-streamer/change-streamer-service.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {initChangeLog} from '../../replicator/schema/change-log.ts';\nimport {\n  getSubscriptionState,\n  initReplicationState,\n  type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {type ChangeSourceUpstream} from '../protocol/current/upstream.ts';\nimport {initSyncSchema} from './sync-schema.ts';\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n  await initSyncSchema(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    shard,\n    replicaDbFile,\n    upstreamURI,\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = [...shard.publications].sort();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startStream(clientWatermark: string): Promise<ChangeStream> {\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(replicaVersion.length);\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = new ReplicationStatusPublisher(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(tx, [...publications].sort(), commitWatermark);\n          initChangeLog(tx);\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db);\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"names":["tag"],"mappings":";;;;;;;;;;;;;;;;AAmCA,eAAsB,6BACpB,IACA,aACA,OACA,eAC6E;AAC7E,QAAM;AAAA,IACJ;AAAA,IACA,WAAW,MAAM,KAAK,IAAI,MAAM,QAAQ;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,UAAU,IAAI,SAAS,IAAI,aAAa;AAC9C,QAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;AAC3E,UAAQ,MAAA;AAER,MAAI,MAAM,aAAa,QAAQ;AAE7B,UAAM,YAAY,CAAC,GAAG,MAAM,YAAY,EAAE,KAAA;AAC1C,UAAM,aAAa,kBAAkB,aAAa,KAAA;AAClD,QAAI,CAAC,UAAU,WAAW,UAAU,GAAG;AACrC,YAAM,IAAI;AAAA,QACR,gDAAgD,SAAS,wCAAwC,UAAU;AAAA,MAAA;AAAA,IAE/G;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAAA,IACvB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,SAAO,EAAC,mBAAmB,aAAA;AAC7B;AAEA,MAAM,mBAA2C;AAAA,EACtC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,IACA,aACA,OACA,mBACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,eAAe;AACtD,SAAK,eAAe;AACpB,SAAK,SAAS;AACd,SAAK,qBAAqB;AAAA,EAC5B;AAAA,EAEA,cAA4B;AAC1B,WAAO,KAAK,aAAA;AAAA,EACd;AAAA,EAEA,YAAY,iBAAgD;AAC1D,WAAO,QAAQ,QAAQ,KAAK,aAAa,eAAe,CAAC;AAAA,EAC3D;AAAA,EAEA,aAAa,iBAAwC;AACnD,UAAM,EAAC,cAAc,eAAA,IAAkB,KAAK;AAC5C,UAAM,EAAC,OAAO,SAAA,IAAY,KAAK;AAC/B,UAAM,MAAM,IAAI,IAAI,KAAK,YAAY;AACrC,QAAI,aAAa,IAAI,SAAS,KAAK;AACnC,QAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;AACjD,eAAW,OAAO,cAAc;AAC9B,UAAI,aAAa,OAAO,gBAAgB,GAAG;AAAA,IAC7C;AACA,QAAI,iBAAiB;AACnB,aAAO,eAAe,MAAM;AAC5B,UAAI,aAAa,IAAI,iBAAiB,eAAe;AACrD,UAAI,aAAa,IAAI,kBAAkB,cAAc;AAAA,IACvD;AAEA,UAAM,KAAK,IAAI,UAAU,GAAG;AAC5B,UAAM,EAAC,UAAU,UAAA,IAAa;AAAA,MAC5B,KAAK;AAAA,MACL;AAAA,MACA;AAAA;AAAA;AAAA,MAGA,EAAC,UAAU,CAAC,SAA+B,KAAA;AAAA,IAAI;AAEjD,WAAO,EAAC,SAAS,UAAU,MAAM,UAAA;AAAA,EACnC;AACF;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA;AACA,QAAM,EAAC,OAAO,IAAI,aAAA,IAAgB;AAClC,QAAM,eAAe,IAAI,mBAAmB,IAAI,aAAa,OAAO;AAAA,IAClE,gBAAgB;AAAA;AAAA,IAChB;AAAA,EAAA,CACD;AACD,QAAM,EAAC,QAAA,IAAW,aAAa,YAAA;AAE/B,QAAM,YAAY,IAAI;AAAA,IACpB,IAAI,gBAAgB,EAAE;AAAA,IACtB;AAAA,IACA,CAAC,GAAG,QAAQ;AACV,YAAM;AAAA,IACR;AAAA,EAAA;AAGF,QAAM,kBAAkB,IAAI,2BAA2B,EAAE;AACzD,MAAI;AACF,QAAI,MAAM;AACV,qBAAiB,UAAU,SAAS;AAClC,YAAM,CAAC,GAAG,IAAI;AACd,cAAQ,KAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,EAAC,gBAAA,IAAmB,OAAO,CAAC;AAClC,aAAG;AAAA,YACD,yBAAyB,EAAE,sBAAsB,eAAe;AAAA,UAAA;AAElE,0BAAgB;AAAA,YACd;AAAA,YACA;AAAA,YACA,sCAAsC,eAAe;AAAA,YACrD;AAAA,UAAA;AAEF,+BAAqB,IAAI,CAAC,GAAG,YAAY,EAAE,KAAA,GAAQ,eAAe;AAClE,wBAAc,EAAE;AAChB,oBAAU,eAAe,IAAI,MAAM;AACnC;AAAA,QACF;AAAA,QACA,KAAK;AACH,oBAAU,eAAe,IAAI,MAAM;AACnC,cAAI,EAAE,MAAM,QAAS,GAAG;AACtB,eAAG,QAAQ,aAAa,GAAG,UAAU;AAAA,UACvC;AACA;AAAA,QACF,KAAK;AACH,oBAAU,eAAe,IAAI,MAAM;AACnC,sCAA4B,IAAI,IAAI,KAAK;AACzC,aAAG,OAAO,4BAA4B,GAAG,UAAU;AACnD;AAAA,QAEF,KAAK;AACH;AAAA;AAAA;AAAA,QAEF,KAAK,WAAW;AACd,gBAAM,EAAC,KAAAA,MAAK,QAAA,IAAW,OAAO,CAAC;AAC/B,cAAIA,SAAQ,kBAAkB;AAC5B,kBAAM,IAAI;AAAA,cACR,WAAW;AAAA,YAAA;AAAA,UAEf;AAAA,QACF;AAAA;AAAA,QAEA,KAAK;AACH,gBAAM,IAAI;AAAA,YACR,2CAA2C,UAAU,MAAM,CAAC;AAAA,UAAA;AAAA,QAEhE;AACE,sBAAY,MAAM;AAAA,MAAA;AAAA,IAExB;AACA,UAAM,IAAI;AAAA,MACR,iBAAiB,WAAW;AAAA,IAAA;AAAA,EAEhC,SAAS,GAAG;AACV,UAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;AAAA,EAClE,UAAA;AACE,oBAAgB,KAAA;AAAA,EAClB;AACF;AAIA,SAAS,kBAAkB;AAAA,EACzB;AAAA,EACA;AACF,GAAyD;AACvD,SAAO;AAAA,IACL,CAAC,GAAG,KAAK,IAAI,QAAQ,UAAU,GAAG;AAAA,MAChC,eAAe,EAAC,MAAM,SAAA;AAAA,MACtB,UAAU,EAAC,MAAM,SAAA;AAAA,MACjB,gBAAgB,EAAC,MAAM,SAAA;AAAA,MACvB,QAAQ,EAAC,MAAM,SAAA;AAAA,IAAQ;AAAA,IAEzB,CAAC,GAAG,KAAK,IAAI,QAAQ,YAAY,GAAG;AAAA,MAClC,eAAe,EAAC,MAAM,SAAA;AAAA,MACtB,UAAU,EAAC,MAAM,SAAA;AAAA,MACjB,YAAY,EAAC,MAAM,SAAA;AAAA,MACnB,UAAU,EAAC,MAAM,OAAA;AAAA,IAAM;AAAA,IAEzB,CAAC,GAAG,KAAK,cAAc,GAAG;AAAA,MACxB,aAAa,EAAC,MAAM,OAAA;AAAA,MACpB,MAAM,EAAC,MAAM,SAAA;AAAA,IAAQ;AAAA,EACvB;AAEJ;AAEA,SAAS,4BACP,IACA,IACA,OACA;AACA,QAAM,SAAS,gBAAgB,IAAI,EAAE;AACrC,QAAM,WAAW,kBAAkB,KAAK;AACxC,aAAW,CAAC,MAAM,OAAO,KAAK,OAAO,QAAQ,QAAQ,GAAG;AACtD,UAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;AAChC,QAAI,CAAC,OAAO;AACV,YAAM,IAAI;AAAA,QACR,4BAA4B,IAAI,mBAAmB;AAAA,UACjD,GAAG,OAAO,KAAA;AAAA,QAAK,CAChB;AAAA,MAAA;AAAA,IAIL;AACA,eAAW,CAAC,KAAK,EAAC,KAAA,CAAK,KAAK,OAAO,QAAQ,OAAO,GAAG;AACnD,YAAM,QAAQ,MAAM,GAAG;AACvB,UAAI,CAAC,OAAO;AACV,cAAM,IAAI;AAAA,UACR,aAAa,KAAK,2BAA2B,GAAG;AAAA,QAAA;AAAA,MAEpD;AACA,UAAI,MAAM,SAAS,MAAM;AACvB,cAAM,IAAI;AAAA,UACR,aAAa,KAAK,IAAI,GAAG,iBAAiB,MAAM,IAAI,uBAAuB,IAAI;AAAA,QAAA;AAAA,MAEnF;AAAA,IACF;AAAA,EACF;AACF;"}
+{"version":3,"file":"change-source.js","sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {stringify} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport type {\n  ChangeSource,\n  ChangeStream,\n} from '../../change-streamer/change-streamer-service.ts';\nimport {\n  AutoResetSignal,\n  type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n  getSubscriptionState,\n  initReplicationState,\n  type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {type ChangeSourceUpstream} from '../protocol/current/upstream.ts';\nimport {initSyncSchema} from './sync-schema.ts';\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n  lc: LogContext,\n  upstreamURI: string,\n  shard: ShardConfig,\n  replicaDbFile: string,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n  await initSyncSchema(\n    lc,\n    `replica-${shard.appID}-${shard.shardNum}`,\n    shard,\n    replicaDbFile,\n    upstreamURI,\n  );\n\n  const replica = new Database(lc, replicaDbFile);\n  const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n  replica.close();\n\n  if (shard.publications.length) {\n    // Verify that the publications match what has been synced.\n    const requested = [...shard.publications].sort();\n    const replicated = subscriptionState.publications.sort();\n    if (!deepEqual(requested, replicated)) {\n      throw new Error(\n        `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n      );\n    }\n  }\n\n  const changeSource = new CustomChangeSource(\n    lc,\n    upstreamURI,\n    shard,\n    subscriptionState,\n  );\n\n  return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n  readonly #lc: LogContext;\n  readonly #upstreamUri: string;\n  readonly #shard: ShardID;\n  readonly #replicationConfig: ReplicationConfig;\n\n  constructor(\n    lc: LogContext,\n    upstreamUri: string,\n    shard: ShardID,\n    replicationConfig: ReplicationConfig,\n  ) {\n    this.#lc = lc.withContext('component', 'change-source');\n    this.#upstreamUri = upstreamUri;\n    this.#shard = shard;\n    this.#replicationConfig = replicationConfig;\n  }\n\n  initialSync(): ChangeStream {\n    return this.#startStream();\n  }\n\n  startStream(clientWatermark: string): Promise<ChangeStream> {\n    return Promise.resolve(this.#startStream(clientWatermark));\n  }\n\n  #startStream(clientWatermark?: string): ChangeStream {\n    const {publications, replicaVersion} = this.#replicationConfig;\n    const {appID, shardNum} = this.#shard;\n    const url = new URL(this.#upstreamUri);\n    url.searchParams.set('appID', appID);\n    url.searchParams.set('shardNum', String(shardNum));\n    for (const pub of publications) {\n      url.searchParams.append('publications', pub);\n    }\n    if (clientWatermark) {\n      assert(\n        replicaVersion.length,\n        'replicaVersion is required when clientWatermark is set',\n      );\n      url.searchParams.set('lastWatermark', clientWatermark);\n      url.searchParams.set('replicaVersion', replicaVersion);\n    }\n\n    const ws = new WebSocket(url);\n    const {instream, outstream} = stream(\n      this.#lc,\n      ws,\n      changeStreamMessageSchema,\n      // Upstream acks coalesce. If upstream exhibits back-pressure,\n      // only the last ACK is kept / buffered.\n      {coalesce: (curr: ChangeSourceUpstream) => curr},\n    );\n    return {changes: instream, acks: outstream};\n  }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n  lc: LogContext,\n  shard: ShardConfig,\n  tx: Database,\n  upstreamURI: string,\n) {\n  const {appID: id, publications} = shard;\n  const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n    replicaVersion: '', // ignored for initialSync()\n    publications,\n  });\n  const {changes} = changeSource.initialSync();\n\n  const processor = new ChangeProcessor(\n    new StatementRunner(tx),\n    'initial-sync',\n    (_, err) => {\n      throw err;\n    },\n  );\n\n  const statusPublisher = new ReplicationStatusPublisher(tx);\n  try {\n    let num = 0;\n    for await (const change of changes) {\n      const [tag] = change;\n      switch (tag) {\n        case 'begin': {\n          const {commitWatermark} = change[2];\n          lc.info?.(\n            `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n          );\n          statusPublisher.publish(\n            lc,\n            'Initializing',\n            `Copying upstream tables at version ${commitWatermark}`,\n            5000,\n          );\n          initReplicationState(tx, [...publications].sort(), commitWatermark);\n          processor.processMessage(lc, change);\n          break;\n        }\n        case 'data':\n          processor.processMessage(lc, change);\n          if (++num % 1000 === 0) {\n            lc.debug?.(`processed ${num} changes`);\n          }\n          break;\n        case 'commit':\n          processor.processMessage(lc, change);\n          validateInitiallySyncedData(lc, tx, shard);\n          lc.info?.(`finished initial-sync of ${num} changes`);\n          return;\n\n        case 'status':\n          break; // Ignored\n        // @ts-expect-error: falls through if the tag is not 'reset-required\n        case 'control': {\n          const {tag, message} = change[1];\n          if (tag === 'reset-required') {\n            throw new AutoResetSignal(\n              message ?? 'auto-reset signaled by change source',\n            );\n          }\n        }\n        // falls through\n        case 'rollback':\n          throw new Error(\n            `unexpected message during initial-sync: ${stringify(change)}`,\n          );\n        default:\n          unreachable(change);\n      }\n    }\n    throw new Error(\n      `change source ${upstreamURI} closed before initial-sync completed`,\n    );\n  } catch (e) {\n    await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n  } finally {\n    statusPublisher.stop();\n  }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n  appID,\n  shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n  return {\n    [`${appID}_${shardNum}.clients`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      lastMutationID: {type: 'number'},\n      userID: {type: 'string'},\n    },\n    [`${appID}_${shardNum}.mutations`]: {\n      clientGroupID: {type: 'string'},\n      clientID: {type: 'string'},\n      mutationID: {type: 'number'},\n      mutation: {type: 'json'},\n    },\n    [`${appID}.permissions`]: {\n      permissions: {type: 'json'},\n      hash: {type: 'string'},\n    },\n  };\n}\n\nfunction validateInitiallySyncedData(\n  lc: LogContext,\n  db: Database,\n  shard: ShardID,\n) {\n  const tables = computeZqlSpecs(lc, db);\n  const required = getRequiredTables(shard);\n  for (const [name, columns] of Object.entries(required)) {\n    const table = tables.get(name)?.zqlSpec;\n    if (!table) {\n      throw new Error(\n        `Upstream is missing the \"${name}\" table. (Found ${[\n          ...tables.keys(),\n        ]})` +\n          `Please ensure that each table has a unique index over one ` +\n          `or more non-null columns.`,\n      );\n    }\n    for (const [col, {type}] of Object.entries(columns)) {\n      const found = table[col];\n      if (!found) {\n        throw new Error(\n          `Upstream \"${table}\" table is missing the \"${col}\" column`,\n        );\n      }\n      if (found.type !== type) {\n        throw new Error(\n          `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n        );\n      }\n    }\n  }\n}\n"],"names":["tag"],"mappings":";;;;;;;;;;;;;;;AAkCA,eAAsB,6BACpB,IACA,aACA,OACA,eAC6E;AAC7E,QAAM;AAAA,IACJ;AAAA,IACA,WAAW,MAAM,KAAK,IAAI,MAAM,QAAQ;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,QAAM,UAAU,IAAI,SAAS,IAAI,aAAa;AAC9C,QAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,OAAO,CAAC;AAC3E,UAAQ,MAAA;AAER,MAAI,MAAM,aAAa,QAAQ;AAE7B,UAAM,YAAY,CAAC,GAAG,MAAM,YAAY,EAAE,KAAA;AAC1C,UAAM,aAAa,kBAAkB,aAAa,KAAA;AAClD,QAAI,CAAC,UAAU,WAAW,UAAU,GAAG;AACrC,YAAM,IAAI;AAAA,QACR,gDAAgD,SAAS,wCAAwC,UAAU;AAAA,MAAA;AAAA,IAE/G;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAAA,IACvB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EAAA;AAGF,SAAO,EAAC,mBAAmB,aAAA;AAC7B;AAEA,MAAM,mBAA2C;AAAA,EACtC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YACE,IACA,aACA,OACA,mBACA;AACA,SAAK,MAAM,GAAG,YAAY,aAAa,eAAe;AACtD,SAAK,eAAe;AACpB,SAAK,SAAS;AACd,SAAK,qBAAqB;AAAA,EAC5B;AAAA,EAEA,cAA4B;AAC1B,WAAO,KAAK,aAAA;AAAA,EACd;AAAA,EAEA,YAAY,iBAAgD;AAC1D,WAAO,QAAQ,QAAQ,KAAK,aAAa,eAAe,CAAC;AAAA,EAC3D;AAAA,EAEA,aAAa,iBAAwC;AACnD,UAAM,EAAC,cAAc,eAAA,IAAkB,KAAK;AAC5C,UAAM,EAAC,OAAO,SAAA,IAAY,KAAK;AAC/B,UAAM,MAAM,IAAI,IAAI,KAAK,YAAY;AACrC,QAAI,aAAa,IAAI,SAAS,KAAK;AACnC,QAAI,aAAa,IAAI,YAAY,OAAO,QAAQ,CAAC;AACjD,eAAW,OAAO,cAAc;AAC9B,UAAI,aAAa,OAAO,gBAAgB,GAAG;AAAA,IAC7C;AACA,QAAI,iBAAiB;AACnB;AAAA,QACE,eAAe;AAAA,QACf;AAAA,MAAA;AAEF,UAAI,aAAa,IAAI,iBAAiB,eAAe;AACrD,UAAI,aAAa,IAAI,kBAAkB,cAAc;AAAA,IACvD;AAEA,UAAM,KAAK,IAAI,UAAU,GAAG;AAC5B,UAAM,EAAC,UAAU,UAAA,IAAa;AAAA,MAC5B,KAAK;AAAA,MACL;AAAA,MACA;AAAA;AAAA;AAAA,MAGA,EAAC,UAAU,CAAC,SAA+B,KAAA;AAAA,IAAI;AAEjD,WAAO,EAAC,SAAS,UAAU,MAAM,UAAA;AAAA,EACnC;AACF;AAaA,eAAsB,YACpB,IACA,OACA,IACA,aACA;AACA,QAAM,EAAC,OAAO,IAAI,aAAA,IAAgB;AAClC,QAAM,eAAe,IAAI,mBAAmB,IAAI,aAAa,OAAO;AAAA,IAClE,gBAAgB;AAAA;AAAA,IAChB;AAAA,EAAA,CACD;AACD,QAAM,EAAC,QAAA,IAAW,aAAa,YAAA;AAE/B,QAAM,YAAY,IAAI;AAAA,IACpB,IAAI,gBAAgB,EAAE;AAAA,IACtB;AAAA,IACA,CAAC,GAAG,QAAQ;AACV,YAAM;AAAA,IACR;AAAA,EAAA;AAGF,QAAM,kBAAkB,IAAI,2BAA2B,EAAE;AACzD,MAAI;AACF,QAAI,MAAM;AACV,qBAAiB,UAAU,SAAS;AAClC,YAAM,CAAC,GAAG,IAAI;AACd,cAAQ,KAAA;AAAA,QACN,KAAK,SAAS;AACZ,gBAAM,EAAC,gBAAA,IAAmB,OAAO,CAAC;AAClC,aAAG;AAAA,YACD,yBAAyB,EAAE,sBAAsB,eAAe;AAAA,UAAA;AAElE,0BAAgB;AAAA,YACd;AAAA,YACA;AAAA,YACA,sCAAsC,eAAe;AAAA,YACrD;AAAA,UAAA;AAEF,+BAAqB,IAAI,CAAC,GAAG,YAAY,EAAE,KAAA,GAAQ,eAAe;AAClE,oBAAU,eAAe,IAAI,MAAM;AACnC;AAAA,QACF;AAAA,QACA,KAAK;AACH,oBAAU,eAAe,IAAI,MAAM;AACnC,cAAI,EAAE,MAAM,QAAS,GAAG;AACtB,eAAG,QAAQ,aAAa,GAAG,UAAU;AAAA,UACvC;AACA;AAAA,QACF,KAAK;AACH,oBAAU,eAAe,IAAI,MAAM;AACnC,sCAA4B,IAAI,IAAI,KAAK;AACzC,aAAG,OAAO,4BAA4B,GAAG,UAAU;AACnD;AAAA,QAEF,KAAK;AACH;AAAA;AAAA;AAAA,QAEF,KAAK,WAAW;AACd,gBAAM,EAAC,KAAAA,MAAK,QAAA,IAAW,OAAO,CAAC;AAC/B,cAAIA,SAAQ,kBAAkB;AAC5B,kBAAM,IAAI;AAAA,cACR,WAAW;AAAA,YAAA;AAAA,UAEf;AAAA,QACF;AAAA;AAAA,QAEA,KAAK;AACH,gBAAM,IAAI;AAAA,YACR,2CAA2C,UAAU,MAAM,CAAC;AAAA,UAAA;AAAA,QAEhE;AACE,sBAAY,MAAM;AAAA,MAAA;AAAA,IAExB;AACA,UAAM,IAAI;AAAA,MACR,iBAAiB,WAAW;AAAA,IAAA;AAAA,EAEhC,SAAS,GAAG;AACV,UAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,CAAC;AAAA,EAClE,UAAA;AACE,oBAAgB,KAAA;AAAA,EAClB;AACF;AAIA,SAAS,kBAAkB;AAAA,EACzB;AAAA,EACA;AACF,GAAyD;AACvD,SAAO;AAAA,IACL,CAAC,GAAG,KAAK,IAAI,QAAQ,UAAU,GAAG;AAAA,MAChC,eAAe,EAAC,MAAM,SAAA;AAAA,MACtB,UAAU,EAAC,MAAM,SAAA;AAAA,MACjB,gBAAgB,EAAC,MAAM,SAAA;AAAA,MACvB,QAAQ,EAAC,MAAM,SAAA;AAAA,IAAQ;AAAA,IAEzB,CAAC,GAAG,KAAK,IAAI,QAAQ,YAAY,GAAG;AAAA,MAClC,eAAe,EAAC,MAAM,SAAA;AAAA,MACtB,UAAU,EAAC,MAAM,SAAA;AAAA,MACjB,YAAY,EAAC,MAAM,SAAA;AAAA,MACnB,UAAU,EAAC,MAAM,OAAA;AAAA,IAAM;AAAA,IAEzB,CAAC,GAAG,KAAK,cAAc,GAAG;AAAA,MACxB,aAAa,EAAC,MAAM,OAAA;AAAA,MACpB,MAAM,EAAC,MAAM,SAAA;AAAA,IAAQ;AAAA,EACvB;AAEJ;AAEA,SAAS,4BACP,IACA,IACA,OACA;AACA,QAAM,SAAS,gBAAgB,IAAI,EAAE;AACrC,QAAM,WAAW,kBAAkB,KAAK;AACxC,aAAW,CAAC,MAAM,OAAO,KAAK,OAAO,QAAQ,QAAQ,GAAG;AACtD,UAAM,QAAQ,OAAO,IAAI,IAAI,GAAG;AAChC,QAAI,CAAC,OAAO;AACV,YAAM,IAAI;AAAA,QACR,4BAA4B,IAAI,mBAAmB;AAAA,UACjD,GAAG,OAAO,KAAA;AAAA,QAAK,CAChB;AAAA,MAAA;AAAA,IAIL;AACA,eAAW,CAAC,KAAK,EAAC,KAAA,CAAK,KAAK,OAAO,QAAQ,OAAO,GAAG;AACnD,YAAM,QAAQ,MAAM,GAAG;AACvB,UAAI,CAAC,OAAO;AACV,cAAM,IAAI;AAAA,UACR,aAAa,KAAK,2BAA2B,GAAG;AAAA,QAAA;AAAA,MAEpD;AACA,UAAI,MAAM,SAAS,MAAM;AACvB,cAAM,IAAI;AAAA,UACR,aAAa,KAAK,IAAI,GAAG,iBAAiB,MAAM,IAAI,uBAAuB,IAAI;AAAA,QAAA;AAAA,MAEnF;AAAA,IACF;AAAA,EACF;AACF;"}

package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAgBjD,OAAO,KAAK,EAEV,kBAAkB,EAEnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,kDAAkD,CAAC;AAE1D,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AAYtD,OAAO,EAAC,KAAK,kBAAkB,EAAC,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAoBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,GAC9B,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAoC7E;AA6PD,qBAAa,KAAK;;gBAIJ,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,SAAS;IAiBT,GAAG,CAAC,SAAS,EAAE,WAAW;CAa3B;AAycD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
+{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAkBjD,OAAO,KAAK,EAAa,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AAEzE,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,kDAAkD,CAAC;AAE1D,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,8CAA8C,CAAC;AAatD,OAAO,EAAC,KAAK,kBAAkB,EAAC,MAAM,mBAAmB,CAAC;AAC1D,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AAwBjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,GAC9B,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAoC7E;AA6PD,qBAAa,KAAK;;gBAIJ,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,SAAS;IAiBT,GAAG,CAAC,SAAS,EAAE,WAAW;CAa3B;AAieD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}

package/out/zero-cache/src/services/change-source/pg/change-source.js

@@ -1,6 +1,8 @@
 import { PG_OBJECT_IN_USE, PG_ADMIN_SHUTDOWN } from "@drdgvhbh/postgres-error-codes";
 import postgres from "postgres";
 import { AbortError } from "../../../../../shared/src/abort-error.js";
+import { areEqual } from "../../../../../shared/src/arrays.js";
+import { unreachable } from "../../../../../shared/src/asserts.js";
 import { stringify } from "../../../../../shared/src/bigint-json.js";
 import { deepEqual } from "../../../../../shared/src/json.js";
 import { must } from "../../../../../shared/src/must.js";
@@ -22,7 +24,6 @@ import "@rocicorp/resolver";
 import "@rocicorp/logger";
 import "../../../../../zero-protocol/src/primary-key.js";
 import "../../../observability/events.js";
-import "../../replicator/schema/change-log.js";
 import { fromBigInt, toLexiVersion } from "./lsn.js";
 import { updateShardSchema } from "./schema/init.js";
 import { getPublicationInfo } from "./schema/published.js";
@@ -340,7 +341,7 @@ class ChangeMaker {
             "data",
             {
               ...msg,
-              relation: withoutColumns(msg.relation),
+              relation: makeRelation(msg.relation),
               // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-DELETE
               key: must(msg.old ?? msg.key)
             }
@@ -353,7 +354,7 @@ class ChangeMaker {
             "data",
             {
               ...msg,
-              relation: withoutColumns(msg.relation),
+              relation: makeRelation(msg.relation),
               // https://www.postgresql.org/docs/current/protocol-logicalrep-message-formats.html#PROTOCOL-LOGICALREP-MESSAGE-FORMATS-UPDATE
               key: msg.old ?? msg.key
             }
@@ -361,11 +362,9 @@ class ChangeMaker {
         ];
       }
       case "insert":
-        return [["data", { ...msg, relation: withoutColumns(msg.relation) }]];
+        return [["data", { ...msg, relation: makeRelation(msg.relation) }]];
       case "truncate":
-        return [
-          ["data", { ...msg, relations: msg.relations.map(withoutColumns) }]
-        ];
+        return [["data", { ...msg, relations: msg.relations.map(makeRelation) }]];
       case "message":
         if (msg.prefix !== this.#shardPrefix) {
           this.#lc.debug?.("ignoring message for different shard", msg.prefix);
@@ -447,7 +446,7 @@ class ChangeMaker {
       const [nextTbl, nextIdx] = specsByID(update.schema);
       const changes = [];
       for (const table of nextTbl.values()) {
-        validate(this.#lc, table, update.schema.indexes);
+        validate(this.#lc, table);
       }
       const [droppedIdx, createdIdx] = symmetricDifferences(prevIdx, nextIdx);
       for (const id of droppedIdx) {
@@ -470,7 +469,11 @@ class ChangeMaker {
       }
       for (const id of createdTbl) {
         const spec = must(nextTbl.get(id));
-        changes.push({ tag: "create-table", spec });
+        changes.push({
+          tag: "create-table",
+          spec,
+          metadata: getMetadata(spec)
+        });
       }
       for (const id of createdIdx) {
         const spec = must(nextIdx.get(id));
@@ -490,6 +493,17 @@ class ChangeMaker {
         new: { schema: newTable.schema, name: newTable.name }
       });
     }
+    if (oldTable.replicaIdentity !== newTable.replicaIdentity || !areEqual(
+      oldTable.replicaIdentityColumns,
+      newTable.replicaIdentityColumns
+    )) {
+      changes.push({
+        tag: "update-table-metadata",
+        table: { schema: newTable.schema, name: newTable.name },
+        old: getMetadata(oldTable),
+        new: getMetadata(newTable)
+      });
+    }
     const table = { schema: newTable.schema, name: newTable.name };
     const oldColumns = columnsByID(oldTable.columns);
     const newColumns = columnsByID(newTable.columns);
@@ -515,7 +529,12 @@ class ChangeMaker {
       const { name, ...spec } = must(newColumns.get(id));
       const column = { name, spec };
       mapPostgresToLiteColumn(table.name, column);
-      changes.push({ tag: "add-column", table, column });
+      changes.push({
+        tag: "add-column",
+        table,
+        column,
+        tableMetadata: getMetadata(newTable)
+      });
     }
     return changes;
   }
@@ -649,9 +668,45 @@ function columnsByID(columns) {
   }
   return colsByID;
 }
-function withoutColumns(relation) {
-  const { columns: _, ...rest } = relation;
-  return rest;
+function getMetadata(table) {
+  const metadata = {
+    rowKey: {
+      columns: table.replicaIdentityColumns
+    }
+  };
+  switch (table.replicaIdentity) {
+    case "d":
+      metadata.rowKey.type = "default";
+      break;
+    case "i":
+      metadata.rowKey.type = "index";
+      break;
+    case "f":
+      metadata.rowKey.type = "full";
+      break;
+    case "n":
+      metadata.rowKey.type = "nothing";
+      break;
+    case void 0:
+      break;
+    default:
+      unreachable(table.replicaIdentity);
+  }
+  return metadata;
+}
+function makeRelation(relation) {
+  const { columns: _, keyColumns, replicaIdentity, ...rest } = relation;
+  return {
+    ...rest,
+    rowKey: {
+      columns: keyColumns,
+      type: replicaIdentity
+    },
+    // For now, deprecated columns are sent for backwards compatibility.
+    // These can be removed when bumping the MIN_PROTOCOL_VERSION to 5.
+    keyColumns,
+    replicaIdentity
+  };
 }
 class UnsupportedSchemaChangeError extends Error {
   name = "UnsupportedSchemaChangeError";