@rocicorp/zero 1.6.0-canary.3 → 1.6.0-canary.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/replicache/src/dag/store-impl.d.ts.map +1 -1
- package/out/replicache/src/dag/store-impl.js +8 -3
- package/out/replicache/src/dag/store-impl.js.map +1 -1
- package/out/replicache/src/persist/persist.d.ts.map +1 -1
- package/out/replicache/src/persist/persist.js +4 -2
- package/out/replicache/src/persist/persist.js.map +1 -1
- package/out/replicache/src/sync/pull.d.ts.map +1 -1
- package/out/replicache/src/sync/pull.js +9 -6
- package/out/replicache/src/sync/pull.js.map +1 -1
- package/out/shared/src/map.d.ts +6 -0
- package/out/shared/src/map.d.ts.map +1 -0
- package/out/shared/src/map.js +39 -0
- package/out/shared/src/map.js.map +1 -0
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +8 -5
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +4 -2
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/mutator.js +4 -2
- package/out/zero-cache/src/server/mutator.js.map +1 -1
- package/out/zero-cache/src/server/reaper.d.ts.map +1 -1
- package/out/zero-cache/src/server/reaper.js +4 -2
- package/out/zero-cache/src/server/reaper.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +4 -2
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/runner/main.js +2 -1
- package/out/zero-cache/src/server/runner/main.js.map +1 -1
- package/out/zero-cache/src/server/shadow-syncer.js +6 -3
- package/out/zero-cache/src/server/shadow-syncer.js.map +1 -1
- package/out/zero-cache/src/server/syncer.d.ts.map +1 -1
- package/out/zero-cache/src/server/syncer.js +4 -2
- package/out/zero-cache/src/server/syncer.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts +8 -2
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +9 -8
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +3 -3
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +10 -3
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +18 -7
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts +32 -0
- package/out/zql/src/ivm/cap.d.ts.map +1 -0
- package/out/zql/src/ivm/cap.js +205 -0
- package/out/zql/src/ivm/cap.js.map +1 -0
- package/out/zql/src/ivm/memory-source.js +1 -1
- package/out/zql/src/ivm/take.js +2 -2
- package/out/zql/src/mutate/mutator.d.ts +12 -3
- package/out/zql/src/mutate/mutator.d.ts.map +1 -1
- package/out/zql/src/mutate/mutator.js.map +1 -1
- package/out/zql/src/query/query-registry.d.ts +10 -3
- package/out/zql/src/query/query-registry.d.ts.map +1 -1
- package/out/zql/src/query/query-registry.js.map +1 -1
- package/out/zqlite/src/table-source.js +1 -1
- package/package.json +1 -1
|
@@ -10,13 +10,15 @@ import { startOtelAuto } from "./otel-start.js";
|
|
|
10
10
|
import { createNotifierFrom, handleSubscriptionsFrom, subscribeTo } from "../workers/replicator.js";
|
|
11
11
|
import { WorkerDispatcher } from "./worker-dispatcher.js";
|
|
12
12
|
import { resolver } from "@rocicorp/resolver";
|
|
13
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
13
14
|
import path from "node:path";
|
|
14
15
|
//#region ../zero-cache/src/server/main.ts
|
|
16
|
+
var lc = new LogContext("info", {}, consoleLogSink);
|
|
15
17
|
async function runWorker(parent, env) {
|
|
16
18
|
const startMs = Date.now();
|
|
17
19
|
const config = getNormalizedZeroConfig({ env });
|
|
18
20
|
startOtelAuto(createLogContext(config, "dispatcher", 0, false), "dispatcher", 0);
|
|
19
|
-
|
|
21
|
+
lc = createLogContext(config, "dispatcher");
|
|
20
22
|
initEventSink(lc, config);
|
|
21
23
|
const processes = new ProcessManager(lc, parent);
|
|
22
24
|
const { numSyncWorkers: numSyncers } = config;
|
|
@@ -90,7 +92,7 @@ async function runWorker(parent, env) {
|
|
|
90
92
|
}
|
|
91
93
|
await processes.done();
|
|
92
94
|
}
|
|
93
|
-
if (!singleProcessMode()) exitAfter(() => runWorker(must(parentWorker), process.env));
|
|
95
|
+
if (!singleProcessMode()) exitAfter(lc, () => runWorker(must(parentWorker), process.env));
|
|
94
96
|
//#endregion
|
|
95
97
|
export { runWorker as default };
|
|
96
98
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n restoreReplica,\n startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SHADOW_SYNCER_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(\n createLogContext(config, 'dispatcher', 0, false),\n 'dispatcher',\n 0,\n );\n
|
|
1
|
+
{"version":3,"file":"main.js","names":[],"sources":["../../../../../zero-cache/src/server/main.ts"],"sourcesContent":["import path from 'node:path';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {\n exitAfter,\n ProcessManager,\n runUntilKilled,\n type WorkerType,\n} from '../services/life-cycle.ts';\nimport {\n restoreReplica,\n startReplicaBackupProcess,\n} from '../services/litestream/commands.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {\n createNotifierFrom,\n handleSubscriptionsFrom,\n type ReplicaFileMode,\n subscribeTo,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {WorkerDispatcher} from './worker-dispatcher.ts';\nimport {\n CHANGE_STREAMER_URL,\n MUTATOR_URL,\n REAPER_URL,\n REPLICATOR_URL,\n SHADOW_SYNCER_URL,\n SYNCER_URL,\n} from './worker-urls.ts';\n\nconst clientConnectionBifurcated = false;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n): Promise<void> {\n const startMs = Date.now();\n const config = getNormalizedZeroConfig({env});\n\n startOtelAuto(\n createLogContext(config, 'dispatcher', 0, false),\n 'dispatcher',\n 0,\n );\n lc = createLogContext(config, 'dispatcher');\n initEventSink(lc, config);\n\n const processes = new ProcessManager(lc, parent);\n\n const {numSyncWorkers: numSyncers} = config;\n if (config.enableCrudMutations && config.upstream.maxConns < numSyncers) {\n throw new Error(\n `Insufficient upstream connections (${config.upstream.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_UPSTREAM_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n if (config.cvr.maxConns < numSyncers) {\n throw new Error(\n `Insufficient cvr connections (${config.cvr.maxConns}) for ${numSyncers} syncers.` +\n `Increase ZERO_CVR_MAX_CONNS or decrease ZERO_NUM_SYNC_WORKERS (which defaults to available cores).`,\n );\n }\n\n const internalFlags: string[] =\n numSyncers === 0\n ? []\n : [\n '--upstream-max-conns-per-worker',\n String(Math.floor(config.upstream.maxConns / numSyncers)),\n '--cvr-max-conns-per-worker',\n String(Math.floor(config.cvr.maxConns / numSyncers)),\n ];\n\n function loadWorker(\n moduleUrl: URL,\n type: WorkerType,\n id?: string | number,\n ...args: string[]\n ): Worker {\n const worker = childWorker(moduleUrl, env, ...args, ...internalFlags);\n const name = path.basename(moduleUrl.pathname) + (id ? ` (${id})` : '');\n return processes.addWorker(worker, type, name);\n }\n\n const {\n taskID,\n changeStreamer: {mode: changeStreamerMode, uri: changeStreamerURI},\n litestream,\n } = config;\n const runChangeStreamer =\n changeStreamerMode === 'dedicated' && changeStreamerURI === undefined;\n\n let changeStreamer: Worker | undefined;\n\n if (!runChangeStreamer) {\n changeStreamer = undefined;\n if (litestream.executable) {\n // For view-syncers, the backup is restored here. For the replication-manager,\n // the backup is restored in the change-streamer worker.\n await restoreReplica(lc, config, null);\n }\n } else {\n const {promise: changeStreamerReady, resolve: changeStreamerStarted} =\n resolver();\n changeStreamer = loadWorker(CHANGE_STREAMER_URL, 'supporting').once(\n 'message',\n changeStreamerStarted,\n );\n\n // Wait for the change-streamer to be ready to guarantee that a replica\n // file is present.\n await changeStreamerReady;\n\n if (litestream.backupURL) {\n // Start a backup replicator and corresponding litestream backup process.\n const {promise: backupReady, resolve} = resolver();\n const mode: ReplicaFileMode = 'backup';\n loadWorker(REPLICATOR_URL, 'supporting', mode, mode).once(\n // Wait for the Replicator's first message (i.e. \"ready\") before starting\n // litestream backup in order to avoid contending on the lock when the\n // replicator first prepares the db file.\n 'message',\n () => {\n processes.addSubprocess(\n startReplicaBackupProcess(lc, config),\n 'supporting',\n 'litestream',\n );\n resolve();\n },\n );\n await backupReady;\n }\n }\n\n if (numSyncers > 0) {\n const {promise: reaperReady, resolve: reaperStarted} = resolver();\n loadWorker(REAPER_URL, 'supporting').once('message', reaperStarted);\n // Before starting the view-syncers, ensure that the reaper has started\n // up, indicating that any CVR db migrations have been performed.\n await reaperReady;\n }\n\n // Only run the shadow-sync canary on the replication-manager (or in\n // single-node mode, where it also owns upstream). Running on every\n // view-syncer would hammer the upstream with N redundant canaries.\n if (config.shadowSync.enabled && runChangeStreamer) {\n const {promise: shadowReady, resolve: shadowStarted} = resolver();\n loadWorker(SHADOW_SYNCER_URL, 'supporting').once('message', shadowStarted);\n await shadowReady;\n }\n\n const syncers: Worker[] = [];\n if (numSyncers) {\n const mode: ReplicaFileMode =\n runChangeStreamer && litestream.backupURL ? 'serving-copy' : 'serving';\n const {promise: replicaReady, resolve} = resolver();\n const replicator = loadWorker(\n REPLICATOR_URL,\n 'supporting',\n mode,\n mode,\n ).once('message', () => {\n subscribeTo(lc, replicator);\n resolve();\n });\n await replicaReady;\n\n const notifier = createNotifierFrom(lc, replicator);\n for (let i = 0; i < numSyncers; i++) {\n syncers.push(loadWorker(SYNCER_URL, 'user-facing', i, mode, String(i)));\n }\n syncers.forEach(syncer => handleSubscriptionsFrom(lc, syncer, notifier));\n }\n let mutator: Worker | undefined;\n if (clientConnectionBifurcated) {\n mutator = loadWorker(MUTATOR_URL, 'supporting', 'mutator');\n }\n\n lc.info?.('waiting for workers to be ready ...');\n const logWaiting = setInterval(\n () => lc.info?.(`still waiting for ${processes.initializing().join(', ')}`),\n 10_000,\n );\n await processes.allWorkersReady();\n clearInterval(logWaiting);\n lc.info?.(`all workers ready (${Date.now() - startMs} ms)`);\n\n parent.send(['ready', {ready: true}]);\n\n try {\n await runUntilKilled(\n lc,\n parent,\n new WorkerDispatcher(\n lc,\n taskID,\n parent,\n syncers,\n mutator,\n changeStreamer,\n ),\n );\n } catch (err) {\n processes.logErrorAndExit(err, 'dispatcher');\n }\n\n await processes.done();\n}\n\nif (!singleProcessMode()) {\n void exitAfter(lc, () => runWorker(must(parentWorker), process.env));\n}\n"],"mappings":";;;;;;;;;;;;;;;AA2CA,IAAI,KAAK,IAAI,WAAW,QAAQ,EAAE,EAAE,eAAe;AAEnD,eAA8B,UAC5B,QACA,KACe;CACf,MAAM,UAAU,KAAK,KAAK;CAC1B,MAAM,SAAS,wBAAwB,EAAC,KAAI,CAAC;AAE7C,eACE,iBAAiB,QAAQ,cAAc,GAAG,MAAM,EAChD,cACA,EACD;AACD,MAAK,iBAAiB,QAAQ,aAAa;AAC3C,eAAc,IAAI,OAAO;CAEzB,MAAM,YAAY,IAAI,eAAe,IAAI,OAAO;CAEhD,MAAM,EAAC,gBAAgB,eAAc;AACrC,KAAI,OAAO,uBAAuB,OAAO,SAAS,WAAW,WAC3D,OAAM,IAAI,MACR,sCAAsC,OAAO,SAAS,SAAS,QAAQ,WAAW,kHAEnF;AAEH,KAAI,OAAO,IAAI,WAAW,WACxB,OAAM,IAAI,MACR,iCAAiC,OAAO,IAAI,SAAS,QAAQ,WAAW,6GAEzE;CAGH,MAAM,gBACJ,eAAe,IACX,EAAE,GACF;EACE;EACA,OAAO,KAAK,MAAM,OAAO,SAAS,WAAW,WAAW,CAAC;EACzD;EACA,OAAO,KAAK,MAAM,OAAO,IAAI,WAAW,WAAW,CAAC;EACrD;CAEP,SAAS,WACP,WACA,MACA,IACA,GAAG,MACK;EACR,MAAM,SAAS,YAAY,WAAW,KAAK,GAAG,MAAM,GAAG,cAAc;EACrE,MAAM,OAAO,KAAK,SAAS,UAAU,SAAS,IAAI,KAAK,KAAK,GAAG,KAAK;AACpE,SAAO,UAAU,UAAU,QAAQ,MAAM,KAAK;;CAGhD,MAAM,EACJ,QACA,gBAAgB,EAAC,MAAM,oBAAoB,KAAK,qBAChD,eACE;CACJ,MAAM,oBACJ,uBAAuB,eAAe,sBAAsB,KAAA;CAE9D,IAAI;AAEJ,KAAI,CAAC,mBAAmB;AACtB,mBAAiB,KAAA;AACjB,MAAI,WAAW,WAGb,OAAM,eAAe,IAAI,QAAQ,KAAK;QAEnC;EACL,MAAM,EAAC,SAAS,qBAAqB,SAAS,0BAC5C,UAAU;AACZ,mBAAiB,WAAW,qBAAqB,aAAa,CAAC,KAC7D,WACA,sBACD;AAID,QAAM;AAEN,MAAI,WAAW,WAAW;GAExB,MAAM,EAAC,SAAS,aAAa,YAAW,UAAU;GAClD,MAAM,OAAwB;AAC9B,cAAW,gBAAgB,cAAc,MAAM,KAAK,CAAC,KAInD,iBACM;AACJ,cAAU,cACR,0BAA0B,IAAI,OAAO,EACrC,cACA,aACD;AACD,aAAS;KAEZ;AACD,SAAM;;;AAIV,KAAI,aAAa,GAAG;EAClB,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,UAAU;AACjE,aAAW,YAAY,aAAa,CAAC,KAAK,WAAW,cAAc;AAGnE,QAAM;;AAMR,KAAI,OAAO,WAAW,WAAW,mBAAmB;EAClD,MAAM,EAAC,SAAS,aAAa,SAAS,kBAAiB,UAAU;AACjE,aAAW,mBAAmB,aAAa,CAAC,KAAK,WAAW,cAAc;AAC1E,QAAM;;CAGR,MAAM,UAAoB,EAAE;AAC5B,KAAI,YAAY;EACd,MAAM,OACJ,qBAAqB,WAAW,YAAY,iBAAiB;EAC/D,MAAM,EAAC,SAAS,cAAc,YAAW,UAAU;EACnD,MAAM,aAAa,WACjB,gBACA,cACA,MACA,KACD,CAAC,KAAK,iBAAiB;AACtB,eAAY,IAAI,WAAW;AAC3B,YAAS;IACT;AACF,QAAM;EAEN,MAAM,WAAW,mBAAmB,IAAI,WAAW;AACnD,OAAK,IAAI,IAAI,GAAG,IAAI,YAAY,IAC9B,SAAQ,KAAK,WAAW,YAAY,eAAe,GAAG,MAAM,OAAO,EAAE,CAAC,CAAC;AAEzE,UAAQ,SAAQ,WAAU,wBAAwB,IAAI,QAAQ,SAAS,CAAC;;CAE1E,IAAI;AAKJ,IAAG,OAAO,sCAAsC;CAChD,MAAM,aAAa,kBACX,GAAG,OAAO,qBAAqB,UAAU,cAAc,CAAC,KAAK,KAAK,GAAG,EAC3E,IACD;AACD,OAAM,UAAU,iBAAiB;AACjC,eAAc,WAAW;AACzB,IAAG,OAAO,sBAAsB,KAAK,KAAK,GAAG,QAAQ,MAAM;AAE3D,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAErC,KAAI;AACF,QAAM,eACJ,IACA,QACA,IAAI,iBACF,IACA,QACA,QACA,SACA,SACA,eACD,CACF;UACM,KAAK;AACZ,YAAU,gBAAgB,KAAK,aAAa;;AAG9C,OAAM,UAAU,MAAM;;AAGxB,IAAI,CAAC,mBAAmB,CACjB,WAAU,UAAU,UAAU,KAAK,aAAa,EAAE,QAAQ,IAAI,CAAC"}
|
|
@@ -6,18 +6,20 @@ import { createLogContext } from "./logging.js";
|
|
|
6
6
|
import { initEventSink } from "../observability/events.js";
|
|
7
7
|
import { startOtelAuto } from "./otel-start.js";
|
|
8
8
|
import { Mutator } from "../workers/mutator.js";
|
|
9
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
9
10
|
//#region ../zero-cache/src/server/mutator.ts
|
|
11
|
+
var lc = new LogContext("info", {}, consoleLogSink);
|
|
10
12
|
function runWorker(parent, env, ...args) {
|
|
11
13
|
const config = getNormalizedZeroConfig({
|
|
12
14
|
env,
|
|
13
15
|
argv: args.slice(1)
|
|
14
16
|
});
|
|
15
17
|
startOtelAuto(createLogContext(config, "mutator", 0, false), "mutator", 0);
|
|
16
|
-
|
|
18
|
+
lc = createLogContext(config, "mutator");
|
|
17
19
|
initEventSink(lc, config);
|
|
18
20
|
return runUntilKilled(lc, parent, new Mutator());
|
|
19
21
|
}
|
|
20
|
-
if (!singleProcessMode()) exitAfter(() => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
22
|
+
if (!singleProcessMode()) exitAfter(lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
21
23
|
//#endregion
|
|
22
24
|
export {};
|
|
23
25
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nfunction runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n startOtelAuto(createLogContext(config, 'mutator', 0, false), 'mutator', 0);\n
|
|
1
|
+
{"version":3,"file":"mutator.js","names":[],"sources":["../../../../../zero-cache/src/server/mutator.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {Mutator} from '../workers/mutator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nfunction runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n startOtelAuto(createLogContext(config, 'mutator', 0, false), 'mutator', 0);\n lc = createLogContext(config, 'mutator');\n initEventSink(lc, config);\n\n // TODO: create `PusherFactory`\n return runUntilKilled(lc, parent, new Mutator());\n}\n\nif (!singleProcessMode()) {\n void exitAfter(lc, () =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;AAeA,IAAI,KAAK,IAAI,WAAW,QAAQ,EAAE,EAAE,eAAe;AAEnD,SAAS,UACP,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;AAClE,eAAc,iBAAiB,QAAQ,WAAW,GAAG,MAAM,EAAE,WAAW,EAAE;AAC1E,MAAK,iBAAiB,QAAQ,UAAU;AACxC,eAAc,IAAI,OAAO;AAGzB,QAAO,eAAe,IAAI,QAAQ,IAAI,SAAS,CAAC;;AAGlD,IAAI,CAAC,mBAAmB,CACjB,WAAU,UACb,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reaper.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/reaper.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"reaper.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/reaper.ts"],"names":[],"mappings":"AASA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAW/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA+Bf"}
|
|
@@ -11,15 +11,17 @@ import { startAnonymousTelemetry } from "./anonymous-otel-start.js";
|
|
|
11
11
|
import { ActiveUsersGauge } from "../services/view-syncer/active-users-gauge.js";
|
|
12
12
|
import { CVRPurger } from "../services/view-syncer/cvr-purger.js";
|
|
13
13
|
import { initViewSyncerSchema } from "../services/view-syncer/schema/init.js";
|
|
14
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
14
15
|
//#region ../zero-cache/src/server/reaper.ts
|
|
15
16
|
var MS_PER_HOUR = 1e3 * 60 * 60;
|
|
17
|
+
var lc = new LogContext("info", {}, consoleLogSink);
|
|
16
18
|
async function runWorker(parent, env, ...argv) {
|
|
17
19
|
const config = getNormalizedZeroConfig({
|
|
18
20
|
env,
|
|
19
21
|
argv
|
|
20
22
|
});
|
|
21
23
|
startOtelAuto(createLogContext(config, "reaper", 0, false), "reaper", 0);
|
|
22
|
-
|
|
24
|
+
lc = createLogContext(config, "reaper");
|
|
23
25
|
initEventSink(lc, config);
|
|
24
26
|
startAnonymousTelemetry(lc, config);
|
|
25
27
|
const { cvr } = config;
|
|
@@ -33,7 +35,7 @@ async function runWorker(parent, env, ...argv) {
|
|
|
33
35
|
initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1e3
|
|
34
36
|
}), new ActiveUsersGauge(lc, cvrDB, shard, { updateIntervalMs: 600 * 1e3 }));
|
|
35
37
|
}
|
|
36
|
-
if (!singleProcessMode()) exitAfter(() => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
38
|
+
if (!singleProcessMode()) exitAfter(lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
37
39
|
//#endregion
|
|
38
40
|
export { runWorker as default };
|
|
39
41
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n\n startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n
|
|
1
|
+
{"version":3,"file":"reaper.js","names":[],"sources":["../../../../../zero-cache/src/server/reaper.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ActiveUsersGauge} from '../services/view-syncer/active-users-gauge.ts';\nimport {CVRPurger} from '../services/view-syncer/cvr-purger.ts';\nimport {initViewSyncerSchema} from '../services/view-syncer/schema/init.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n\n startOtelAuto(createLogContext(config, 'reaper', 0, false), 'reaper', 0);\n lc = createLogContext(config, 'reaper');\n initEventSink(lc, config);\n startAnonymousTelemetry(lc, config);\n\n const {cvr} = config;\n const shard = getShardID(config);\n const cvrDB = pgClient(lc, cvr.db, `sync-cvr-purger`, {\n max: 1,\n });\n await initViewSyncerSchema(lc, cvrDB, shard);\n parent.send(['ready', {ready: true}]);\n\n return runUntilKilled(\n lc,\n parent,\n new CVRPurger(lc, cvrDB, shard, {\n inactivityThresholdMs:\n cvr.garbageCollectionInactivityThresholdHours * MS_PER_HOUR,\n initialBatchSize: cvr.garbageCollectionInitialBatchSize,\n initialIntervalMs: cvr.garbageCollectionInitialIntervalSeconds * 1000,\n }),\n // Periodically computes and exports active users gauge to anonymous telemetry\n new ActiveUsersGauge(lc, cvrDB, shard, {\n // Default 10minutes refresh; can be made configurable later if needed\n updateIntervalMs: 10 * 60 * 1000,\n }),\n );\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(lc, () =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;AAmBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,EAAE,EAAE,eAAe;AAEnD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;EAAK,CAAC;AAEnD,eAAc,iBAAiB,QAAQ,UAAU,GAAG,MAAM,EAAE,UAAU,EAAE;AACxE,MAAK,iBAAiB,QAAQ,SAAS;AACvC,eAAc,IAAI,OAAO;AACzB,yBAAwB,IAAI,OAAO;CAEnC,MAAM,EAAC,QAAO;CACd,MAAM,QAAQ,WAAW,OAAO;CAChC,MAAM,QAAQ,SAAS,IAAI,IAAI,IAAI,mBAAmB,EACpD,KAAK,GACN,CAAC;AACF,OAAM,qBAAqB,IAAI,OAAO,MAAM;AAC5C,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAErC,QAAO,eACL,IACA,QACA,IAAI,UAAU,IAAI,OAAO,OAAO;EAC9B,uBACE,IAAI,4CAA4C;EAClD,kBAAkB,IAAI;EACtB,mBAAmB,IAAI,0CAA0C;EAClE,CAAC,EAEF,IAAI,iBAAiB,IAAI,OAAO,OAAO,EAErC,kBAAkB,MAAU,KAC7B,CAAC,CACH;;AAIH,IAAI,CAAC,mBAAmB,CACjB,WAAU,UACb,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAkBA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAkBA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAe/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAoEf"}
|
|
@@ -14,9 +14,11 @@ import { startOtelAuto } from "./otel-start.js";
|
|
|
14
14
|
import { ThreadWriteWorkerClient } from "../services/replicator/write-worker-client.js";
|
|
15
15
|
import { getPragmaConfig, replicaFileModeSchema, setUpMessageHandlers, setupReplica } from "../workers/replicator.js";
|
|
16
16
|
import { ReplicatorService } from "../services/replicator/replicator.js";
|
|
17
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
17
18
|
import { pid } from "node:process";
|
|
18
19
|
import { stat } from "node:fs/promises";
|
|
19
20
|
//#region ../zero-cache/src/server/replicator.ts
|
|
21
|
+
var lc = new LogContext("info", {}, consoleLogSink);
|
|
20
22
|
async function runWorker(parent, env, ...args) {
|
|
21
23
|
assert(args.length > 0, `replicator mode not specified`);
|
|
22
24
|
const fileMode = parse(args[0], replicaFileModeSchema);
|
|
@@ -27,7 +29,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
27
29
|
const mode = fileMode === "backup" ? "backup" : "serving";
|
|
28
30
|
const workerName = `${mode}-replicator`;
|
|
29
31
|
startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);
|
|
30
|
-
|
|
32
|
+
lc = createLogContext(config, workerName);
|
|
31
33
|
initEventSink(lc, config);
|
|
32
34
|
const { file: dbPath, walMode } = await setupReplica(lc, fileMode, config.replica);
|
|
33
35
|
setupMetrics(lc, dbPath, walMode);
|
|
@@ -71,7 +73,7 @@ function observeFileSize(lc, file) {
|
|
|
71
73
|
}
|
|
72
74
|
};
|
|
73
75
|
}
|
|
74
|
-
if (!singleProcessMode()) exitAfter(() => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
76
|
+
if (!singleProcessMode()) exitAfter(lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
75
77
|
//#endregion
|
|
76
78
|
export { runWorker as default };
|
|
77
79
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport
|
|
1
|
+
{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const workerName = `${mode}-replicator`;\n startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n lc = createLogContext(config, workerName);\n initEventSink(lc, config);\n\n const {file: dbPath, walMode} = await setupReplica(\n lc,\n fileMode,\n config.replica,\n );\n\n setupMetrics(lc, dbPath, walMode);\n\n // Create the write worker for async SQLite writes.\n const pragmas = getPragmaConfig(fileMode);\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log);\n\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n const running = runUntilKilled(lc, parent, replicator);\n\n // Signal readiness once the first ReplicaVersionReady notification is received.\n for await (const _ of replicator.subscribe()) {\n parent.send(['ready', {ready: true}]);\n break;\n }\n\n return running;\n}\n\nfunction setupMetrics(lc: LogContext, file: string, walMode: WalMode) {\n getOrCreateGauge('replica', 'db_size', {\n description:\n `The size of the replica's main db file, ` +\n `which does not include the wal file(s)`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, file));\n\n getOrCreateGauge('replica', 'wal_size', {\n description: `The size of the replica's wal file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n if (walMode === 'wal2') {\n getOrCreateGauge('replica', 'wal2_size', {\n description: `The size of the replica's wal2 file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal2`));\n }\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n return async o => {\n try {\n const stats = await stat(file);\n o.observe(stats.size);\n } catch (e) {\n lc.warn?.(`unable to stat ${file} for size metrics`, e);\n }\n };\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(lc, () =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAmCA,IAAI,KAAK,IAAI,WAAW,QAAQ,EAAE,EAAE,eAAe;AAEnD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;AACf,QAAO,KAAK,SAAS,GAAG,gCAAgC;CACxD,MAAM,WAAW,MAAQ,KAAK,IAAI,sBAAsB;CAExD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;CAClE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,aAAa,GAAG,KAAK;AAC3B,eAAc,iBAAiB,QAAQ,YAAY,GAAG,MAAM,EAAE,YAAY,EAAE;AAC5E,MAAK,iBAAiB,QAAQ,WAAW;AACzC,eAAc,IAAI,OAAO;CAEzB,MAAM,EAAC,MAAM,QAAQ,YAAW,MAAM,aACpC,IACA,UACA,OAAO,QACR;AAED,cAAa,IAAI,QAAQ,QAAQ;CAGjC,MAAM,UAAU,gBAAgB,SAAS;CACzC,MAAM,eAAe,IAAI,yBAAyB;AAClD,OAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,IAAI;CAE1D,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,QAAQ,eAAe,OAAO;CACpC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,kBACD;CAED,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,OAAO,GACjD,KACL;AAED,sBAAqB,IAAI,YAAY,OAAO;CAE5C,MAAM,UAAU,eAAe,IAAI,QAAQ,WAAW;AAGtD,YAAW,MAAM,KAAK,WAAW,WAAW,EAAE;AAC5C,SAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AACrC;;AAGF,QAAO;;AAGT,SAAS,aAAa,IAAgB,MAAc,SAAkB;AACpE,kBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;EACP,CAAC,CAAC,YAAY,gBAAgB,IAAI,KAAK,CAAC;AAEzC,kBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;EACP,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;AAElD,KAAI,YAAY,OACd,kBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;EACP,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,OAAO,CAAC;;AAIvD,SAAS,gBAAgB,IAAgB,MAAkC;AACzE,QAAO,OAAM,MAAK;AAChB,MAAI;GACF,MAAM,QAAQ,MAAM,KAAK,KAAK;AAC9B,KAAE,QAAQ,MAAM,KAAK;WACd,GAAG;AACV,MAAG,OAAO,kBAAkB,KAAK,oBAAoB,EAAE;;;;AAM7D,IAAI,CAAC,mBAAmB,CACjB,WAAU,UACb,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/main.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/server/runner/main.ts"],"names":[],"mappings":"AACA,OAAO,kCAAkC,CAAC;AAI1C,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAQ1C,eAAe,SAAS,CAAC"}
|
|
@@ -2,8 +2,9 @@ import "../../../../shared/src/dotenv.js";
|
|
|
2
2
|
import { parentWorker, singleProcessMode } from "../../types/processes.js";
|
|
3
3
|
import { exitAfter } from "../../services/life-cycle.js";
|
|
4
4
|
import { runWorker } from "./run-worker.js";
|
|
5
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
5
6
|
//#region ../zero-cache/src/server/runner/main.ts
|
|
6
|
-
if (!singleProcessMode()) exitAfter(() => runWorker(parentWorker, process.env));
|
|
7
|
+
if (!singleProcessMode()) exitAfter(new LogContext("info", {}, consoleLogSink), () => runWorker(parentWorker, process.env));
|
|
7
8
|
//#endregion
|
|
8
9
|
export {};
|
|
9
10
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/main.ts"],"sourcesContent":["import '../../../../shared/src/dotenv.ts';\n\nimport {exitAfter} from '../../services/life-cycle.ts';\nimport {parentWorker, singleProcessMode} from '../../types/processes.ts';\nimport {runWorker} from './run-worker.ts';\n\nif (!singleProcessMode()) {\n void exitAfter(()
|
|
1
|
+
{"version":3,"file":"main.js","names":[],"sources":["../../../../../../zero-cache/src/server/runner/main.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport '../../../../shared/src/dotenv.ts';\n\nimport {exitAfter} from '../../services/life-cycle.ts';\nimport {parentWorker, singleProcessMode} from '../../types/processes.ts';\nimport {runWorker} from './run-worker.ts';\n\nif (!singleProcessMode()) {\n void exitAfter(new LogContext('info', {}, consoleLogSink), () =>\n runWorker(parentWorker, process.env),\n );\n}\n\nexport default runWorker;\n"],"mappings":";;;;;;AAOA,IAAI,CAAC,mBAAmB,CACjB,WAAU,IAAI,WAAW,QAAQ,EAAE,EAAE,eAAe,QACvD,UAAU,cAAc,QAAQ,IAAI,CACrC"}
|
|
@@ -8,18 +8,21 @@ import { getServerContext } from "../config/server-context.js";
|
|
|
8
8
|
import { initEventSink } from "../observability/events.js";
|
|
9
9
|
import { startOtelAuto } from "./otel-start.js";
|
|
10
10
|
import { ShadowSyncService } from "../services/shadow-sync/shadow-sync-service.js";
|
|
11
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
11
12
|
//#region ../zero-cache/src/server/shadow-syncer.ts
|
|
12
13
|
var MS_PER_HOUR = 1e3 * 60 * 60;
|
|
14
|
+
var lc = new LogContext("info", {}, consoleLogSink);
|
|
13
15
|
function runWorker(parent, env, ...argv) {
|
|
14
16
|
const config = getNormalizedZeroConfig({
|
|
15
17
|
env,
|
|
16
18
|
argv
|
|
17
19
|
});
|
|
18
20
|
startOtelAuto(createLogContext(config, "shadow-syncer", 0, false), "shadow-syncer", 0);
|
|
19
|
-
|
|
21
|
+
lc = createLogContext(config, "shadow-syncer");
|
|
20
22
|
initEventSink(lc, config);
|
|
21
23
|
const { shadowSync, upstream, initialSync } = config;
|
|
22
|
-
const
|
|
24
|
+
const shard = getShardConfig(config);
|
|
25
|
+
const service = new ShadowSyncService(lc, shard, upstream.db, getServerContext(config), {
|
|
23
26
|
intervalMs: shadowSync.intervalHours * MS_PER_HOUR,
|
|
24
27
|
sampleRate: shadowSync.sampleRate,
|
|
25
28
|
maxRowsPerTable: shadowSync.maxRowsPerTable,
|
|
@@ -28,7 +31,7 @@ function runWorker(parent, env, ...argv) {
|
|
|
28
31
|
parent.send(["ready", { ready: true }]);
|
|
29
32
|
return runUntilKilled(lc, parent, service);
|
|
30
33
|
}
|
|
31
|
-
if (!singleProcessMode()) exitAfter(() => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
34
|
+
if (!singleProcessMode()) exitAfter(lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
32
35
|
//#endregion
|
|
33
36
|
export { runWorker as default };
|
|
34
37
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shadow-syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/shadow-syncer.ts"],"sourcesContent":["import {must} from '../../../shared/src/must.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ShadowSyncService} from '../services/shadow-sync/shadow-sync-service.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\nexport default function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n\n startOtelAuto(\n createLogContext(config, 'shadow-syncer', 0, false),\n 'shadow-syncer',\n 0,\n );\n
|
|
1
|
+
{"version":3,"file":"shadow-syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/shadow-syncer.ts"],"sourcesContent":["import {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {must} from '../../../shared/src/must.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ShadowSyncService} from '../services/shadow-sync/shadow-sync-service.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nconst MS_PER_HOUR = 1000 * 60 * 60;\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...argv: string[]\n): Promise<void> {\n const config = getNormalizedZeroConfig({env, argv});\n\n startOtelAuto(\n createLogContext(config, 'shadow-syncer', 0, false),\n 'shadow-syncer',\n 0,\n );\n lc = createLogContext(config, 'shadow-syncer');\n initEventSink(lc, config);\n\n const {shadowSync, upstream, initialSync} = config;\n const shard = getShardConfig(config);\n const service = new ShadowSyncService(\n lc,\n shard,\n upstream.db,\n getServerContext(config),\n {\n intervalMs: shadowSync.intervalHours * MS_PER_HOUR,\n sampleRate: shadowSync.sampleRate,\n maxRowsPerTable: shadowSync.maxRowsPerTable,\n textCopy: initialSync.textCopy,\n },\n );\n\n parent.send(['ready', {ready: true}]);\n\n return runUntilKilled(lc, parent, service);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(lc, () =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;AAgBA,IAAM,cAAc,MAAO,KAAK;AAGhC,IAAI,KAAK,IAAI,WAAW,QAAQ,EAAE,EAAE,eAAe;AAEnD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,SAAS,wBAAwB;EAAC;EAAK;EAAK,CAAC;AAEnD,eACE,iBAAiB,QAAQ,iBAAiB,GAAG,MAAM,EACnD,iBACA,EACD;AACD,MAAK,iBAAiB,QAAQ,gBAAgB;AAC9C,eAAc,IAAI,OAAO;CAEzB,MAAM,EAAC,YAAY,UAAU,gBAAe;CAC5C,MAAM,QAAQ,eAAe,OAAO;CACpC,MAAM,UAAU,IAAI,kBAClB,IACA,OACA,SAAS,IACT,iBAAiB,OAAO,EACxB;EACE,YAAY,WAAW,gBAAgB;EACvC,YAAY,WAAW;EACvB,iBAAiB,WAAW;EAC5B,UAAU,YAAY;EACvB,CACF;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAErC,QAAO,eAAe,IAAI,QAAQ,QAAQ;;AAI5C,IAAI,CAAC,mBAAmB,CACjB,WAAU,UACb,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"syncer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/syncer.ts"],"names":[],"mappings":"AAgCA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAmC/B,MAAM,CAAC,OAAO,UAAU,SAAS,CAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CAsMf"}
|
|
@@ -27,6 +27,7 @@ import { ViewSyncerService } from "../services/view-syncer/view-syncer.js";
|
|
|
27
27
|
import { Syncer } from "../workers/syncer.js";
|
|
28
28
|
import { InspectorDelegate } from "./inspector-delegate.js";
|
|
29
29
|
import { isPriorityOpRunning, runPriorityOp } from "./priority-op.js";
|
|
30
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
30
31
|
import { pid } from "node:process";
|
|
31
32
|
import { randomUUID } from "node:crypto";
|
|
32
33
|
import { tmpdir } from "node:os";
|
|
@@ -45,6 +46,7 @@ function getCustomQueryConfig(config) {
|
|
|
45
46
|
forwardCookies: queryConfig.forwardCookies ?? false
|
|
46
47
|
};
|
|
47
48
|
}
|
|
49
|
+
var lc = new LogContext("info", {}, consoleLogSink);
|
|
48
50
|
function runWorker(parent, env, ...args) {
|
|
49
51
|
assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);
|
|
50
52
|
const fileMode = parse(args[0], replicaFileModeSchema);
|
|
@@ -54,7 +56,7 @@ function runWorker(parent, env, ...args) {
|
|
|
54
56
|
argv: args.slice(2)
|
|
55
57
|
});
|
|
56
58
|
startOtelAuto(createLogContext(config, "syncer", workerIndex, false), "syncer", workerIndex);
|
|
57
|
-
|
|
59
|
+
lc = createLogContext(config, "syncer", workerIndex);
|
|
58
60
|
initEventSink(lc, config);
|
|
59
61
|
const { cvr, upstream, enableCrudMutations } = config;
|
|
60
62
|
const replicaFile = replicaFileName(config.replica.file, fileMode);
|
|
@@ -105,7 +107,7 @@ function runWorker(parent, env, ...args) {
|
|
|
105
107
|
dbWarmup.then(() => parent.send(["ready", { ready: true }]));
|
|
106
108
|
return runUntilKilled(lc, parent, syncer);
|
|
107
109
|
}
|
|
108
|
-
if (!singleProcessMode()) exitAfter(() => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
110
|
+
if (!singleProcessMode()) exitAfter(lc, () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)));
|
|
109
111
|
//#endregion
|
|
110
112
|
export { runWorker as default };
|
|
111
113
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n type ConnectionContextManager,\n ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n if (!queryConfig?.url) {\n return undefined;\n }\n\n return {\n url: queryConfig.url,\n apiKey: queryConfig.apiKey,\n allowedClientHeaders: queryConfig.allowedClientHeaders,\n forwardCookies: queryConfig.forwardCookies ?? false,\n };\n}\n\nexport default function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n const workerIndex = Number(args[1]);\n const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n startOtelAuto(\n createLogContext(config, 'syncer', workerIndex, false),\n 'syncer',\n workerIndex,\n );\n const lc = createLogContext(config, 'syncer', workerIndex);\n initEventSink(lc, config);\n\n const {cvr, upstream, enableCrudMutations} = config;\n\n const replicaFile = replicaFileName(config.replica.file, fileMode);\n lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n const cvrDB = pgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n });\n\n const upstreamDB = enableCrudMutations\n ? pgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n max: must(\n upstream.maxConnsPerWorker,\n 'upstream.maxConnsPerWorker must be set',\n ),\n })\n : undefined;\n\n const dbWarmup = Promise.allSettled([\n warmupConnections(lc, cvrDB, 'cvr'),\n upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n ]);\n\n const tmpDir = config.storageDBTmpDir ?? tmpdir();\n const operatorStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `sync-worker-${randomUUID()}`),\n );\n const writeAuthzStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `mutagen-${randomUUID()}`),\n );\n\n const shard = getShardID(config);\n const customQueryConfig = getCustomQueryConfig(config);\n const pushConfig =\n config.push.url === undefined && config.mutate.url === undefined\n ? undefined\n : {\n ...config.push,\n ...config.mutate,\n url: must(\n config.push.url ?? config.mutate.url,\n 'No push or mutate URL configured',\n ),\n };\n\n /** @deprecated used in JWT validation */\n let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n const tokenOptions = tokenConfigOptions(config.auth ?? {});\n if (tokenOptions.length === 1) {\n validateLegacyJWT = async (token, {userID}) => {\n if (!userID) {\n throw new ProtocolErrorWithLevel(\n {\n kind: 'Unauthorized',\n message: 'UserID is required for JWT validation.',\n origin: 'zeroCache',\n },\n 'warn',\n );\n }\n\n const decoded = await verifyToken(config.auth, token, {\n subject: userID,\n ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n ...(config.auth?.audience && {\n audience: config.auth.audience,\n }),\n });\n return {\n type: 'jwt',\n raw: token,\n decoded,\n };\n };\n }\n\n const viewSyncerFactory = (\n id: string,\n sub: Subscription<ReplicaState>,\n drainCoordinator: DrainCoordinator,\n ) => {\n const logger = lc\n .withContext('component', 'view-syncer')\n .withContext('clientGroupID', id)\n .withContext('instance', randomID());\n\n const customQueryTransformer =\n customQueryConfig && new CustomQueryTransformer(logger, shard);\n const connContextManager = new ConnectionContextManagerImpl(\n logger,\n config.auth.revalidateIntervalSeconds,\n config.auth.retransformIntervalSeconds,\n customQueryConfig,\n pushConfig,\n validateLegacyJWT,\n );\n\n lc.debug?.(\n `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n );\n\n const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n const priorityOpRunningYieldThresholdMs = Math.max(\n config.yieldThresholdMs / 4,\n 2,\n );\n const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n return new ViewSyncerService(\n config,\n logger,\n shard,\n config.taskID,\n id,\n cvrDB,\n new PipelineDriver(\n logger,\n config.log,\n new Snapshotter(logger, replicaFile, shard),\n shard,\n operatorStorage.createClientGroupStorage(id),\n id,\n inspectorDelegate,\n () =>\n isPriorityOpRunning()\n ? priorityOpRunningYieldThresholdMs\n : normalYieldThresholdMs,\n config.enableQueryPlanner,\n config,\n ),\n sub,\n drainCoordinator,\n config.log.slowHydrateThreshold,\n inspectorDelegate,\n connContextManager,\n customQueryTransformer,\n runPriorityOp,\n );\n };\n\n const mutagenFactory = upstreamDB\n ? (id: string) =>\n new MutagenService(\n lc\n .withContext('component', 'mutagen')\n .withContext('clientGroupID', id),\n shard,\n id,\n upstreamDB,\n config,\n writeAuthzStorage,\n )\n : undefined;\n\n const pusherFactory =\n pushConfig === undefined\n ? undefined\n : (id: string, connContextManager: ConnectionContextManager) =>\n new PusherService(\n config,\n lc.withContext('clientGroupID', id),\n id,\n connContextManager,\n );\n\n const syncer = new Syncer(\n lc,\n config,\n viewSyncerFactory,\n mutagenFactory,\n pusherFactory,\n parent,\n validateLegacyJWT,\n );\n\n startAnonymousTelemetry(lc, config);\n\n void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA8CA,SAAS,WAAW;AAClB,QAAO,QAAQ,GAAG,OAAO,iBAAiB,CAAC,SAAS,GAAG;;AAGzD,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;AAE9D,KAAI,CAAC,aAAa,IAChB;AAGF,QAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,gBAAgB,YAAY,kBAAkB;EAC/C;;AAGH,SAAwB,UACtB,QACA,KACA,GAAG,MACY;AACf,QAAO,KAAK,UAAU,GAAG,6CAA6C;CACtE,MAAM,WAAW,MAAQ,KAAK,IAAI,sBAAsB;CACxD,MAAM,cAAc,OAAO,KAAK,GAAG;CACnC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;AAElE,eACE,iBAAiB,QAAQ,UAAU,aAAa,MAAM,EACtD,UACA,YACD;CACD,MAAM,KAAK,iBAAiB,QAAQ,UAAU,YAAY;AAC1D,eAAc,IAAI,OAAO;CAEzB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,SAAS;AAClE,IAAG,QAAQ,0BAA0B,cAAc;CAEnD,MAAM,QAAQ,SAAS,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EAC3D,KAAK,KAAK,IAAI,mBAAmB,oCAAoC,EACtE,CAAC;CAEF,MAAM,aAAa,sBACf,SAAS,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACvD,KAAK,KACH,SAAS,mBACT,yCACD,EACF,CAAC,GACF,KAAA;CAEJ,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,MAAM,EACnC,aAAa,kBAAkB,IAAI,YAAY,WAAW,GAAG,YAC9D,CAAC;CAEF,MAAM,SAAS,OAAO,mBAAmB,QAAQ;CACjD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,YAAY,GAAG,CACjD;CACD,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,YAAY,GAAG,CAC7C;CAED,MAAM,QAAQ,WAAW,OAAO;CAChC,MAAM,oBAAoB,qBAAqB,OAAO;CACtD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,mCACD;EACF;;CAGP,IAAI,oBAAmD,KAAA;AAGvD,KADqB,mBAAmB,OAAO,QAAQ,EAAE,CAAC,CACzC,WAAW,EAC1B,qBAAoB,OAAO,OAAO,EAAC,aAAY;AAC7C,MAAI,CAAC,OACH,OAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;GACT,EACD,OACD;AAUH,SAAO;GACL,MAAM;GACN,KAAK;GACL,SAVc,MAAM,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,QAAO;IACvD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,UACvB;IACF,CAAC;GAKD;;CAIL,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,aAAa,cAAc,CACvC,YAAY,iBAAiB,GAAG,CAChC,YAAY,YAAY,UAAU,CAAC;EAEtC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,MAAM;EAChE,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,kBACD;AAED,KAAG,QACD,gDAAgD,OAAO,qBACxD;EAED,MAAM,oBAAoB,IAAI,kBAAkB,uBAAuB;EAEvE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,EACD;EACD,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,EAAE;AAEnE,SAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,MAAM,EAC3C,OACA,gBAAgB,yBAAyB,GAAG,EAC5C,IACA,yBAEE,qBAAqB,GACjB,oCACA,wBACN,OAAO,oBACP,OACD,EACD,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,cACD;;CA4BH,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,UAAU,CACnC,YAAY,iBAAiB,GAAG,EACnC,OACA,IACA,YACA,QACA,kBACD,GACH,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,GAAG,EACnC,IACA,mBACD,EAQP,QACA,kBACD;AAED,yBAAwB,IAAI,OAAO;AAE9B,UAAS,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC,CAAC;AAE/D,QAAO,eAAe,IAAI,QAAQ,OAAO;;AAI3C,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
1
|
+
{"version":3,"file":"syncer.js","names":[],"sources":["../../../../../zero-cache/src/server/syncer.ts"],"sourcesContent":["import {randomUUID} from 'node:crypto';\nimport {tmpdir} from 'node:os';\nimport path from 'node:path';\nimport {pid} from 'node:process';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport {promiseVoid} from '../../../shared/src/resolved-promises.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {DatabaseStorage} from '../../../zqlite/src/database-storage.ts';\nimport type {ValidateLegacyJWT} from '../auth/auth.ts';\nimport {tokenConfigOptions, verifyToken} from '../auth/jwt.ts';\nimport type {NormalizedZeroConfig} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {CustomQueryTransformer} from '../custom-queries/transform-query.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {MutagenService} from '../services/mutagen/mutagen.ts';\nimport {PusherService} from '../services/mutagen/pusher.ts';\nimport type {ReplicaState} from '../services/replicator/replicator.ts';\nimport {\n type ConnectionContextManager,\n ConnectionContextManagerImpl,\n} from '../services/view-syncer/connection-context-manager.ts';\nimport type {DrainCoordinator} from '../services/view-syncer/drain-coordinator.ts';\nimport {PipelineDriver} from '../services/view-syncer/pipeline-driver.ts';\nimport {Snapshotter} from '../services/view-syncer/snapshotter.ts';\nimport {ViewSyncerService} from '../services/view-syncer/view-syncer.ts';\nimport {ProtocolErrorWithLevel} from '../types/error-with-level.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardID} from '../types/shards.ts';\nimport type {Subscription} from '../types/subscription.ts';\nimport {replicaFileModeSchema, replicaFileName} from '../workers/replicator.ts';\nimport {Syncer} from '../workers/syncer.ts';\nimport {startAnonymousTelemetry} from './anonymous-otel-start.ts';\nimport {InspectorDelegate} from './inspector-delegate.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {isPriorityOpRunning, runPriorityOp} from './priority-op.ts';\n\nfunction randomID() {\n return randInt(1, Number.MAX_SAFE_INTEGER).toString(36);\n}\n\nfunction getCustomQueryConfig(\n config: Pick<NormalizedZeroConfig, 'query' | 'getQueries'>,\n) {\n const queryConfig = config.query?.url ? config.query : config.getQueries;\n\n if (!queryConfig?.url) {\n return undefined;\n }\n\n return {\n url: queryConfig.url,\n apiKey: queryConfig.apiKey,\n allowedClientHeaders: queryConfig.allowedClientHeaders,\n forwardCookies: queryConfig.forwardCookies ?? false,\n };\n}\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length >= 2, `expected [fileMode, workerIndex, ...flags]`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n const workerIndex = Number(args[1]);\n const config = getNormalizedZeroConfig({env, argv: args.slice(2)});\n\n startOtelAuto(\n createLogContext(config, 'syncer', workerIndex, false),\n 'syncer',\n workerIndex,\n );\n lc = createLogContext(config, 'syncer', workerIndex);\n initEventSink(lc, config);\n\n const {cvr, upstream, enableCrudMutations} = config;\n\n const replicaFile = replicaFileName(config.replica.file, fileMode);\n lc.debug?.(`running view-syncer on ${replicaFile}`);\n\n const cvrDB = pgClient(lc, cvr.db, `sync-worker-${pid}-cvr`, {\n max: must(cvr.maxConnsPerWorker, 'cvr.maxConnsPerWorker must be set'),\n });\n\n const upstreamDB = enableCrudMutations\n ? pgClient(lc, upstream.db, `sync-worker-${pid}-upstream`, {\n max: must(\n upstream.maxConnsPerWorker,\n 'upstream.maxConnsPerWorker must be set',\n ),\n })\n : undefined;\n\n const dbWarmup = Promise.allSettled([\n warmupConnections(lc, cvrDB, 'cvr'),\n upstreamDB ? warmupConnections(lc, upstreamDB, 'upstream') : promiseVoid,\n ]);\n\n const tmpDir = config.storageDBTmpDir ?? tmpdir();\n const operatorStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `sync-worker-${randomUUID()}`),\n );\n const writeAuthzStorage = DatabaseStorage.create(\n lc,\n path.join(tmpDir, `mutagen-${randomUUID()}`),\n );\n\n const shard = getShardID(config);\n const customQueryConfig = getCustomQueryConfig(config);\n const pushConfig =\n config.push.url === undefined && config.mutate.url === undefined\n ? undefined\n : {\n ...config.push,\n ...config.mutate,\n url: must(\n config.push.url ?? config.mutate.url,\n 'No push or mutate URL configured',\n ),\n };\n\n /** @deprecated used in JWT validation */\n let validateLegacyJWT: ValidateLegacyJWT | undefined = undefined;\n\n const tokenOptions = tokenConfigOptions(config.auth ?? {});\n if (tokenOptions.length === 1) {\n validateLegacyJWT = async (token, {userID}) => {\n if (!userID) {\n throw new ProtocolErrorWithLevel(\n {\n kind: 'Unauthorized',\n message: 'UserID is required for JWT validation.',\n origin: 'zeroCache',\n },\n 'warn',\n );\n }\n\n const decoded = await verifyToken(config.auth, token, {\n subject: userID,\n ...(config.auth?.issuer && {issuer: config.auth.issuer}),\n ...(config.auth?.audience && {\n audience: config.auth.audience,\n }),\n });\n return {\n type: 'jwt',\n raw: token,\n decoded,\n };\n };\n }\n\n const viewSyncerFactory = (\n id: string,\n sub: Subscription<ReplicaState>,\n drainCoordinator: DrainCoordinator,\n ) => {\n const logger = lc\n .withContext('component', 'view-syncer')\n .withContext('clientGroupID', id)\n .withContext('instance', randomID());\n\n const customQueryTransformer =\n customQueryConfig && new CustomQueryTransformer(logger, shard);\n const connContextManager = new ConnectionContextManagerImpl(\n logger,\n config.auth.revalidateIntervalSeconds,\n config.auth.retransformIntervalSeconds,\n customQueryConfig,\n pushConfig,\n validateLegacyJWT,\n );\n\n lc.debug?.(\n `creating view syncer. Query Planner Enabled: ${config.enableQueryPlanner}`,\n );\n\n const inspectorDelegate = new InspectorDelegate(customQueryTransformer);\n\n const priorityOpRunningYieldThresholdMs = Math.max(\n config.yieldThresholdMs / 4,\n 2,\n );\n const normalYieldThresholdMs = Math.max(config.yieldThresholdMs, 2);\n\n return new ViewSyncerService(\n config,\n logger,\n shard,\n config.taskID,\n id,\n cvrDB,\n new PipelineDriver(\n logger,\n config.log,\n new Snapshotter(logger, replicaFile, shard),\n shard,\n operatorStorage.createClientGroupStorage(id),\n id,\n inspectorDelegate,\n () =>\n isPriorityOpRunning()\n ? priorityOpRunningYieldThresholdMs\n : normalYieldThresholdMs,\n config.enableQueryPlanner,\n config,\n ),\n sub,\n drainCoordinator,\n config.log.slowHydrateThreshold,\n inspectorDelegate,\n connContextManager,\n customQueryTransformer,\n runPriorityOp,\n );\n };\n\n const mutagenFactory = upstreamDB\n ? (id: string) =>\n new MutagenService(\n lc\n .withContext('component', 'mutagen')\n .withContext('clientGroupID', id),\n shard,\n id,\n upstreamDB,\n config,\n writeAuthzStorage,\n )\n : undefined;\n\n const pusherFactory =\n pushConfig === undefined\n ? undefined\n : (id: string, connContextManager: ConnectionContextManager) =>\n new PusherService(\n config,\n lc.withContext('clientGroupID', id),\n id,\n connContextManager,\n );\n\n const syncer = new Syncer(\n lc,\n config,\n viewSyncerFactory,\n mutagenFactory,\n pusherFactory,\n parent,\n validateLegacyJWT,\n );\n\n startAnonymousTelemetry(lc, config);\n\n void dbWarmup.then(() => parent.send(['ready', {ready: true}]));\n\n return runUntilKilled(lc, parent, syncer);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(lc, () =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+CA,SAAS,WAAW;AAClB,QAAO,QAAQ,GAAG,OAAO,iBAAiB,CAAC,SAAS,GAAG;;AAGzD,SAAS,qBACP,QACA;CACA,MAAM,cAAc,OAAO,OAAO,MAAM,OAAO,QAAQ,OAAO;AAE9D,KAAI,CAAC,aAAa,IAChB;AAGF,QAAO;EACL,KAAK,YAAY;EACjB,QAAQ,YAAY;EACpB,sBAAsB,YAAY;EAClC,gBAAgB,YAAY,kBAAkB;EAC/C;;AAIH,IAAI,KAAK,IAAI,WAAW,QAAQ,EAAE,EAAE,eAAe;AAEnD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;AACf,QAAO,KAAK,UAAU,GAAG,6CAA6C;CACtE,MAAM,WAAW,MAAQ,KAAK,IAAI,sBAAsB;CACxD,MAAM,cAAc,OAAO,KAAK,GAAG;CACnC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;AAElE,eACE,iBAAiB,QAAQ,UAAU,aAAa,MAAM,EACtD,UACA,YACD;AACD,MAAK,iBAAiB,QAAQ,UAAU,YAAY;AACpD,eAAc,IAAI,OAAO;CAEzB,MAAM,EAAC,KAAK,UAAU,wBAAuB;CAE7C,MAAM,cAAc,gBAAgB,OAAO,QAAQ,MAAM,SAAS;AAClE,IAAG,QAAQ,0BAA0B,cAAc;CAEnD,MAAM,QAAQ,SAAS,IAAI,IAAI,IAAI,eAAe,IAAI,OAAO,EAC3D,KAAK,KAAK,IAAI,mBAAmB,oCAAoC,EACtE,CAAC;CAEF,MAAM,aAAa,sBACf,SAAS,IAAI,SAAS,IAAI,eAAe,IAAI,YAAY,EACvD,KAAK,KACH,SAAS,mBACT,yCACD,EACF,CAAC,GACF,KAAA;CAEJ,MAAM,WAAW,QAAQ,WAAW,CAClC,kBAAkB,IAAI,OAAO,MAAM,EACnC,aAAa,kBAAkB,IAAI,YAAY,WAAW,GAAG,YAC9D,CAAC;CAEF,MAAM,SAAS,OAAO,mBAAmB,QAAQ;CACjD,MAAM,kBAAkB,gBAAgB,OACtC,IACA,KAAK,KAAK,QAAQ,eAAe,YAAY,GAAG,CACjD;CACD,MAAM,oBAAoB,gBAAgB,OACxC,IACA,KAAK,KAAK,QAAQ,WAAW,YAAY,GAAG,CAC7C;CAED,MAAM,QAAQ,WAAW,OAAO;CAChC,MAAM,oBAAoB,qBAAqB,OAAO;CACtD,MAAM,aACJ,OAAO,KAAK,QAAQ,KAAA,KAAa,OAAO,OAAO,QAAQ,KAAA,IACnD,KAAA,IACA;EACE,GAAG,OAAO;EACV,GAAG,OAAO;EACV,KAAK,KACH,OAAO,KAAK,OAAO,OAAO,OAAO,KACjC,mCACD;EACF;;CAGP,IAAI,oBAAmD,KAAA;AAGvD,KADqB,mBAAmB,OAAO,QAAQ,EAAE,CAAC,CACzC,WAAW,EAC1B,qBAAoB,OAAO,OAAO,EAAC,aAAY;AAC7C,MAAI,CAAC,OACH,OAAM,IAAI,uBACR;GACE,MAAM;GACN,SAAS;GACT,QAAQ;GACT,EACD,OACD;AAUH,SAAO;GACL,MAAM;GACN,KAAK;GACL,SAVc,MAAM,YAAY,OAAO,MAAM,OAAO;IACpD,SAAS;IACT,GAAI,OAAO,MAAM,UAAU,EAAC,QAAQ,OAAO,KAAK,QAAO;IACvD,GAAI,OAAO,MAAM,YAAY,EAC3B,UAAU,OAAO,KAAK,UACvB;IACF,CAAC;GAKD;;CAIL,MAAM,qBACJ,IACA,KACA,qBACG;EACH,MAAM,SAAS,GACZ,YAAY,aAAa,cAAc,CACvC,YAAY,iBAAiB,GAAG,CAChC,YAAY,YAAY,UAAU,CAAC;EAEtC,MAAM,yBACJ,qBAAqB,IAAI,uBAAuB,QAAQ,MAAM;EAChE,MAAM,qBAAqB,IAAI,6BAC7B,QACA,OAAO,KAAK,2BACZ,OAAO,KAAK,4BACZ,mBACA,YACA,kBACD;AAED,KAAG,QACD,gDAAgD,OAAO,qBACxD;EAED,MAAM,oBAAoB,IAAI,kBAAkB,uBAAuB;EAEvE,MAAM,oCAAoC,KAAK,IAC7C,OAAO,mBAAmB,GAC1B,EACD;EACD,MAAM,yBAAyB,KAAK,IAAI,OAAO,kBAAkB,EAAE;AAEnE,SAAO,IAAI,kBACT,QACA,QACA,OACA,OAAO,QACP,IACA,OACA,IAAI,eACF,QACA,OAAO,KACP,IAAI,YAAY,QAAQ,aAAa,MAAM,EAC3C,OACA,gBAAgB,yBAAyB,GAAG,EAC5C,IACA,yBAEE,qBAAqB,GACjB,oCACA,wBACN,OAAO,oBACP,OACD,EACD,KACA,kBACA,OAAO,IAAI,sBACX,mBACA,oBACA,wBACA,cACD;;CA4BH,MAAM,SAAS,IAAI,OACjB,IACA,QACA,mBA5BqB,cAClB,OACC,IAAI,eACF,GACG,YAAY,aAAa,UAAU,CACnC,YAAY,iBAAiB,GAAG,EACnC,OACA,IACA,YACA,QACA,kBACD,GACH,KAAA,GAGF,eAAe,KAAA,IACX,KAAA,KACC,IAAY,uBACX,IAAI,cACF,QACA,GAAG,YAAY,iBAAiB,GAAG,EACnC,IACA,mBACD,EAQP,QACA,kBACD;AAED,yBAAwB,IAAI,OAAO;AAE9B,UAAS,WAAW,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC,CAAC;AAE/D,QAAO,eAAe,IAAI,QAAQ,OAAO;;AAI3C,IAAI,CAAC,mBAAmB,CACjB,WAAU,UACb,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -53,8 +53,14 @@ export declare function shadowInitialSync(lc: LogContext, shard: ShardConfig, up
|
|
|
53
53
|
/**
|
|
54
54
|
* Runs structural assertions over a just-synced replica and throws if any
|
|
55
55
|
* fail. Only called in shadow mode — a successful return means the replica
|
|
56
|
-
* is schema-complete, row-count consistent,
|
|
57
|
-
*
|
|
56
|
+
* is schema-complete, row-count consistent, and its column metadata is in
|
|
57
|
+
* sync with its lite schema.
|
|
58
|
+
*
|
|
59
|
+
* Note: this intentionally does NOT verify ZQL-queryability. Tables that
|
|
60
|
+
* `computeZqlSpecs` drops (no PK / no all-NOT-NULL unique index, unsupported
|
|
61
|
+
* column types, etc.) are silently skipped in production too — there's
|
|
62
|
+
* nothing shadow-specific about them, so failing here would diverge from
|
|
63
|
+
* prod over an upstream-schema condition prod accepts.
|
|
58
64
|
*
|
|
59
65
|
* Exported for testing.
|
|
60
66
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"initial-sync.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/initial-sync.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,0CAA0C,CAAC;AAGzE,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,0CAA0C,CAAC;AAC7E,OAAO,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAoBzD,OAAO,KAAK,EAAC,SAAS,EAAE,kBAAkB,EAAC,MAAM,sBAAsB,CAAC;AASxE,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAC;AAiB1D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,uBAAuB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9C;;;;;;OAMG;IACH,MAAM,CAAC,EACH;QACE,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB;;;WAGG;QACH,eAAe,EAAE,MAAM,CAAC;KACzB,GACD,SAAS,CAAC;CACf,CAAC;AAEF,4EAA4E;AAC5E,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC;AAEvC,wBAAsB,WAAW,CAC/B,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,EAAE,EAAE,QAAQ,EACZ,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,iBA8NvB;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,WAAW,EAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,aAAa,EACtB,WAAW,CAAC,EAAE,IAAI,CAAC,kBAAkB,EAAE,UAAU,CAAC,GACjD,OAAO,CAAC,IAAI,CAAC,CA+Bf;AAuLD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,SAAS,EAAE;IAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,EAAE,CAAA;CAAC,EAC/D,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GACvC,IAAI,CAgFN;AAMD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAMpC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAwBF;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,GACb,MAAM,EAAE,CAKV;AAED,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,kBAAkB,EACzB,IAAI,EAAE,MAAM,EAAE,EACd,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,EAC/B,eAAe,CAAC,EAAE,MAAM,GAAG,SAAS,EACpC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,GACjC,kBAAkB,CA8BpB;AAED,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,cAAc,CAAC;CACxB,CAAC;AAGF,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,IAAI,EAAE,kBAAkB,EACxB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,aAAa,CAAC,CA2CxB"}
|
|
@@ -7,7 +7,7 @@ import { liteValue } from "../../../types/lite.js";
|
|
|
7
7
|
import { liteTableName } from "../../../types/names.js";
|
|
8
8
|
import { mapPostgresToLite, mapPostgresToLiteIndex } from "../../../db/pg-to-lite.js";
|
|
9
9
|
import { ColumnMetadataStore } from "../../replicator/schema/column-metadata.js";
|
|
10
|
-
import {
|
|
10
|
+
import { listIndexes, listTables } from "../../../db/lite-tables.js";
|
|
11
11
|
import { initReplicationState } from "../../replicator/schema/replication-state.js";
|
|
12
12
|
import { id } from "../../../types/sql.js";
|
|
13
13
|
import { pgClient } from "../../../types/pg.js";
|
|
@@ -264,8 +264,14 @@ function createLiteIndices(tx, indices) {
|
|
|
264
264
|
/**
|
|
265
265
|
* Runs structural assertions over a just-synced replica and throws if any
|
|
266
266
|
* fail. Only called in shadow mode — a successful return means the replica
|
|
267
|
-
* is schema-complete, row-count consistent,
|
|
268
|
-
*
|
|
267
|
+
* is schema-complete, row-count consistent, and its column metadata is in
|
|
268
|
+
* sync with its lite schema.
|
|
269
|
+
*
|
|
270
|
+
* Note: this intentionally does NOT verify ZQL-queryability. Tables that
|
|
271
|
+
* `computeZqlSpecs` drops (no PK / no all-NOT-NULL unique index, unsupported
|
|
272
|
+
* column types, etc.) are silently skipped in production too — there's
|
|
273
|
+
* nothing shadow-specific about them, so failing here would diverge from
|
|
274
|
+
* prod over an upstream-schema condition prod accepts.
|
|
269
275
|
*
|
|
270
276
|
* Exported for testing.
|
|
271
277
|
*/
|
|
@@ -299,11 +305,6 @@ function verifyShadowReplica(lc, db, published, rowsByTable) {
|
|
|
299
305
|
} catch (e) {
|
|
300
306
|
issues.push(`could not count rows in table ${table}: ${String(e)}`);
|
|
301
307
|
}
|
|
302
|
-
const tableSpecs = computeZqlSpecs(lc, db, { includeBackfillingColumns: false });
|
|
303
|
-
for (const pt of published.tables) {
|
|
304
|
-
const name = liteTableName(pt);
|
|
305
|
-
if (!tableSpecs.has(name)) issues.push(`table not queryable via ZQL (dropped by computeZqlSpecs): ${name}`);
|
|
306
|
-
}
|
|
307
308
|
const meta = must(ColumnMetadataStore.getInstance(db));
|
|
308
309
|
for (const pt of published.tables) {
|
|
309
310
|
const name = liteTableName(pt);
|