@rocicorp/zero 1.12.0-canary.11 → 1.12.0-canary.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/server/replicator.js +4 -2
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/serving-checkpointer.js +41 -0
- package/out/zero-cache/src/server/serving-checkpointer.js.map +1 -0
- package/out/zero-cache/src/server/worker-urls.d.ts +1 -0
- package/out/zero-cache/src/server/worker-urls.d.ts.map +1 -1
- package/out/zero-cache/src/server/worker-urls.js +2 -1
- package/out/zero-cache/src/server/worker-urls.js.map +1 -1
- package/out/zero-cache/src/services/replicator/serving-checkpointer.js +85 -0
- package/out/zero-cache/src/services/replicator/serving-checkpointer.js.map +1 -0
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +1 -1
- package/out/zero-cache/src/workers/replicator.js +9 -4
- package/out/zero-cache/src/workers/replicator.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/package.json +1 -1
package/out/zero/package.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
//#region package.json
|
|
2
2
|
var name = "@rocicorp/zero";
|
|
3
|
-
var version = "1.12.0-canary.
|
|
3
|
+
var version = "1.12.0-canary.12";
|
|
4
4
|
var description = "Zero is a web framework for serverless web development.";
|
|
5
5
|
var homepage = "https://zero.rocicorp.dev";
|
|
6
6
|
var bugs = { "url": "https://bugs.rocicorp.dev" };
|
package/out/zero/package.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.12.0-canary.
|
|
1
|
+
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.12.0-canary.12\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"license\": \"Apache-2.0\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"bin\": {\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\",\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\"\n },\n \"files\": [\n \"out\",\n \"!*.tsbuildinfo\"\n ],\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./analyze\": {\n \"types\": \"./out/zero/src/analyze.d.ts\",\n \"default\": \"./out/zero/src/analyze.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/kysely\": {\n \"types\": \"./out/zero/src/adapters/kysely.d.ts\",\n \"default\": \"./out/zero/src/adapters/kysely.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"oxfmt .\",\n \"check-format\": \"oxfmt --check .\",\n \"lint\": \"oxlint --quiet --config ../../oxlint.config.ts src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\",\n \"fmt\": \"oxfmt .\",\n \"check-fmt\": \"oxfmt --check .\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.5.5\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.66.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.1.0\",\n \"@fastify/websocket\": \"^11.2.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"1.9.1\",\n \"@opentelemetry/api-logs\": \"0.223.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"0.81.2\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"0.223.0\",\n \"@opentelemetry/resources\": \"2.12.0\",\n \"@opentelemetry/sdk-metrics\": \"2.12.0\",\n \"@opentelemetry/sdk-node\": \"0.223.0\",\n \"@opentelemetry/sdk-trace-node\": \"2.12.0\",\n \"@opentelemetry/semantic-conventions\": \"1.43.0\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^2.0.1\",\n \"@rocicorp/logger\": \"^6.1.0\",\n \"@rocicorp/zero-sqlite3\": \"^1.1.7\",\n \"@standard-schema/spec\": \"^1.1.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.18.1\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk-template\": \"^1.1.2\",\n \"chokidar\": \"^4.0.3\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.2\",\n \"command-line-usage\": \"^7.0.4\",\n \"compare-utf8\": \"^0.2.3\",\n \"defu\": \"^6.1.7\",\n \"eventemitter3\": \"^5.0.4\",\n \"fastify\": \"^5.12.7\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.10.0\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.2\",\n \"nanoid\": \"^5.1.16\",\n \"oxc-parser\": \"^0.129.0\",\n \"oxfmt\": \"^0.74.0\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"semver\": \"^7.8.0\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.22.0\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \"^18.2.5\",\n \"analyze-query\": \"workspace:*\",\n \"ast-to-zql\": \"workspace:*\",\n \"expo\": \"^58.0.14\",\n \"expo-sqlite\": \"^58.0.13\",\n \"react-native\": \"^0.88.0\",\n \"replicache\": \"workspace:*\",\n \"shared\": \"workspace:*\",\n \"syncpack\": \"^14.3.1\",\n \"typedoc\": \"^0.28.19\",\n \"typescript\": \"~7.0.2\",\n \"vite\": \"^8.3.4\",\n \"vitest\": \"^5.0.5\",\n \"zero-cache\": \"workspace:*\",\n \"zero-client\": \"workspace:*\",\n \"zero-pg\": \"workspace:*\",\n \"zero-react\": \"workspace:*\",\n \"zero-server\": \"workspace:*\",\n \"zero-solid\": \"workspace:*\",\n \"zqlite\": \"workspace:*\"\n },\n \"engines\": {\n \"node\": \">=22\"\n },\n \"packageManager\": \"pnpm@11.11.0\"\n}\n"],"mappings":""}
|
|
@@ -2,7 +2,7 @@ import { assert } from "../../../shared/src/asserts.js";
|
|
|
2
2
|
import { sleep } from "../../../shared/src/sleep.js";
|
|
3
3
|
import { parse } from "../../../shared/src/valita.js";
|
|
4
4
|
import { must } from "../../../shared/src/must.js";
|
|
5
|
-
import { parentWorker, singleProcessMode } from "../types/processes.js";
|
|
5
|
+
import { childWorker, parentWorker, singleProcessMode } from "../types/processes.js";
|
|
6
6
|
import { getShardConfig } from "../types/shards.js";
|
|
7
7
|
import { getNormalizedZeroConfig } from "../config/zero-config.js";
|
|
8
8
|
import { Database } from "../../../zqlite/src/db.js";
|
|
@@ -10,6 +10,7 @@ import { registerSQLiteCorruptionDiagnosticTarget } from "../db/sqlite-corruptio
|
|
|
10
10
|
import { getOrCreateGauge } from "../observability/metrics.js";
|
|
11
11
|
import { SupportingProcesses, exitAfter, runUntilKilled } from "../services/life-cycle.js";
|
|
12
12
|
import { createLogContext } from "./logging.js";
|
|
13
|
+
import { SERVING_CHECKPOINTER_URL } from "./worker-urls.js";
|
|
13
14
|
import { installProfileHandler } from "../types/profiler.js";
|
|
14
15
|
import { initEventSink } from "../observability/events.js";
|
|
15
16
|
import { ReplicationStatusPublisher } from "../services/replicator/replication-status.js";
|
|
@@ -59,7 +60,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
59
60
|
}, config.sqliteCorruptionChecks);
|
|
60
61
|
deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);
|
|
61
62
|
setupMetrics(lc, dbPath, walMode, pageSize);
|
|
62
|
-
const pragmas = getPragmaConfig(
|
|
63
|
+
const pragmas = getPragmaConfig();
|
|
63
64
|
const checkpoint = mode === "backup" && config.litestream.backupURL && config.litestream.backupUsingV5 && config.litestream.forceCheckpointThresholdMB ? {
|
|
64
65
|
checkpointThresholdPages: config.litestream.forceCheckpointThresholdMB * 1024 ** 2 / pageSize,
|
|
65
66
|
maxWalPages: !config.litestream.maxWalSizeMB ? void 0 : config.litestream.maxWalSizeMB * 1024 ** 2 / pageSize
|
|
@@ -68,6 +69,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
68
69
|
await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);
|
|
69
70
|
const processes = new SupportingProcesses(lc, parent);
|
|
70
71
|
if (mode === "backup") processes.add(startReplicaBackupProcess(lc, config.litestream, dbPath), "litestream");
|
|
72
|
+
else processes.add(childWorker(SERVING_CHECKPOINTER_URL, env, dbPath), "serving-checkpointer");
|
|
71
73
|
const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null, {
|
|
72
74
|
readinessThresholdMs: initialReadinessThresholdMs,
|
|
73
75
|
maxReadinessDelayMs
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n LitestreamConfig,\n NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n ChangeStreamerHttpClient,\n type ReservationFollowup,\n type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n exitAfter,\n runUntilKilled,\n SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n readLocalReplica,\n startReplicaBackupProcess,\n tryRestore,\n type RestoreAttempt,\n type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n ThreadWriteWorkerClient,\n type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type DrainRequestedMessage,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n deleteStaleChangeLog,\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const workerName = `${mode}-replicator`;\n installProfileHandler(parent, workerName);\n startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n lc = createLogContext(config, workerName);\n const unregisterInitialCorruptionDiagnosticTarget =\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n // Remote view-syncers restore from the replication-manager, which supplies\n // the backup URL. A local change-streamer owns the canonical replica and\n // must not infer backup configuration from bundled executable paths.\n //\n // The reservation's connection (`initialConnection`) is threaded through to\n // IncrementalSyncer's first subscribe(), so the replication-manager that\n // reserved the change log for this restore is the one that serves the\n // subscription (single-connection handoff).\n let initialConnection: ReservationFollowup | undefined;\n if (\n fileMode === 'serving' &&\n !runningLocalChangeStreamer &&\n (config.litestream.executable || config.litestream.executableV5)\n ) {\n initialConnection = await restoreReplica(lc, config, changeStreamer);\n }\n\n const {\n file: dbPath,\n walMode,\n pageSize,\n } = await setupReplica(lc, fileMode, config.replica);\n unregisterInitialCorruptionDiagnosticTarget();\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath,\n },\n config.sqliteCorruptionChecks,\n );\n\n // A file left behind by a run with the writer enabled would otherwise hold\n // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n // this is keyed on the config flag and nothing else.\n deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n setupMetrics(lc, dbPath, walMode, pageSize);\n\n // Create the write worker for async SQLite writes. Enable write-path\n // checkpoint backpressure only for the backup replicator on litestream v5,\n // which owns the replica litestream backs up.\n const pragmas = getPragmaConfig(fileMode);\n const checkpoint: ForceCheckpointConfig | null =\n mode === 'backup' &&\n config.litestream.backupURL &&\n config.litestream.backupUsingV5 &&\n config.litestream.forceCheckpointThresholdMB\n ? {\n checkpointThresholdPages:\n (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n pageSize,\n maxWalPages: !config.litestream.maxWalSizeMB\n ? undefined\n : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n }\n : null;\n\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n // The backup-replicator manages the litestream backup process. It is\n // started after the write worker has prepared the db file in order to\n // avoid contending on the sqlite lock.\n const processes = new SupportingProcesses(lc, parent);\n if (mode === 'backup') {\n processes.add(\n startReplicaBackupProcess(lc, config.litestream, dbPath),\n 'litestream',\n );\n }\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n {\n readinessThresholdMs: initialReadinessThresholdMs,\n maxReadinessDelayMs,\n },\n initialConnection,\n // View-syncers (i.e. with a remote change-streamer) switch over to\n // another replication-manager when theirs drains.\n runningLocalChangeStreamer\n ? undefined\n : {\n onRestoreGranted: () =>\n parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n replicaFile: dbPath,\n },\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n void replicator.ready().then(() => {\n parent.send(['ready', {ready: true}]);\n });\n\n return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n lc: LogContext,\n file: string,\n walMode: WalMode,\n pageSize: number,\n) {\n let db: Database | undefined;\n const getDb = () => {\n if (!db) {\n try {\n db = new Database(lc, file, {readonly: true});\n } catch (e) {\n lc.warn?.(`unable to open ${file} for wal metrics`, e);\n }\n }\n return db;\n };\n\n getOrCreateGauge('replica', 'db_size', {\n description:\n `The size of the replica's main db file, ` +\n `which does not include the wal file(s)`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, file));\n\n getOrCreateGauge('replica', 'wal_size', {\n description: `The size of the replica's wal file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n if (walMode === 'wal2') {\n getOrCreateGauge('replica', 'wal2_size', {\n description: `The size of the replica's wal2 file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal2`));\n }\n\n getOrCreateGauge('replica', 'wal_active_bytes', {\n description:\n `The actual size of active WAL frames currently in use ` +\n `(excluding pre-allocated or reclaimed space).`,\n unit: 'bytes',\n }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n description:\n `The size of un-checkpointed WAL frames that have not yet ` +\n `been backfilled into the main database.`,\n unit: 'bytes',\n }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n // A whole-database total rather than a per-file one. Its counterpart,\n // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n // the process that writes the log.\n getOrCreateGauge('replica', 'file_bytes', {\n description:\n `The replica's total on-disk footprint: its main db file plus its ` +\n `wal sidecars.`,\n unit: 'By',\n }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n return async o => {\n try {\n const stats = await stat(file);\n o.observe(stats.size);\n } catch (e) {\n lc.warn?.(`unable to stat ${file} for size metrics`, e);\n }\n };\n}\n\nfunction observeWalActiveBytes(\n lc: LogContext,\n getDb: () => Database | undefined,\n pageSize: number,\n): ObservableCallback {\n return o => {\n try {\n const db = getDb();\n if (!db) {\n return;\n }\n const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n if (typeof log === 'number') {\n o.observe(Math.max(0, log) * pageSize);\n }\n } catch (e) {\n lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n }\n };\n}\n\nfunction observeWalUncheckpointedBytes(\n lc: LogContext,\n getDb: () => Database | undefined,\n pageSize: number,\n): ObservableCallback {\n return o => {\n try {\n const db = getDb();\n if (!db) {\n return;\n }\n const [{log, checkpointed}] = db.pragma<{\n log: number;\n checkpointed: number;\n }>('wal_checkpoint(NOOP)');\n if (typeof log === 'number' && typeof checkpointed === 'number') {\n o.observe(Math.max(0, log - checkpointed) * pageSize);\n }\n } catch (e) {\n lc.warn?.(\n 'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n e,\n );\n }\n };\n}\n\nfunction observeSQLiteFileBytes(\n lc: LogContext,\n file: string,\n): ObservableCallback {\n return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n lc: LogContext,\n config: NormalizedZeroConfig,\n changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n const start = performance.now();\n let backupURL: string | undefined;\n let result: RestoreResult | undefined;\n const progress = new RestoreProgressReporter(lc, config.replica.file);\n let olderReplicaVersionRetries = 0;\n // Describing the local replica (if any) allows the replication-manager to\n // account for replicas that are being replaced (see RestoreAdmission). It\n // is read once, before the first attempt, as a failed attempt may have\n // already deleted it; it continues to describe the replica being replaced\n // (and the version not to downgrade from) on every retry.\n const localReplica = readLocalReplica(lc, config.replica.file);\n const localReplicaVersion = localReplica?.replicaVersion;\n try {\n for (;;) {\n const {reserved, followup} = await reserveAndGetSnapshotStatus(\n lc,\n config.taskID,\n changeStreamer,\n localReplica,\n );\n if (\n localReplicaVersion !== undefined &&\n reserved.replicaVersion < localReplicaVersion &&\n olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n ) {\n // Don't replace a newer local replica with an older one, e.g. from\n // a replication-manager that has yet to be replaced by a resync.\n followup.cancel();\n lc.info?.(\n `replication-manager is at an older replica version ` +\n `(${reserved.replicaVersion}) than the local replica ` +\n `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n continue;\n }\n // The backupURL comes from the replication-manager's snapshot response.\n ({backupURL} = reserved);\n const litestream: LitestreamConfig = {...config.litestream, backupURL};\n progress.start(reserved.replicaSize);\n let attempt: RestoreAttempt;\n try {\n attempt = await tryRestore(\n lc,\n litestream,\n config.replica.file,\n reserved,\n 'view_syncer',\n followup.signal, // abort the restore if the reservation disconnects\n );\n } catch (e) {\n if (followup.signal.aborted) {\n lc.info?.(\n `reservation disconnected during restore. retrying`,\n String(e),\n );\n continue;\n }\n followup.cancel();\n throw e;\n } finally {\n progress.stop();\n }\n if (attempt.restored) {\n result = attempt.result;\n progress.done();\n return followup; // connection to be converted to a subscription\n }\n // This attempt's reservation is stale; a fresh one is made on retry.\n followup.cancel();\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } finally {\n progress.stop();\n const attrs = litestreamRestoreMetricAttrs(\n config.litestream,\n 'view_syncer',\n backupURL,\n );\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB,QAAQ;CACxC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAKrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,CAAC,CAAC,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
|
|
1
|
+
{"version":3,"file":"replicator.js","names":["v.parse"],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {stat} from 'node:fs/promises';\nimport {pid} from 'node:process';\nimport type {ObservableCallback} from '@opentelemetry/api';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {\n LitestreamConfig,\n NormalizedZeroConfig,\n} from '../config/normalize.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {registerSQLiteCorruptionDiagnosticTarget} from '../db/sqlite-corruption.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {getOrCreateGauge} from '../observability/metrics.ts';\nimport {\n ChangeStreamerHttpClient,\n type ReservationFollowup,\n type SnapshotReserver,\n} from '../services/change-streamer/change-streamer-http.ts';\nimport {reserveAndGetSnapshotStatus} from '../services/change-streamer/snapshot.ts';\nimport {\n exitAfter,\n runUntilKilled,\n SupportingProcesses,\n} from '../services/life-cycle.ts';\nimport {\n readLocalReplica,\n startReplicaBackupProcess,\n tryRestore,\n type RestoreAttempt,\n type RestoreResult,\n} from '../services/litestream/commands.ts';\nimport {\n litestreamRestoreDuration,\n litestreamRestoreMetricAttrs,\n litestreamRestoreRuns,\n} from '../services/litestream/metrics.ts';\nimport {RestoreProgressReporter} from '../services/litestream/restore-progress.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {sqliteFileBytes} from '../services/replicator/sqlite-change-log-observability.ts';\nimport {\n ThreadWriteWorkerClient,\n type ForceCheckpointConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport {\n childWorker,\n parentWorker,\n singleProcessMode,\n type DrainRequestedMessage,\n type Worker,\n} from '../types/processes.ts';\nimport {installProfileHandler} from '../types/profiler.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n deleteStaleChangeLog,\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n type WalMode,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\nimport {SERVING_CHECKPOINTER_URL} from './worker-urls.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const workerName = `${mode}-replicator`;\n installProfileHandler(parent, workerName);\n startOtelAuto(createLogContext(config, workerName, 0, false), workerName, 0);\n lc = createLogContext(config, workerName);\n const unregisterInitialCorruptionDiagnosticTarget =\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath: config.replica.file,\n },\n config.sqliteCorruptionChecks,\n );\n initEventSink(lc, config);\n\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n replicationLag: {initialReadinessThresholdMs, maxReadinessDelayMs},\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n // Remote view-syncers restore from the replication-manager, which supplies\n // the backup URL. A local change-streamer owns the canonical replica and\n // must not infer backup configuration from bundled executable paths.\n //\n // The reservation's connection (`initialConnection`) is threaded through to\n // IncrementalSyncer's first subscribe(), so the replication-manager that\n // reserved the change log for this restore is the one that serves the\n // subscription (single-connection handoff).\n let initialConnection: ReservationFollowup | undefined;\n if (\n fileMode === 'serving' &&\n !runningLocalChangeStreamer &&\n (config.litestream.executable || config.litestream.executableV5)\n ) {\n initialConnection = await restoreReplica(lc, config, changeStreamer);\n }\n\n const {\n file: dbPath,\n walMode,\n pageSize,\n } = await setupReplica(lc, fileMode, config.replica);\n unregisterInitialCorruptionDiagnosticTarget();\n registerSQLiteCorruptionDiagnosticTarget(\n {\n debugName: `${workerName} replica`,\n dbPath,\n },\n config.sqliteCorruptionChecks,\n );\n\n // A file left behind by a run with the writer enabled would otherwise hold\n // local disk indefinitely. See `replicatorDeletesStaleChangeLog` for why\n // this is keyed on the config flag and nothing else.\n deleteStaleChangeLog(config.changeStreamer.sqliteChangeLogMode, dbPath);\n\n setupMetrics(lc, dbPath, walMode, pageSize);\n\n // Create the write worker for async SQLite writes. Enable write-path\n // checkpoint backpressure only for the backup replicator on litestream v5,\n // which owns the replica litestream backs up.\n const pragmas = getPragmaConfig();\n const checkpoint: ForceCheckpointConfig | null =\n mode === 'backup' &&\n config.litestream.backupURL &&\n config.litestream.backupUsingV5 &&\n config.litestream.forceCheckpointThresholdMB\n ? {\n checkpointThresholdPages:\n (config.litestream.forceCheckpointThresholdMB * 1024 ** 2) /\n pageSize,\n maxWalPages: !config.litestream.maxWalSizeMB\n ? undefined\n : (config.litestream.maxWalSizeMB * 1024 ** 2) / pageSize,\n }\n : null;\n\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log, checkpoint);\n\n // The backup-replicator manages the litestream backup process, which\n // checkpoints the backup replica. A serving replicator instead manages a\n // checkpointer for its replica. Either is started after the write worker\n // has prepared the db file in order to avoid contending on the sqlite lock.\n const processes = new SupportingProcesses(lc, parent);\n if (mode === 'backup') {\n processes.add(\n startReplicaBackupProcess(lc, config.litestream, dbPath),\n 'litestream',\n );\n } else {\n processes.add(\n childWorker(SERVING_CHECKPOINTER_URL, env, dbPath),\n 'serving-checkpointer',\n );\n }\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n {\n readinessThresholdMs: initialReadinessThresholdMs,\n maxReadinessDelayMs,\n },\n initialConnection,\n // View-syncers (i.e. with a remote change-streamer) switch over to\n // another replication-manager when theirs drains.\n runningLocalChangeStreamer\n ? undefined\n : {\n onRestoreGranted: () =>\n parent.send<DrainRequestedMessage>(['drainRequested', {}]),\n replicaFile: dbPath,\n },\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n void replicator.ready().then(() => {\n parent.send(['ready', {ready: true}]);\n });\n\n return runUntilKilled(lc, parent, replicator, processes);\n}\n\nexport function setupMetrics(\n lc: LogContext,\n file: string,\n walMode: WalMode,\n pageSize: number,\n) {\n let db: Database | undefined;\n const getDb = () => {\n if (!db) {\n try {\n db = new Database(lc, file, {readonly: true});\n } catch (e) {\n lc.warn?.(`unable to open ${file} for wal metrics`, e);\n }\n }\n return db;\n };\n\n getOrCreateGauge('replica', 'db_size', {\n description:\n `The size of the replica's main db file, ` +\n `which does not include the wal file(s)`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, file));\n\n getOrCreateGauge('replica', 'wal_size', {\n description: `The size of the replica's wal file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal`));\n\n if (walMode === 'wal2') {\n getOrCreateGauge('replica', 'wal2_size', {\n description: `The size of the replica's wal2 file`,\n unit: 'bytes',\n }).addCallback(observeFileSize(lc, `${file}-wal2`));\n }\n\n getOrCreateGauge('replica', 'wal_active_bytes', {\n description:\n `The actual size of active WAL frames currently in use ` +\n `(excluding pre-allocated or reclaimed space).`,\n unit: 'bytes',\n }).addCallback(observeWalActiveBytes(lc, getDb, pageSize));\n\n getOrCreateGauge('replica', 'wal_uncheckpointed_bytes', {\n description:\n `The size of un-checkpointed WAL frames that have not yet ` +\n `been backfilled into the main database.`,\n unit: 'bytes',\n }).addCallback(observeWalUncheckpointedBytes(lc, getDb, pageSize));\n\n // A whole-database total rather than a per-file one. Its counterpart,\n // `sqlite_change_log.file_bytes`, is emitted by the change-streamer, which is\n // the process that writes the log.\n getOrCreateGauge('replica', 'file_bytes', {\n description:\n `The replica's total on-disk footprint: its main db file plus its ` +\n `wal sidecars.`,\n unit: 'By',\n }).addCallback(observeSQLiteFileBytes(lc, file));\n}\n\nfunction observeFileSize(lc: LogContext, file: string): ObservableCallback {\n return async o => {\n try {\n const stats = await stat(file);\n o.observe(stats.size);\n } catch (e) {\n lc.warn?.(`unable to stat ${file} for size metrics`, e);\n }\n };\n}\n\nfunction observeWalActiveBytes(\n lc: LogContext,\n getDb: () => Database | undefined,\n pageSize: number,\n): ObservableCallback {\n return o => {\n try {\n const db = getDb();\n if (!db) {\n return;\n }\n const [{log}] = db.pragma<{log: number}>('wal_checkpoint(NOOP)');\n if (typeof log === 'number') {\n o.observe(Math.max(0, log) * pageSize);\n }\n } catch (e) {\n lc.warn?.('unable to inspect wal_checkpoint for active bytes metric', e);\n }\n };\n}\n\nfunction observeWalUncheckpointedBytes(\n lc: LogContext,\n getDb: () => Database | undefined,\n pageSize: number,\n): ObservableCallback {\n return o => {\n try {\n const db = getDb();\n if (!db) {\n return;\n }\n const [{log, checkpointed}] = db.pragma<{\n log: number;\n checkpointed: number;\n }>('wal_checkpoint(NOOP)');\n if (typeof log === 'number' && typeof checkpointed === 'number') {\n o.observe(Math.max(0, log - checkpointed) * pageSize);\n }\n } catch (e) {\n lc.warn?.(\n 'unable to inspect wal_checkpoint for uncheckpointed bytes metric',\n e,\n );\n }\n };\n}\n\nfunction observeSQLiteFileBytes(\n lc: LogContext,\n file: string,\n): ObservableCallback {\n return async o => o.observe(await sqliteFileBytes(lc, file));\n}\n\nconst RETRY_INTERVAL_MS = 3000;\n\n// The number of times a view-syncer retries a snapshot reservation from a\n// replication-manager with an older replica version than its local replica\n// (e.g. a not-yet-terminated replication-manager during a resync rollout),\n// before accepting the older version (e.g. if the upstream was legitimately\n// replaced by one with a lower replica version).\nconst MAX_OLDER_REPLICA_VERSION_RETRIES = 20;\n\n// View-syncers wait indefinitely for the replication-manager to publish a\n// restorable backup. On a fresh stack the first backup is not durable until\n// the initial sync completes and litestream uploads the initial snapshot,\n// which can take many minutes for a large replica. The platform's startup\n// probe budget (which scales with replica size) is the backstop, so\n// restoreReplica must not impose its own shorter cap and self-terminate\n// while the backup is still being produced.\n//\n// Returns the `followup` of the reservation whose backup was actually\n// restored, still open, so the caller can hand it to IncrementalSyncer's\n// first subscribe() via `followup.subscribe()`: the same replication-manager\n// that reserved the change log is the one that serves the subscription.\n/** Exported for testing. */\nexport async function restoreReplica(\n lc: LogContext,\n config: NormalizedZeroConfig,\n changeStreamer: SnapshotReserver,\n): Promise<ReservationFollowup> {\n const start = performance.now();\n let backupURL: string | undefined;\n let result: RestoreResult | undefined;\n const progress = new RestoreProgressReporter(lc, config.replica.file);\n let olderReplicaVersionRetries = 0;\n // Describing the local replica (if any) allows the replication-manager to\n // account for replicas that are being replaced (see RestoreAdmission). It\n // is read once, before the first attempt, as a failed attempt may have\n // already deleted it; it continues to describe the replica being replaced\n // (and the version not to downgrade from) on every retry.\n const localReplica = readLocalReplica(lc, config.replica.file);\n const localReplicaVersion = localReplica?.replicaVersion;\n try {\n for (;;) {\n const {reserved, followup} = await reserveAndGetSnapshotStatus(\n lc,\n config.taskID,\n changeStreamer,\n localReplica,\n );\n if (\n localReplicaVersion !== undefined &&\n reserved.replicaVersion < localReplicaVersion &&\n olderReplicaVersionRetries++ < MAX_OLDER_REPLICA_VERSION_RETRIES\n ) {\n // Don't replace a newer local replica with an older one, e.g. from\n // a replication-manager that has yet to be replaced by a resync.\n followup.cancel();\n lc.info?.(\n `replication-manager is at an older replica version ` +\n `(${reserved.replicaVersion}) than the local replica ` +\n `(${localReplicaVersion}). retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n continue;\n }\n // The backupURL comes from the replication-manager's snapshot response.\n ({backupURL} = reserved);\n const litestream: LitestreamConfig = {...config.litestream, backupURL};\n progress.start(reserved.replicaSize);\n let attempt: RestoreAttempt;\n try {\n attempt = await tryRestore(\n lc,\n litestream,\n config.replica.file,\n reserved,\n 'view_syncer',\n followup.signal, // abort the restore if the reservation disconnects\n );\n } catch (e) {\n if (followup.signal.aborted) {\n lc.info?.(\n `reservation disconnected during restore. retrying`,\n String(e),\n );\n continue;\n }\n followup.cancel();\n throw e;\n } finally {\n progress.stop();\n }\n if (attempt.restored) {\n result = attempt.result;\n progress.done();\n return followup; // connection to be converted to a subscription\n }\n // This attempt's reservation is stale; a fresh one is made on retry.\n followup.cancel();\n lc.info?.(\n `replica not found. retrying in ${RETRY_INTERVAL_MS / 1000} seconds`,\n );\n await sleep(RETRY_INTERVAL_MS);\n }\n } finally {\n progress.stop();\n const attrs = litestreamRestoreMetricAttrs(\n config.litestream,\n 'view_syncer',\n backupURL,\n );\n const labels = {...attrs, result: result ?? 'error'};\n litestreamRestoreRuns().add(1, labels);\n litestreamRestoreDuration().recordMs(performance.now() - start, labels);\n }\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyEA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;AAElD,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;CACf,OAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,WAAWA,MAAQ,KAAK,IAAI,qBAAqB;CAEvD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,CAAC;CAAC,CAAC;CACjE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,aAAa,GAAG,KAAK;CAC3B,sBAAsB,QAAQ,UAAU;CACxC,cAAc,iBAAiB,QAAQ,YAAY,GAAG,KAAK,GAAG,YAAY,CAAC;CAC3E,KAAK,iBAAiB,QAAQ,UAAU;CACxC,MAAM,8CACJ,yCACE;EACE,WAAW,GAAG,WAAW;EACzB,QAAQ,OAAO,QAAQ;CACzB,GACA,OAAO,sBACT;CACF,cAAc,IAAI,MAAM;CAExB,MAAM,QAAQ,eAAe,MAAM;CACnC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,KAEN,gBAAgB,EAAC,6BAA6B,0BAC5C;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,iBACF;CAUA,IAAI;CACJ,IACE,aAAa,aACb,CAAC,+BACA,OAAO,WAAW,cAAc,OAAO,WAAW,eAEnD,oBAAoB,MAAM,eAAe,IAAI,QAAQ,cAAc;CAGrE,MAAM,EACJ,MAAM,QACN,SACA,aACE,MAAM,aAAa,IAAI,UAAU,OAAO,OAAO;CACnD,4CAA4C;CAC5C,yCACE;EACE,WAAW,GAAG,WAAW;EACzB;CACF,GACA,OAAO,sBACT;CAKA,qBAAqB,OAAO,eAAe,qBAAqB,MAAM;CAEtE,aAAa,IAAI,QAAQ,SAAS,QAAQ;CAK1C,MAAM,UAAU,gBAAgB;CAChC,MAAM,aACJ,SAAS,YACT,OAAO,WAAW,aAClB,OAAO,WAAW,iBAClB,OAAO,WAAW,6BACd;EACE,0BACG,OAAO,WAAW,6BAA6B,QAAQ,IACxD;EACF,aAAa,CAAC,OAAO,WAAW,eAC5B,KAAA,IACC,OAAO,WAAW,eAAe,QAAQ,IAAK;CACrD,IACA;CAEN,MAAM,eAAe,IAAI,wBAAwB;CACjD,MAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,KAAK,UAAU;CAMrE,MAAM,YAAY,IAAI,oBAAoB,IAAI,MAAM;CACpD,IAAI,SAAS,UACX,UAAU,IACR,0BAA0B,IAAI,OAAO,YAAY,MAAM,GACvD,YACF;MAEA,UAAU,IACR,YAAY,0BAA0B,KAAK,MAAM,GACjD,sBACF;CAGF,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,MAAM,IAChD,MACJ;EACE,sBAAsB;EACtB;CACF,GACA,mBAGA,6BACI,KAAA,IACA;EACE,wBACE,OAAO,KAA4B,CAAC,kBAAkB,CAAC,CAAC,CAAC;EAC3D,aAAa;CACf,CACN;CAEA,qBAAqB,IAAI,YAAY,MAAM;CAE3C,WAAgB,MAAM,CAAC,CAAC,WAAW;EACjC,OAAO,KAAK,CAAC,SAAS,EAAC,OAAO,KAAI,CAAC,CAAC;CACtC,CAAC;CAED,OAAO,eAAe,IAAI,QAAQ,YAAY,SAAS;AACzD;AAEA,SAAgB,aACd,IACA,MACA,SACA,UACA;CACA,IAAI;CACJ,MAAM,cAAc;EAClB,IAAI,CAAC,IACH,IAAI;GACF,KAAK,IAAI,SAAS,IAAI,MAAM,EAAC,UAAU,KAAI,CAAC;EAC9C,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,mBAAmB,CAAC;EACvD;EAEF,OAAO;CACT;CAEA,iBAAiB,WAAW,WAAW;EACrC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,IAAI,CAAC;CAExC,iBAAiB,WAAW,YAAY;EACtC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,KAAK,CAAC;CAEjD,IAAI,YAAY,QACd,iBAAiB,WAAW,aAAa;EACvC,aAAa;EACb,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,gBAAgB,IAAI,GAAG,KAAK,MAAM,CAAC;CAGpD,iBAAiB,WAAW,oBAAoB;EAC9C,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,sBAAsB,IAAI,OAAO,QAAQ,CAAC;CAEzD,iBAAiB,WAAW,4BAA4B;EACtD,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,8BAA8B,IAAI,OAAO,QAAQ,CAAC;CAKjE,iBAAiB,WAAW,cAAc;EACxC,aACE;EAEF,MAAM;CACR,CAAC,CAAC,CAAC,YAAY,uBAAuB,IAAI,IAAI,CAAC;AACjD;AAEA,SAAS,gBAAgB,IAAgB,MAAkC;CACzE,OAAO,OAAM,MAAK;EAChB,IAAI;GACF,MAAM,QAAQ,MAAM,KAAK,IAAI;GAC7B,EAAE,QAAQ,MAAM,IAAI;EACtB,SAAS,GAAG;GACV,GAAG,OAAO,kBAAkB,KAAK,oBAAoB,CAAC;EACxD;CACF;AACF;AAEA,SAAS,sBACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,SAAQ,GAAG,OAAsB,sBAAsB;GAC/D,IAAI,OAAO,QAAQ,UACjB,EAAE,QAAQ,KAAK,IAAI,GAAG,GAAG,IAAI,QAAQ;EAEzC,SAAS,GAAG;GACV,GAAG,OAAO,4DAA4D,CAAC;EACzE;CACF;AACF;AAEA,SAAS,8BACP,IACA,OACA,UACoB;CACpB,QAAO,MAAK;EACV,IAAI;GACF,MAAM,KAAK,MAAM;GACjB,IAAI,CAAC,IACH;GAEF,MAAM,CAAC,EAAC,KAAK,kBAAiB,GAAG,OAG9B,sBAAsB;GACzB,IAAI,OAAO,QAAQ,YAAY,OAAO,iBAAiB,UACrD,EAAE,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,IAAI,QAAQ;EAExD,SAAS,GAAG;GACV,GAAG,OACD,oEACA,CACF;EACF;CACF;AACF;AAEA,SAAS,uBACP,IACA,MACoB;CACpB,OAAO,OAAM,MAAK,EAAE,QAAQ,MAAM,gBAAgB,IAAI,IAAI,CAAC;AAC7D;AAEA,IAAM,oBAAoB;AAO1B,IAAM,oCAAoC;;AAe1C,eAAsB,eACpB,IACA,QACA,gBAC8B;CAC9B,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI;CACJ,IAAI;CACJ,MAAM,WAAW,IAAI,wBAAwB,IAAI,OAAO,QAAQ,IAAI;CACpE,IAAI,6BAA6B;CAMjC,MAAM,eAAe,iBAAiB,IAAI,OAAO,QAAQ,IAAI;CAC7D,MAAM,sBAAsB,cAAc;CAC1C,IAAI;EACF,SAAS;GACP,MAAM,EAAC,UAAU,aAAY,MAAM,4BACjC,IACA,OAAO,QACP,gBACA,YACF;GACA,IACE,wBAAwB,KAAA,KACxB,SAAS,iBAAiB,uBAC1B,+BAA+B,mCAC/B;IAGA,SAAS,OAAO;IAChB,GAAG,OACD,uDACM,SAAS,eAAe,4BACxB,oBAAoB,iBAAiB,oBAAoB,IAAK,SACtE;IACA,MAAM,MAAM,iBAAiB;IAC7B;GACF;GAEA,CAAC,CAAC,aAAa;GACf,MAAM,aAA+B;IAAC,GAAG,OAAO;IAAY;GAAS;GACrE,SAAS,MAAM,SAAS,WAAW;GACnC,IAAI;GACJ,IAAI;IACF,UAAU,MAAM,WACd,IACA,YACA,OAAO,QAAQ,MACf,UACA,eACA,SAAS,MACX;GACF,SAAS,GAAG;IACV,IAAI,SAAS,OAAO,SAAS;KAC3B,GAAG,OACD,qDACA,OAAO,CAAC,CACV;KACA;IACF;IACA,SAAS,OAAO;IAChB,MAAM;GACR,UAAU;IACR,SAAS,KAAK;GAChB;GACA,IAAI,QAAQ,UAAU;IACpB,SAAS,QAAQ;IACjB,SAAS,KAAK;IACd,OAAO;GACT;GAEA,SAAS,OAAO;GAChB,GAAG,OACD,kCAAkC,oBAAoB,IAAK,SAC7D;GACA,MAAM,MAAM,iBAAiB;EAC/B;CACF,UAAU;EACR,SAAS,KAAK;EAMd,MAAM,SAAS;GAAC,GALF,6BACZ,OAAO,YACP,eACA,SAEiB;GAAO,QAAQ,UAAU;EAAO;EACnD,sBAAsB,CAAC,CAAC,IAAI,GAAG,MAAM;EACrC,0BAA0B,CAAC,CAAC,SAAS,YAAY,IAAI,IAAI,OAAO,MAAM;CACxE;AACF;AAGA,IAAI,CAAC,kBAAkB,GACrB,gBACQ,UACA,UAAU,KAAK,YAAY,GAAG,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC3E"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { assert } from "../../../shared/src/asserts.js";
|
|
2
|
+
import { parentWorker, singleProcessMode } from "../types/processes.js";
|
|
3
|
+
import { getLogConfig } from "../config/zero-config.js";
|
|
4
|
+
import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
|
|
5
|
+
import { createLogContext } from "./logging.js";
|
|
6
|
+
import { startOtelAuto } from "./otel-start.js";
|
|
7
|
+
import { ServingCheckpointer } from "../services/replicator/serving-checkpointer.js";
|
|
8
|
+
import { LogContext, consoleLogSink } from "@rocicorp/logger";
|
|
9
|
+
import { resolve } from "node:path";
|
|
10
|
+
import { fileURLToPath } from "node:url";
|
|
11
|
+
//#region ../zero-cache/src/server/serving-checkpointer.ts
|
|
12
|
+
var lc = new LogContext("info", {}, consoleLogSink);
|
|
13
|
+
/**
|
|
14
|
+
* Checkpoints a serving replica on behalf of its replicator, which starts
|
|
15
|
+
* this worker as one of its supporting processes once the replica is set up,
|
|
16
|
+
* passing the path of its replica file as the first argument.
|
|
17
|
+
*
|
|
18
|
+
* Only the logging config is parsed from the remaining arguments, which
|
|
19
|
+
* ignores everything else. In particular, `childWorker()` forwards the
|
|
20
|
+
* replicator's own arguments, which start with its replica file mode and
|
|
21
|
+
* would fail a full config parse.
|
|
22
|
+
*/
|
|
23
|
+
function runWorker(parent, env, ...args) {
|
|
24
|
+
const [dbPath, ...argv] = args;
|
|
25
|
+
assert(dbPath, `replica file not specified`);
|
|
26
|
+
const logConfig = getLogConfig({
|
|
27
|
+
env,
|
|
28
|
+
argv
|
|
29
|
+
});
|
|
30
|
+
const workerName = "serving-checkpointer";
|
|
31
|
+
startOtelAuto(createLogContext(logConfig, workerName, 0, false), workerName, 0);
|
|
32
|
+
lc = createLogContext(logConfig, workerName);
|
|
33
|
+
return runUntilKilled(lc, parent, new ServingCheckpointer(lc, dbPath));
|
|
34
|
+
}
|
|
35
|
+
var isMain = typeof process.argv[1] === "string" && fileURLToPath(import.meta.url) === resolve(process.argv[1]);
|
|
36
|
+
var parent = parentWorker;
|
|
37
|
+
if (isMain && parent !== null && !singleProcessMode()) exitAfter(() => lc, () => runWorker(parent, process.env, ...process.argv.slice(2)));
|
|
38
|
+
//#endregion
|
|
39
|
+
export { runWorker as default };
|
|
40
|
+
|
|
41
|
+
//# sourceMappingURL=serving-checkpointer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serving-checkpointer.js","names":[],"sources":["../../../../../zero-cache/src/server/serving-checkpointer.ts"],"sourcesContent":["import {resolve} from 'node:path';\nimport {fileURLToPath} from 'node:url';\nimport {consoleLogSink, LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {getLogConfig} from '../config/zero-config.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ServingCheckpointer} from '../services/replicator/serving-checkpointer.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\n// Default LogContext, overridden in runWorker\nlet lc = new LogContext('info', {}, consoleLogSink);\n\n/**\n * Checkpoints a serving replica on behalf of its replicator, which starts\n * this worker as one of its supporting processes once the replica is set up,\n * passing the path of its replica file as the first argument.\n *\n * Only the logging config is parsed from the remaining arguments, which\n * ignores everything else. In particular, `childWorker()` forwards the\n * replicator's own arguments, which start with its replica file mode and\n * would fail a full config parse.\n */\nexport default function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n const [dbPath, ...argv] = args;\n assert(dbPath, `replica file not specified`);\n const logConfig = getLogConfig({env, argv});\n const workerName = 'serving-checkpointer';\n startOtelAuto(\n createLogContext(logConfig, workerName, 0, false),\n workerName,\n 0,\n );\n lc = createLogContext(logConfig, workerName);\n\n return runUntilKilled(lc, parent, new ServingCheckpointer(lc, dbPath));\n}\n\n// fork()\nconst isMain =\n typeof process.argv[1] === 'string' &&\n fileURLToPath(import.meta.url) === resolve(process.argv[1]);\n\nconst parent = parentWorker;\nif (isMain && parent !== null && !singleProcessMode()) {\n void exitAfter(\n () => lc,\n () => runWorker(parent, process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;AAgBA,IAAI,KAAK,IAAI,WAAW,QAAQ,CAAC,GAAG,cAAc;;;;;;;;;;;AAYlD,SAAwB,UACtB,QACA,KACA,GAAG,MACY;CACf,MAAM,CAAC,QAAQ,GAAG,QAAQ;CAC1B,OAAO,QAAQ,4BAA4B;CAC3C,MAAM,YAAY,aAAa;EAAC;EAAK;CAAI,CAAC;CAC1C,MAAM,aAAa;CACnB,cACE,iBAAiB,WAAW,YAAY,GAAG,KAAK,GAChD,YACA,CACF;CACA,KAAK,iBAAiB,WAAW,UAAU;CAE3C,OAAO,eAAe,IAAI,QAAQ,IAAI,oBAAoB,IAAI,MAAM,CAAC;AACvE;AAGA,IAAM,SACJ,OAAO,QAAQ,KAAK,OAAO,YAC3B,cAAc,YAAY,GAAG,MAAM,QAAQ,QAAQ,KAAK,EAAE;AAE5D,IAAM,SAAS;AACf,IAAI,UAAU,WAAW,QAAQ,CAAC,kBAAkB,GAClD,gBACQ,UACA,UAAU,QAAQ,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,CAAC,CAAC,CAC/D"}
|
|
@@ -3,6 +3,7 @@ export declare const MAIN_URL: URL;
|
|
|
3
3
|
export declare const MUTATOR_URL: URL;
|
|
4
4
|
export declare const REAPER_URL: URL;
|
|
5
5
|
export declare const REPLICATOR_URL: URL;
|
|
6
|
+
export declare const SERVING_CHECKPOINTER_URL: URL;
|
|
6
7
|
export declare const SHADOW_SYNCER_URL: URL;
|
|
7
8
|
export declare const SLOT_KEEPER_URL: URL;
|
|
8
9
|
export declare const SYNCER_URL: URL;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,eAAe,KAA8B,CAAC;AAC3D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}
|
|
1
|
+
{"version":3,"file":"worker-urls.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,mBAAmB,KAAkC,CAAC;AACnE,eAAO,MAAM,QAAQ,KAAuB,CAAC;AAC7C,eAAO,MAAM,WAAW,KAA0B,CAAC;AACnD,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,cAAc,KAA6B,CAAC;AACzD,eAAO,MAAM,wBAAwB,KAAuC,CAAC;AAC7E,eAAO,MAAM,iBAAiB,KAAgC,CAAC;AAC/D,eAAO,MAAM,eAAe,KAA8B,CAAC;AAC3D,eAAO,MAAM,UAAU,KAAyB,CAAC;AACjD,eAAO,MAAM,gBAAgB,KAA+B,CAAC"}
|
|
@@ -10,11 +10,12 @@ var MAIN_URL = resolve("./main.ts");
|
|
|
10
10
|
resolve("./mutator.ts");
|
|
11
11
|
var REAPER_URL = resolve("./reaper.ts");
|
|
12
12
|
var REPLICATOR_URL = resolve("./replicator.ts");
|
|
13
|
+
var SERVING_CHECKPOINTER_URL = resolve("./serving-checkpointer.ts");
|
|
13
14
|
var SHADOW_SYNCER_URL = resolve("./shadow-syncer.ts");
|
|
14
15
|
var SLOT_KEEPER_URL = resolve("./slot-keeper.ts");
|
|
15
16
|
var SYNCER_URL = resolve("./syncer.ts");
|
|
16
17
|
var WRITE_WORKER_URL = resolve("./write-worker.ts");
|
|
17
18
|
//#endregion
|
|
18
|
-
export { CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SHADOW_SYNCER_URL, SLOT_KEEPER_URL, SYNCER_URL, WRITE_WORKER_URL };
|
|
19
|
+
export { CHANGE_STREAMER_URL, MAIN_URL, REAPER_URL, REPLICATOR_URL, SERVING_CHECKPOINTER_URL, SHADOW_SYNCER_URL, SLOT_KEEPER_URL, SYNCER_URL, WRITE_WORKER_URL };
|
|
19
20
|
|
|
20
21
|
//# sourceMappingURL=worker-urls.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n const {url} = import.meta;\n if (url.endsWith('.js')) {\n // When compiled, change .ts to .js\n path = path.replace(tsRe, '.js');\n }\n return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SLOT_KEEPER_URL = resolve('./slot-keeper.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO;CACd,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,kBAAkB,QAAQ,kBAAkB;AACzD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}
|
|
1
|
+
{"version":3,"file":"worker-urls.js","names":[],"sources":["../../../../../zero-cache/src/server/worker-urls.ts"],"sourcesContent":["// This module provides URLs for worker files.\n\nconst tsRe = /\\.ts$/;\n\nfunction resolve(path: string): URL {\n const {url} = import.meta;\n if (url.endsWith('.js')) {\n // When compiled, change .ts to .js\n path = path.replace(tsRe, '.js');\n }\n return new URL(path, url);\n}\n\n// These URLs are part of the build process. See ../../zero/tool/build.ts\n// All these urls must be relative to this file and be located in the same directory.\n\nexport const CHANGE_STREAMER_URL = resolve('./change-streamer.ts');\nexport const MAIN_URL = resolve('./main.ts');\nexport const MUTATOR_URL = resolve('./mutator.ts');\nexport const REAPER_URL = resolve('./reaper.ts');\nexport const REPLICATOR_URL = resolve('./replicator.ts');\nexport const SERVING_CHECKPOINTER_URL = resolve('./serving-checkpointer.ts');\nexport const SHADOW_SYNCER_URL = resolve('./shadow-syncer.ts');\nexport const SLOT_KEEPER_URL = resolve('./slot-keeper.ts');\nexport const SYNCER_URL = resolve('./syncer.ts');\nexport const WRITE_WORKER_URL = resolve('./write-worker.ts');\n"],"mappings":";AAEA,IAAM,OAAO;AAEb,SAAS,QAAQ,MAAmB;CAClC,MAAM,EAAC,QAAO;CACd,IAAI,IAAI,SAAS,KAAK,GAEpB,OAAO,KAAK,QAAQ,MAAM,KAAK;CAEjC,OAAO,IAAI,IAAI,MAAM,GAAG;AAC1B;AAKA,IAAa,sBAAsB,QAAQ,sBAAsB;AACjE,IAAa,WAAW,QAAQ,WAAW;AAChB,QAAQ,cAAc;AACjD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,iBAAiB,QAAQ,iBAAiB;AACvD,IAAa,2BAA2B,QAAQ,2BAA2B;AAC3E,IAAa,oBAAoB,QAAQ,oBAAoB;AAC7D,IAAa,kBAAkB,QAAQ,kBAAkB;AACzD,IAAa,aAAa,QAAQ,aAAa;AAC/C,IAAa,mBAAmB,QAAQ,mBAAmB"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { Database } from "../../../../zqlite/src/db.js";
|
|
2
|
+
import { RunningState } from "../running-state.js";
|
|
3
|
+
import { LogThrottle } from "../../observability/log-throttle.js";
|
|
4
|
+
//#region ../zero-cache/src/services/replicator/serving-checkpointer.ts
|
|
5
|
+
var CHECKPOINT_INTERVAL_MS = 100;
|
|
6
|
+
var SLOW_CHECKPOINT_MS = 1e3;
|
|
7
|
+
var WAL_WARNING_BYTES = 67108864;
|
|
8
|
+
var LOG_WINDOW_MS = 6e4;
|
|
9
|
+
var MB = 1048576;
|
|
10
|
+
/**
|
|
11
|
+
* Checkpoints the serving (wal2) replica from its own connection, so that the
|
|
12
|
+
* replicator's writer, which runs with `wal_autocheckpoint = 0`, never waits
|
|
13
|
+
* on a checkpoint's disk syncs.
|
|
14
|
+
*
|
|
15
|
+
* In wal2 mode the writer switches to the other wal file once the current
|
|
16
|
+
* one reaches its size limit, provided the other one has been checkpointed.
|
|
17
|
+
* Until then, it keeps appending to the current file rather than waiting. A
|
|
18
|
+
* `PASSIVE` checkpoint copies the whole non-current file, or does nothing
|
|
19
|
+
* (without any I/O) if that file is already checkpointed, empty, or still
|
|
20
|
+
* being read (e.g. by an old snapshot). Polling it on a short interval thus
|
|
21
|
+
* keeps the wal files near their size limit without any coordination with
|
|
22
|
+
* the writer.
|
|
23
|
+
*
|
|
24
|
+
* A failed checkpoint is logged and retried rather than stopping the
|
|
25
|
+
* replicator. This matches the writer's automatic checkpoints that this
|
|
26
|
+
* replaces, whose errors SQLite ignores: a persistent failure (e.g. a full or
|
|
27
|
+
* failing disk) surfaces as the writer's own failure once the wal can no
|
|
28
|
+
* longer grow.
|
|
29
|
+
*/
|
|
30
|
+
var ServingCheckpointer = class {
|
|
31
|
+
id = "serving-checkpointer";
|
|
32
|
+
#lc;
|
|
33
|
+
#db;
|
|
34
|
+
#pageSize;
|
|
35
|
+
#intervalMs;
|
|
36
|
+
#walWarningBytes;
|
|
37
|
+
#state = new RunningState(this.id);
|
|
38
|
+
#logThrottle = new LogThrottle({ windowMs: LOG_WINDOW_MS });
|
|
39
|
+
constructor(lc, dbPath, opts = {}) {
|
|
40
|
+
this.#lc = lc.withContext("component", this.id);
|
|
41
|
+
this.#db = new Database(this.#lc, dbPath, { fileMustExist: true }, SLOW_CHECKPOINT_MS);
|
|
42
|
+
[{page_size: this.#pageSize}] = this.#db.pragma("page_size");
|
|
43
|
+
this.#intervalMs = opts.intervalMs ?? CHECKPOINT_INTERVAL_MS;
|
|
44
|
+
this.#walWarningBytes = opts.walWarningBytes ?? WAL_WARNING_BYTES;
|
|
45
|
+
}
|
|
46
|
+
async run() {
|
|
47
|
+
try {
|
|
48
|
+
while (this.#state.shouldRun()) {
|
|
49
|
+
this.#checkpoint();
|
|
50
|
+
await this.#state.sleep(this.#intervalMs);
|
|
51
|
+
}
|
|
52
|
+
} finally {
|
|
53
|
+
this.#db.close();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
#checkpoint() {
|
|
57
|
+
let result;
|
|
58
|
+
try {
|
|
59
|
+
[result] = this.#db.pragma("wal_checkpoint(PASSIVE)");
|
|
60
|
+
} catch (e) {
|
|
61
|
+
const suppressed = this.#logThrottle.admit("error");
|
|
62
|
+
if (suppressed !== void 0) this.#lc.error?.(`error checkpointing the replica` + (suppressed ? ` (${suppressed} more since last logged)` : ""), e);
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
const { busy, log } = result;
|
|
66
|
+
const walBytes = log * this.#pageSize;
|
|
67
|
+
if (walBytes >= this.#walWarningBytes) {
|
|
68
|
+
const reason = busy ? "a reader (e.g. a long-running snapshot) is still reading the older wal file" : "checkpoints are not keeping up with writes";
|
|
69
|
+
const suppressed = this.#logThrottle.admit(busy ? "busy" : "behind");
|
|
70
|
+
if (suppressed !== void 0) this.#lc.warn?.(`replica wal is ${(walBytes / MB).toFixed(1)} MB: ${reason}`, {
|
|
71
|
+
walBytes,
|
|
72
|
+
busy: !!busy,
|
|
73
|
+
suppressed
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
stop() {
|
|
78
|
+
this.#state.stop(this.#lc);
|
|
79
|
+
return Promise.resolve();
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
//#endregion
|
|
83
|
+
export { ServingCheckpointer };
|
|
84
|
+
|
|
85
|
+
//# sourceMappingURL=serving-checkpointer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serving-checkpointer.js","names":[],"sources":["../../../../../../zero-cache/src/services/replicator/serving-checkpointer.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {LogThrottle} from '../../observability/log-throttle.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {Service} from '../service.ts';\n\nconst CHECKPOINT_INTERVAL_MS = 100;\n\n// A checkpoint copies at most one wal file (~4 MB with the default 1000 page\n// limit and 4 KB pages), so only a disk that is struggling makes one slow.\nconst SLOW_CHECKPOINT_MS = 1000;\n\n// The wal files normally hold about two files' worth of frames (~8 MB). A wal\n// several times that is not being checkpointed as fast as it is written.\nconst WAL_WARNING_BYTES = 64 * 1024 * 1024;\n\nconst LOG_WINDOW_MS = 60 * 1000;\n\nconst MB = 1024 * 1024;\n\ntype Options = {\n intervalMs?: number | undefined;\n walWarningBytes?: number | undefined;\n};\n\n/**\n * Checkpoints the serving (wal2) replica from its own connection, so that the\n * replicator's writer, which runs with `wal_autocheckpoint = 0`, never waits\n * on a checkpoint's disk syncs.\n *\n * In wal2 mode the writer switches to the other wal file once the current\n * one reaches its size limit, provided the other one has been checkpointed.\n * Until then, it keeps appending to the current file rather than waiting. A\n * `PASSIVE` checkpoint copies the whole non-current file, or does nothing\n * (without any I/O) if that file is already checkpointed, empty, or still\n * being read (e.g. by an old snapshot). Polling it on a short interval thus\n * keeps the wal files near their size limit without any coordination with\n * the writer.\n *\n * A failed checkpoint is logged and retried rather than stopping the\n * replicator. This matches the writer's automatic checkpoints that this\n * replaces, whose errors SQLite ignores: a persistent failure (e.g. a full or\n * failing disk) surfaces as the writer's own failure once the wal can no\n * longer grow.\n */\nexport class ServingCheckpointer implements Service {\n readonly id = 'serving-checkpointer';\n readonly #lc: LogContext;\n readonly #db: Database;\n readonly #pageSize: number;\n readonly #intervalMs: number;\n readonly #walWarningBytes: number;\n readonly #state = new RunningState(this.id);\n readonly #logThrottle = new LogThrottle({windowMs: LOG_WINDOW_MS});\n\n constructor(lc: LogContext, dbPath: string, opts: Options = {}) {\n this.#lc = lc.withContext('component', this.id);\n this.#db = new Database(\n this.#lc,\n dbPath,\n {fileMustExist: true},\n SLOW_CHECKPOINT_MS,\n );\n [{page_size: this.#pageSize}] = this.#db.pragma<{page_size: number}>(\n 'page_size',\n );\n this.#intervalMs = opts.intervalMs ?? CHECKPOINT_INTERVAL_MS;\n this.#walWarningBytes = opts.walWarningBytes ?? WAL_WARNING_BYTES;\n }\n\n async run(): Promise<void> {\n try {\n while (this.#state.shouldRun()) {\n this.#checkpoint();\n await this.#state.sleep(this.#intervalMs);\n }\n } finally {\n this.#db.close();\n }\n }\n\n #checkpoint() {\n let result;\n try {\n [result] = this.#db.pragma<{busy: number; log: number}>(\n 'wal_checkpoint(PASSIVE)',\n );\n } catch (e) {\n const suppressed = this.#logThrottle.admit('error');\n if (suppressed !== undefined) {\n this.#lc.error?.(\n `error checkpointing the replica` +\n (suppressed ? ` (${suppressed} more since last logged)` : ''),\n e,\n );\n }\n return;\n }\n\n // In wal2 mode `log` counts the frames in both wal files, and `busy` is\n // set if a reader prevented the checkpoint.\n const {busy, log} = result;\n const walBytes = log * this.#pageSize;\n if (walBytes >= this.#walWarningBytes) {\n const reason = busy\n ? 'a reader (e.g. a long-running snapshot) is still reading the older wal file'\n : 'checkpoints are not keeping up with writes';\n const suppressed = this.#logThrottle.admit(busy ? 'busy' : 'behind');\n if (suppressed !== undefined) {\n this.#lc.warn?.(\n `replica wal is ${(walBytes / MB).toFixed(1)} MB: ${reason}`,\n {walBytes, busy: !!busy, suppressed},\n );\n }\n }\n }\n\n stop(): Promise<void> {\n this.#state.stop(this.#lc);\n return Promise.resolve();\n }\n}\n"],"mappings":";;;;AAMA,IAAM,yBAAyB;AAI/B,IAAM,qBAAqB;AAI3B,IAAM,oBAAoB;AAE1B,IAAM,gBAAgB;AAEtB,IAAM,KAAK;;;;;;;;;;;;;;;;;;;;;AA2BX,IAAa,sBAAb,MAAoD;CAClD,KAAc;CACd;CACA;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,KAAK,EAAE;CAC1C,eAAwB,IAAI,YAAY,EAAC,UAAU,cAAa,CAAC;CAEjE,YAAY,IAAgB,QAAgB,OAAgB,CAAC,GAAG;EAC9D,KAAK,MAAM,GAAG,YAAY,aAAa,KAAK,EAAE;EAC9C,KAAK,MAAM,IAAI,SACb,KAAK,KACL,QACA,EAAC,eAAe,KAAI,GACpB,kBACF;EACA,CAAC,CAAC,WAAW,KAAK,cAAc,KAAK,IAAI,OACvC,WACF;EACA,KAAK,cAAc,KAAK,cAAc;EACtC,KAAK,mBAAmB,KAAK,mBAAmB;CAClD;CAEA,MAAM,MAAqB;EACzB,IAAI;GACF,OAAO,KAAK,OAAO,UAAU,GAAG;IAC9B,KAAK,YAAY;IACjB,MAAM,KAAK,OAAO,MAAM,KAAK,WAAW;GAC1C;EACF,UAAU;GACR,KAAK,IAAI,MAAM;EACjB;CACF;CAEA,cAAc;EACZ,IAAI;EACJ,IAAI;GACF,CAAC,UAAU,KAAK,IAAI,OAClB,yBACF;EACF,SAAS,GAAG;GACV,MAAM,aAAa,KAAK,aAAa,MAAM,OAAO;GAClD,IAAI,eAAe,KAAA,GACjB,KAAK,IAAI,QACP,qCACG,aAAa,KAAK,WAAW,4BAA4B,KAC5D,CACF;GAEF;EACF;EAIA,MAAM,EAAC,MAAM,QAAO;EACpB,MAAM,WAAW,MAAM,KAAK;EAC5B,IAAI,YAAY,KAAK,kBAAkB;GACrC,MAAM,SAAS,OACX,gFACA;GACJ,MAAM,aAAa,KAAK,aAAa,MAAM,OAAO,SAAS,QAAQ;GACnE,IAAI,eAAe,KAAA,GACjB,KAAK,IAAI,OACP,mBAAmB,WAAW,GAAA,CAAI,QAAQ,CAAC,EAAE,OAAO,UACpD;IAAC;IAAU,MAAM,CAAC,CAAC;IAAM;GAAU,CACrC;EAEJ;CACF;CAEA,OAAsB;EACpB,KAAK,OAAO,KAAK,KAAK,GAAG;EACzB,OAAO,QAAQ,QAAQ;CACzB;AACF"}
|
|
@@ -19,10 +19,10 @@ import { computeZqlSpecs, mustGetTableSpec } from "../../db/lite-tables.js";
|
|
|
19
19
|
import { rowKeyString } from "../../types/row-key.js";
|
|
20
20
|
import { getSubscriptionState } from "../replicator/schema/replication-state.js";
|
|
21
21
|
import { versionToLexi } from "../../types/lexi-version.js";
|
|
22
|
+
import { LogThrottle } from "../../observability/log-throttle.js";
|
|
22
23
|
import { reloadPermissionsIfChanged } from "../../auth/load-permissions.js";
|
|
23
24
|
import { resolveSimpleScalarSubqueries } from "../../../../zqlite/src/resolve-scalar-subqueries.js";
|
|
24
25
|
import { createSQLiteCostModel } from "../../../../zqlite/src/sqlite-cost-model.js";
|
|
25
|
-
import { LogThrottle } from "../../observability/log-throttle.js";
|
|
26
26
|
import { checkClientSchema } from "./client-schema.js";
|
|
27
27
|
import { planWarningMessage } from "./plan-warnings.js";
|
|
28
28
|
import { PushCutoff } from "./push-cutoff.js";
|
|
@@ -21,8 +21,8 @@ import { Subscription } from "../../types/subscription.js";
|
|
|
21
21
|
import { CustomKeyMap } from "../../../../shared/src/custom-key-map.js";
|
|
22
22
|
import { ttlClockAsNumber, ttlClockFromNumber } from "./ttl-clock.js";
|
|
23
23
|
import { EMPTY_CVR_VERSION, cmpVersions, versionFromString, versionString, versionToCookie } from "./schema/types.js";
|
|
24
|
-
import { isAuthErrorBody } from "../../auth/auth.js";
|
|
25
24
|
import { LogThrottle } from "../../observability/log-throttle.js";
|
|
25
|
+
import { isAuthErrorBody } from "../../auth/auth.js";
|
|
26
26
|
import { planWarningMessage } from "./plan-warnings.js";
|
|
27
27
|
import { shouldCutOffPush } from "./push-cutoff.js";
|
|
28
28
|
import { queryShape } from "./query-shape.js";
|
|
@@ -90,7 +90,7 @@ async function prepare(lc, { file, vacuumIntervalHours }, walMode, mode) {
|
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
92
|
await setJournalMode(lc, replica, walMode);
|
|
93
|
-
const pragmas = getPragmaConfig(
|
|
93
|
+
const pragmas = getPragmaConfig();
|
|
94
94
|
applyPragmas(replica, pragmas);
|
|
95
95
|
replica.pragma("optimize = 0x10002");
|
|
96
96
|
lc.info?.(`optimized ${file}`);
|
|
@@ -119,15 +119,20 @@ async function setJournalMode(lc, replica, mode) {
|
|
|
119
119
|
throw err;
|
|
120
120
|
}
|
|
121
121
|
/**
|
|
122
|
-
* Returns the PragmaConfig for
|
|
122
|
+
* Returns the PragmaConfig for the replica.
|
|
123
123
|
* This is used by both the main thread (setupReplica) and
|
|
124
124
|
* the write worker thread to apply the same pragma settings.
|
|
125
|
+
*
|
|
126
|
+
* The writer never checkpoints, since an automatic checkpoint runs (and
|
|
127
|
+
* syncs the disk) within the COMMIT, stalling replication on a slow disk.
|
|
128
|
+
* Instead, litestream checkpoints the backup replica, and the
|
|
129
|
+
* `ServingCheckpointer` checkpoints the serving replica.
|
|
125
130
|
*/
|
|
126
|
-
function getPragmaConfig(
|
|
131
|
+
function getPragmaConfig() {
|
|
127
132
|
return {
|
|
128
133
|
busyTimeout: 3e4,
|
|
129
134
|
analysisLimit: 1e3,
|
|
130
|
-
walAutocheckpoint:
|
|
135
|
+
walAutocheckpoint: 0
|
|
131
136
|
};
|
|
132
137
|
}
|
|
133
138
|
function setupReplica(lc, mode, replicaOptions) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.js","names":["v.literalUnion"],"sources":["../../../../../zero-cache/src/workers/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {ReplicaOptions} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {deleteChangeLogDB} from '../services/replicator/change-log-db.ts';\nimport {Notifier} from '../services/replicator/notifier.ts';\nimport type {\n ReplicaState,\n ReplicaStateNotifier,\n Replicator,\n} from '../services/replicator/replicator.ts';\nimport {\n getAscendingEvents,\n recordEvent,\n} from '../services/replicator/schema/replication-state.ts';\nimport {\n applyPragmas,\n type PragmaConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport type {Worker} from '../types/processes.ts';\n\nexport const replicaFileModeSchema = v.literalUnion(\n 'serving',\n 'serving-copy',\n 'backup',\n);\n\nexport type ReplicaFileMode = v.Infer<typeof replicaFileModeSchema>;\n\nexport type WalMode = 'wal' | 'wal2';\n\nexport function replicaFileName(replicaFile: string, mode: ReplicaFileMode) {\n return mode === 'serving-copy' ? `${replicaFile}-serving-copy` : replicaFile;\n}\n\n/**\n * Whether a replicator worker deletes the change-log file beside its replica at\n * startup.\n *\n * The change log belongs to the change-streamer, so this only ever cleans up a\n * file that nothing is writing. The predicate is deliberately the config flag\n * and nothing else: keying it on anything derived per replica -- a file mode, or\n * the retired `logsChangeStream` -- would make every replicator start unlink the\n * change-streamer's live log, because no replicator writes it any more. Paths\n * coincide whenever `fileMode !== 'serving-copy'`, so that hazard is real in the\n * shipped no-`backupURL` configuration.\n *\n * Deleting is safe when nothing writes the log: it is excluded from the\n * litestream backup, only the writer purges it, and re-enabling the writer\n * reseeds at the resume watermark.\n */\nexport function replicatorDeletesStaleChangeLog(\n sqliteChangeLogMode: string,\n): boolean {\n return sqliteChangeLogMode === 'off';\n}\n\n/**\n * The `mode=off` cleanup itself: deletes the change-log file beside `dbPath`\n * when {@link replicatorDeletesStaleChangeLog} says nothing writes it, and\n * reports whether it did. The decision and the delete live in one function so\n * that the guard test covers the call the replicator actually makes, not just\n * the predicate.\n */\nexport function deleteStaleChangeLog(\n sqliteChangeLogMode: string,\n dbPath: string,\n): boolean {\n if (!replicatorDeletesStaleChangeLog(sqliteChangeLogMode)) {\n return false;\n }\n deleteChangeLogDB(dbPath);\n return true;\n}\n\nconst MILLIS_PER_HOUR = 1000 * 60 * 60;\nconst MB = 1024 * 1024;\n\nasync function prepare(\n lc: LogContext,\n {file, vacuumIntervalHours}: ReplicaOptions,\n walMode: WalMode,\n mode: ReplicaFileMode,\n): Promise<{file: string; walMode: WalMode; pageSize: number}> {\n // Perform any upgrades to the replica in case the backup is an\n // earlier version.\n await upgradeReplica(lc, `${mode}-replica`, file);\n\n const replica = new Database(lc, file);\n let pageSize: number;\n try {\n // Switching between WAL modes (e.g. from a restored 'wal' replica to\n // 'wal2') goes through 'delete', which folds the WAL(2) files into the\n // main db. Leaving WAL mode requires the WAL not to be persisted (see\n // Database.persistWal()). A replica already in the target mode keeps its\n // WAL, from which a litestream backup may continue.\n const [{journal_mode: journalMode}] = replica.pragma<{\n journal_mode: string;\n }>('journal_mode');\n if (journalMode !== walMode) {\n replica.persistWal(false);\n try {\n await setJournalMode(lc, replica, 'delete');\n } finally {\n replica.persistWal(true);\n }\n }\n\n [{page_size: pageSize}] = replica.pragma<{page_size: number}>('page_size');\n const [{page_count: pageCount}] = replica.pragma<{page_count: number}>(\n 'page_count',\n );\n const [{freelist_count: freelistCount}] = replica.pragma<{\n freelist_count: number;\n }>('freelist_count');\n\n const dbSize = ((pageCount * pageSize) / MB).toFixed(2);\n const freelistSize = ((freelistCount * pageSize) / MB).toFixed(2);\n\n // TODO: Consider adding a freelist size or ratio based vacuum trigger.\n lc.info?.(`Size of db ${file}: ${dbSize} MB (${freelistSize} MB freeable)`);\n\n // Check for the VACUUM threshold.\n const events = getAscendingEvents(replica);\n lc.debug?.(`Runtime events for db ${file}`, {events});\n if (vacuumIntervalHours !== undefined) {\n const millisSinceLastEvent =\n Date.now() - (events.at(-1)?.timestamp.getTime() ?? 0);\n if (millisSinceLastEvent / MILLIS_PER_HOUR > vacuumIntervalHours) {\n lc.info?.(`Performing maintenance cleanup on ${file}`);\n const t0 = performance.now();\n replica.unsafeMode(true);\n // Leaving WAL mode requires the WAL not to be persisted (see\n // Database.persistWal()). The VACUUM rewrites the database outside\n // of the WAL, so a litestream backup cannot continue from it anyway.\n replica.persistWal(false);\n try {\n replica.pragma('journal_mode = OFF');\n replica.exec('VACUUM');\n recordEvent(replica, 'vacuum');\n const t1 = performance.now();\n lc.info?.(`VACUUM completed (${t1 - t0} ms)`);\n } catch (e) {\n // Another connection holds the replica open. The VACUUM is retried\n // at the next startup.\n if (!(e instanceof SqliteError && e.code === 'SQLITE_BUSY')) {\n throw e;\n }\n lc.warn?.(`Skipping VACUUM of ${file}: the database is locked`, e);\n } finally {\n replica.persistWal(true);\n replica.unsafeMode(false);\n }\n }\n }\n\n await setJournalMode(lc, replica, walMode);\n\n const pragmas = getPragmaConfig(mode);\n applyPragmas(replica, pragmas);\n\n replica.pragma('optimize = 0x10002');\n lc.info?.(`optimized ${file}`);\n } finally {\n replica.close();\n }\n return {file, walMode, pageSize};\n}\n\n// Setting the journal_mode requires an exclusive lock on the replica.\n// Add resilience against random replica reads (for stats, etc.) by\n// retrying if the database is locked. Note that the busy_timeout doesn't\n// work here.\nasync function setJournalMode(\n lc: LogContext,\n replica: Database,\n mode: 'delete' | 'wal' | 'wal2',\n) {\n lc.info?.(`setting ${replica.name} to ${mode} mode`);\n let err: unknown;\n for (let i = 0; i < 5; i++) {\n try {\n replica.pragma(`journal_mode = ${mode}`);\n return;\n } catch (e) {\n lc.warn?.(`error setting journal_mode to ${mode} (attempt ${i + 1})`, e);\n err = e;\n }\n await sleep(500);\n }\n throw err;\n}\n\n/**\n * Returns the PragmaConfig for a given replica file mode.\n * This is used by both the main thread (setupReplica) and\n * the write worker thread to apply the same pragma settings.\n */\nexport function getPragmaConfig(mode: ReplicaFileMode): PragmaConfig {\n return {\n busyTimeout: 30000,\n analysisLimit: 1000,\n walAutocheckpoint: mode === 'backup' ? 0 : undefined,\n };\n}\n\nexport function setupReplica(\n lc: LogContext,\n mode: ReplicaFileMode,\n replicaOptions: ReplicaOptions,\n) {\n lc.info?.(`setting up ${mode} replica`);\n\n switch (mode) {\n case 'backup':\n return prepare(lc, replicaOptions, 'wal', mode);\n\n case 'serving-copy': {\n // In 'serving-copy' mode, the original file is being used for 'backup'\n // mode, so we make a copy for servicing sync requests.\n const {file} = replicaOptions;\n const copyLocation = replicaFileName(file, mode);\n deleteLiteDB(copyLocation);\n\n const start = Date.now();\n lc.info?.(`copying ${file} to ${copyLocation}`);\n const replica = new Database(lc, file);\n replica.prepare(`VACUUM INTO ?`).run(copyLocation);\n replica.close();\n lc.info?.(`finished copy (${Date.now() - start} ms)`);\n\n return prepare(lc, {...replicaOptions, file: copyLocation}, 'wal2', mode);\n }\n\n case 'serving':\n return prepare(lc, replicaOptions, 'wal2', mode);\n\n default:\n throw new Error(`Invalid ReplicaMode ${mode}`);\n }\n}\n\nexport function setUpMessageHandlers(\n lc: LogContext,\n replicator: Replicator,\n parent: Worker,\n) {\n handleSubscriptionsFrom(lc, parent, replicator);\n}\n\ntype Notification = ['notify', ReplicaState];\n\nexport function handleSubscriptionsFrom(\n lc: LogContext,\n subscriber: Worker,\n notifier: ReplicaStateNotifier,\n) {\n subscriber.onMessageType('subscribe', async () => {\n const subscription = notifier.subscribe();\n\n subscriber.on('close', () => {\n lc.debug?.(`closing replication subscription from ${subscriber.pid}`);\n subscription.cancel();\n });\n\n for await (const msg of subscription) {\n try {\n subscriber.send<Notification>(['notify', msg]);\n } catch (e) {\n const log =\n e instanceof Error &&\n 'code' in e &&\n // This can happen in a race condition if the subscribing process\n // is closed before the 'close' message is processed.\n e.code === 'ERR_IPC_CHANNEL_CLOSED'\n ? 'warn'\n : 'error';\n\n lc[log]?.(\n `error sending replicator notification to ${subscriber.pid}: ${String(e)}`,\n e,\n );\n }\n }\n });\n}\n\n/**\n * Creates a Notifier to relay notifications the notifier of another Worker.\n * This does not send the initial subscription message. Use {@link subscribeTo}\n * to initiate the subscription.\n */\nexport function createNotifierFrom(\n _lc: LogContext,\n source: Worker,\n onNotify?: (state: ReplicaState) => void,\n): Notifier {\n const notifier = new Notifier();\n source.onMessageType<Notification>('notify', msg => {\n onNotify?.(msg);\n void notifier.notifySubscribers(msg);\n });\n return notifier;\n}\n\nexport function subscribeTo(_lc: LogContext, source: Worker) {\n source.send(['subscribe', {}]);\n}\n"],"mappings":";;;;;;;;;;;AAyBA,IAAa,wBAAwBA,aACnC,WACA,gBACA,QACF;AAMA,SAAgB,gBAAgB,aAAqB,MAAuB;CAC1E,OAAO,SAAS,iBAAiB,GAAG,YAAY,iBAAiB;AACnE;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gCACd,qBACS;CACT,OAAO,wBAAwB;AACjC;;;;;;;;AASA,SAAgB,qBACd,qBACA,QACS;CACT,IAAI,CAAC,gCAAgC,mBAAmB,GACtD,OAAO;CAET,kBAAkB,MAAM;CACxB,OAAO;AACT;AAEA,IAAM,kBAAkB;AACxB,IAAM,KAAK;AAEX,eAAe,QACb,IACA,EAAC,MAAM,uBACP,SACA,MAC6D;CAG7D,MAAM,eAAe,IAAI,GAAG,KAAK,WAAW,IAAI;CAEhD,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;CACrC,IAAI;CACJ,IAAI;EAMF,MAAM,CAAC,EAAC,cAAc,iBAAgB,QAAQ,OAE3C,cAAc;EACjB,IAAI,gBAAgB,SAAS;GAC3B,QAAQ,WAAW,KAAK;GACxB,IAAI;IACF,MAAM,eAAe,IAAI,SAAS,QAAQ;GAC5C,UAAU;IACR,QAAQ,WAAW,IAAI;GACzB;EACF;EAEA,CAAC,CAAC,WAAW,aAAa,QAAQ,OAA4B,WAAW;EACzE,MAAM,CAAC,EAAC,YAAY,eAAc,QAAQ,OACxC,YACF;EACA,MAAM,CAAC,EAAC,gBAAgB,mBAAkB,QAAQ,OAE/C,gBAAgB;EAEnB,MAAM,UAAW,YAAY,WAAY,GAAA,CAAI,QAAQ,CAAC;EACtD,MAAM,gBAAiB,gBAAgB,WAAY,GAAA,CAAI,QAAQ,CAAC;EAGhE,GAAG,OAAO,cAAc,KAAK,IAAI,OAAO,OAAO,aAAa,cAAc;EAG1E,MAAM,SAAS,mBAAmB,OAAO;EACzC,GAAG,QAAQ,yBAAyB,QAAQ,EAAC,OAAM,CAAC;EACpD,IAAI,wBAAwB,KAAA,GAExB;QAAA,KAAK,IAAI,KAAK,OAAO,GAAG,EAAE,CAAC,EAAE,UAAU,QAAQ,KAAK,MAC3B,kBAAkB,qBAAqB;IAChE,GAAG,OAAO,qCAAqC,MAAM;IACrD,MAAM,KAAK,YAAY,IAAI;IAC3B,QAAQ,WAAW,IAAI;IAIvB,QAAQ,WAAW,KAAK;IACxB,IAAI;KACF,QAAQ,OAAO,oBAAoB;KACnC,QAAQ,KAAK,QAAQ;KACrB,YAAY,SAAS,QAAQ;KAC7B,MAAM,KAAK,YAAY,IAAI;KAC3B,GAAG,OAAO,qBAAqB,KAAK,GAAG,KAAK;IAC9C,SAAS,GAAG;KAGV,IAAI,EAAE,aAAa,eAAe,EAAE,SAAS,gBAC3C,MAAM;KAER,GAAG,OAAO,sBAAsB,KAAK,2BAA2B,CAAC;IACnE,UAAU;KACR,QAAQ,WAAW,IAAI;KACvB,QAAQ,WAAW,KAAK;IAC1B;GACF;;EAGF,MAAM,eAAe,IAAI,SAAS,OAAO;EAEzC,MAAM,UAAU,gBAAgB,IAAI;EACpC,aAAa,SAAS,OAAO;EAE7B,QAAQ,OAAO,oBAAoB;EACnC,GAAG,OAAO,aAAa,MAAM;CAC/B,UAAU;EACR,QAAQ,MAAM;CAChB;CACA,OAAO;EAAC;EAAM;EAAS;CAAQ;AACjC;AAMA,eAAe,eACb,IACA,SACA,MACA;CACA,GAAG,OAAO,WAAW,QAAQ,KAAK,MAAM,KAAK,MAAM;CACnD,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI;GACF,QAAQ,OAAO,kBAAkB,MAAM;GACvC;EACF,SAAS,GAAG;GACV,GAAG,OAAO,iCAAiC,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE,MAAM;EACR;EACA,MAAM,MAAM,GAAG;CACjB;CACA,MAAM;AACR;;;;;;AAOA,SAAgB,gBAAgB,MAAqC;CACnE,OAAO;EACL,aAAa;EACb,eAAe;EACf,mBAAmB,SAAS,WAAW,IAAI,KAAA;CAC7C;AACF;AAEA,SAAgB,aACd,IACA,MACA,gBACA;CACA,GAAG,OAAO,cAAc,KAAK,SAAS;CAEtC,QAAQ,MAAR;EACE,KAAK,UACH,OAAO,QAAQ,IAAI,gBAAgB,OAAO,IAAI;EAEhD,KAAK,gBAAgB;GAGnB,MAAM,EAAC,SAAQ;GACf,MAAM,eAAe,gBAAgB,MAAM,IAAI;GAC/C,aAAa,YAAY;GAEzB,MAAM,QAAQ,KAAK,IAAI;GACvB,GAAG,OAAO,WAAW,KAAK,MAAM,cAAc;GAC9C,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;GACrC,QAAQ,QAAQ,eAAe,CAAC,CAAC,IAAI,YAAY;GACjD,QAAQ,MAAM;GACd,GAAG,OAAO,kBAAkB,KAAK,IAAI,IAAI,MAAM,KAAK;GAEpD,OAAO,QAAQ,IAAI;IAAC,GAAG;IAAgB,MAAM;GAAY,GAAG,QAAQ,IAAI;EAC1E;EAEA,KAAK,WACH,OAAO,QAAQ,IAAI,gBAAgB,QAAQ,IAAI;EAEjD,SACE,MAAM,IAAI,MAAM,uBAAuB,MAAM;CACjD;AACF;AAEA,SAAgB,qBACd,IACA,YACA,QACA;CACA,wBAAwB,IAAI,QAAQ,UAAU;AAChD;AAIA,SAAgB,wBACd,IACA,YACA,UACA;CACA,WAAW,cAAc,aAAa,YAAY;EAChD,MAAM,eAAe,SAAS,UAAU;EAExC,WAAW,GAAG,eAAe;GAC3B,GAAG,QAAQ,yCAAyC,WAAW,KAAK;GACpE,aAAa,OAAO;EACtB,CAAC;EAED,WAAW,MAAM,OAAO,cACtB,IAAI;GACF,WAAW,KAAmB,CAAC,UAAU,GAAG,CAAC;EAC/C,SAAS,GAAG;GAUV,GARE,aAAa,SACb,UAAU,KAGV,EAAE,SAAS,2BACP,SACA,QAEC,GACL,4CAA4C,WAAW,IAAI,IAAI,OAAO,CAAC,KACvE,CACF;EACF;CAEJ,CAAC;AACH;;;;;;AAOA,SAAgB,mBACd,KACA,QACA,UACU;CACV,MAAM,WAAW,IAAI,SAAS;CAC9B,OAAO,cAA4B,WAAU,QAAO;EAClD,WAAW,GAAG;EACd,SAAc,kBAAkB,GAAG;CACrC,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,YAAY,KAAiB,QAAgB;CAC3D,OAAO,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;AAC/B"}
|
|
1
|
+
{"version":3,"file":"replicator.js","names":["v.literalUnion"],"sources":["../../../../../zero-cache/src/workers/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {SqliteError} from '@rocicorp/zero-sqlite3';\nimport {sleep} from '../../../shared/src/sleep.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\nimport type {ReplicaOptions} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {upgradeReplica} from '../services/change-source/common/replica-schema.ts';\nimport {deleteChangeLogDB} from '../services/replicator/change-log-db.ts';\nimport {Notifier} from '../services/replicator/notifier.ts';\nimport type {\n ReplicaState,\n ReplicaStateNotifier,\n Replicator,\n} from '../services/replicator/replicator.ts';\nimport {\n getAscendingEvents,\n recordEvent,\n} from '../services/replicator/schema/replication-state.ts';\nimport {\n applyPragmas,\n type PragmaConfig,\n} from '../services/replicator/write-worker-client.ts';\nimport type {Worker} from '../types/processes.ts';\n\nexport const replicaFileModeSchema = v.literalUnion(\n 'serving',\n 'serving-copy',\n 'backup',\n);\n\nexport type ReplicaFileMode = v.Infer<typeof replicaFileModeSchema>;\n\nexport type WalMode = 'wal' | 'wal2';\n\nexport function replicaFileName(replicaFile: string, mode: ReplicaFileMode) {\n return mode === 'serving-copy' ? `${replicaFile}-serving-copy` : replicaFile;\n}\n\n/**\n * Whether a replicator worker deletes the change-log file beside its replica at\n * startup.\n *\n * The change log belongs to the change-streamer, so this only ever cleans up a\n * file that nothing is writing. The predicate is deliberately the config flag\n * and nothing else: keying it on anything derived per replica -- a file mode, or\n * the retired `logsChangeStream` -- would make every replicator start unlink the\n * change-streamer's live log, because no replicator writes it any more. Paths\n * coincide whenever `fileMode !== 'serving-copy'`, so that hazard is real in the\n * shipped no-`backupURL` configuration.\n *\n * Deleting is safe when nothing writes the log: it is excluded from the\n * litestream backup, only the writer purges it, and re-enabling the writer\n * reseeds at the resume watermark.\n */\nexport function replicatorDeletesStaleChangeLog(\n sqliteChangeLogMode: string,\n): boolean {\n return sqliteChangeLogMode === 'off';\n}\n\n/**\n * The `mode=off` cleanup itself: deletes the change-log file beside `dbPath`\n * when {@link replicatorDeletesStaleChangeLog} says nothing writes it, and\n * reports whether it did. The decision and the delete live in one function so\n * that the guard test covers the call the replicator actually makes, not just\n * the predicate.\n */\nexport function deleteStaleChangeLog(\n sqliteChangeLogMode: string,\n dbPath: string,\n): boolean {\n if (!replicatorDeletesStaleChangeLog(sqliteChangeLogMode)) {\n return false;\n }\n deleteChangeLogDB(dbPath);\n return true;\n}\n\nconst MILLIS_PER_HOUR = 1000 * 60 * 60;\nconst MB = 1024 * 1024;\n\nasync function prepare(\n lc: LogContext,\n {file, vacuumIntervalHours}: ReplicaOptions,\n walMode: WalMode,\n mode: ReplicaFileMode,\n): Promise<{file: string; walMode: WalMode; pageSize: number}> {\n // Perform any upgrades to the replica in case the backup is an\n // earlier version.\n await upgradeReplica(lc, `${mode}-replica`, file);\n\n const replica = new Database(lc, file);\n let pageSize: number;\n try {\n // Switching between WAL modes (e.g. from a restored 'wal' replica to\n // 'wal2') goes through 'delete', which folds the WAL(2) files into the\n // main db. Leaving WAL mode requires the WAL not to be persisted (see\n // Database.persistWal()). A replica already in the target mode keeps its\n // WAL, from which a litestream backup may continue.\n const [{journal_mode: journalMode}] = replica.pragma<{\n journal_mode: string;\n }>('journal_mode');\n if (journalMode !== walMode) {\n replica.persistWal(false);\n try {\n await setJournalMode(lc, replica, 'delete');\n } finally {\n replica.persistWal(true);\n }\n }\n\n [{page_size: pageSize}] = replica.pragma<{page_size: number}>('page_size');\n const [{page_count: pageCount}] = replica.pragma<{page_count: number}>(\n 'page_count',\n );\n const [{freelist_count: freelistCount}] = replica.pragma<{\n freelist_count: number;\n }>('freelist_count');\n\n const dbSize = ((pageCount * pageSize) / MB).toFixed(2);\n const freelistSize = ((freelistCount * pageSize) / MB).toFixed(2);\n\n // TODO: Consider adding a freelist size or ratio based vacuum trigger.\n lc.info?.(`Size of db ${file}: ${dbSize} MB (${freelistSize} MB freeable)`);\n\n // Check for the VACUUM threshold.\n const events = getAscendingEvents(replica);\n lc.debug?.(`Runtime events for db ${file}`, {events});\n if (vacuumIntervalHours !== undefined) {\n const millisSinceLastEvent =\n Date.now() - (events.at(-1)?.timestamp.getTime() ?? 0);\n if (millisSinceLastEvent / MILLIS_PER_HOUR > vacuumIntervalHours) {\n lc.info?.(`Performing maintenance cleanup on ${file}`);\n const t0 = performance.now();\n replica.unsafeMode(true);\n // Leaving WAL mode requires the WAL not to be persisted (see\n // Database.persistWal()). The VACUUM rewrites the database outside\n // of the WAL, so a litestream backup cannot continue from it anyway.\n replica.persistWal(false);\n try {\n replica.pragma('journal_mode = OFF');\n replica.exec('VACUUM');\n recordEvent(replica, 'vacuum');\n const t1 = performance.now();\n lc.info?.(`VACUUM completed (${t1 - t0} ms)`);\n } catch (e) {\n // Another connection holds the replica open. The VACUUM is retried\n // at the next startup.\n if (!(e instanceof SqliteError && e.code === 'SQLITE_BUSY')) {\n throw e;\n }\n lc.warn?.(`Skipping VACUUM of ${file}: the database is locked`, e);\n } finally {\n replica.persistWal(true);\n replica.unsafeMode(false);\n }\n }\n }\n\n await setJournalMode(lc, replica, walMode);\n\n const pragmas = getPragmaConfig();\n applyPragmas(replica, pragmas);\n\n replica.pragma('optimize = 0x10002');\n lc.info?.(`optimized ${file}`);\n } finally {\n replica.close();\n }\n return {file, walMode, pageSize};\n}\n\n// Setting the journal_mode requires an exclusive lock on the replica.\n// Add resilience against random replica reads (for stats, etc.) by\n// retrying if the database is locked. Note that the busy_timeout doesn't\n// work here.\nasync function setJournalMode(\n lc: LogContext,\n replica: Database,\n mode: 'delete' | 'wal' | 'wal2',\n) {\n lc.info?.(`setting ${replica.name} to ${mode} mode`);\n let err: unknown;\n for (let i = 0; i < 5; i++) {\n try {\n replica.pragma(`journal_mode = ${mode}`);\n return;\n } catch (e) {\n lc.warn?.(`error setting journal_mode to ${mode} (attempt ${i + 1})`, e);\n err = e;\n }\n await sleep(500);\n }\n throw err;\n}\n\n/**\n * Returns the PragmaConfig for the replica.\n * This is used by both the main thread (setupReplica) and\n * the write worker thread to apply the same pragma settings.\n *\n * The writer never checkpoints, since an automatic checkpoint runs (and\n * syncs the disk) within the COMMIT, stalling replication on a slow disk.\n * Instead, litestream checkpoints the backup replica, and the\n * `ServingCheckpointer` checkpoints the serving replica.\n */\nexport function getPragmaConfig(): PragmaConfig {\n return {\n busyTimeout: 30000,\n analysisLimit: 1000,\n walAutocheckpoint: 0,\n };\n}\n\nexport function setupReplica(\n lc: LogContext,\n mode: ReplicaFileMode,\n replicaOptions: ReplicaOptions,\n) {\n lc.info?.(`setting up ${mode} replica`);\n\n switch (mode) {\n case 'backup':\n return prepare(lc, replicaOptions, 'wal', mode);\n\n case 'serving-copy': {\n // In 'serving-copy' mode, the original file is being used for 'backup'\n // mode, so we make a copy for servicing sync requests.\n const {file} = replicaOptions;\n const copyLocation = replicaFileName(file, mode);\n deleteLiteDB(copyLocation);\n\n const start = Date.now();\n lc.info?.(`copying ${file} to ${copyLocation}`);\n const replica = new Database(lc, file);\n replica.prepare(`VACUUM INTO ?`).run(copyLocation);\n replica.close();\n lc.info?.(`finished copy (${Date.now() - start} ms)`);\n\n return prepare(lc, {...replicaOptions, file: copyLocation}, 'wal2', mode);\n }\n\n case 'serving':\n return prepare(lc, replicaOptions, 'wal2', mode);\n\n default:\n throw new Error(`Invalid ReplicaMode ${mode}`);\n }\n}\n\nexport function setUpMessageHandlers(\n lc: LogContext,\n replicator: Replicator,\n parent: Worker,\n) {\n handleSubscriptionsFrom(lc, parent, replicator);\n}\n\ntype Notification = ['notify', ReplicaState];\n\nexport function handleSubscriptionsFrom(\n lc: LogContext,\n subscriber: Worker,\n notifier: ReplicaStateNotifier,\n) {\n subscriber.onMessageType('subscribe', async () => {\n const subscription = notifier.subscribe();\n\n subscriber.on('close', () => {\n lc.debug?.(`closing replication subscription from ${subscriber.pid}`);\n subscription.cancel();\n });\n\n for await (const msg of subscription) {\n try {\n subscriber.send<Notification>(['notify', msg]);\n } catch (e) {\n const log =\n e instanceof Error &&\n 'code' in e &&\n // This can happen in a race condition if the subscribing process\n // is closed before the 'close' message is processed.\n e.code === 'ERR_IPC_CHANNEL_CLOSED'\n ? 'warn'\n : 'error';\n\n lc[log]?.(\n `error sending replicator notification to ${subscriber.pid}: ${String(e)}`,\n e,\n );\n }\n }\n });\n}\n\n/**\n * Creates a Notifier to relay notifications the notifier of another Worker.\n * This does not send the initial subscription message. Use {@link subscribeTo}\n * to initiate the subscription.\n */\nexport function createNotifierFrom(\n _lc: LogContext,\n source: Worker,\n onNotify?: (state: ReplicaState) => void,\n): Notifier {\n const notifier = new Notifier();\n source.onMessageType<Notification>('notify', msg => {\n onNotify?.(msg);\n void notifier.notifySubscribers(msg);\n });\n return notifier;\n}\n\nexport function subscribeTo(_lc: LogContext, source: Worker) {\n source.send(['subscribe', {}]);\n}\n"],"mappings":";;;;;;;;;;;AAyBA,IAAa,wBAAwBA,aACnC,WACA,gBACA,QACF;AAMA,SAAgB,gBAAgB,aAAqB,MAAuB;CAC1E,OAAO,SAAS,iBAAiB,GAAG,YAAY,iBAAiB;AACnE;;;;;;;;;;;;;;;;;AAkBA,SAAgB,gCACd,qBACS;CACT,OAAO,wBAAwB;AACjC;;;;;;;;AASA,SAAgB,qBACd,qBACA,QACS;CACT,IAAI,CAAC,gCAAgC,mBAAmB,GACtD,OAAO;CAET,kBAAkB,MAAM;CACxB,OAAO;AACT;AAEA,IAAM,kBAAkB;AACxB,IAAM,KAAK;AAEX,eAAe,QACb,IACA,EAAC,MAAM,uBACP,SACA,MAC6D;CAG7D,MAAM,eAAe,IAAI,GAAG,KAAK,WAAW,IAAI;CAEhD,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;CACrC,IAAI;CACJ,IAAI;EAMF,MAAM,CAAC,EAAC,cAAc,iBAAgB,QAAQ,OAE3C,cAAc;EACjB,IAAI,gBAAgB,SAAS;GAC3B,QAAQ,WAAW,KAAK;GACxB,IAAI;IACF,MAAM,eAAe,IAAI,SAAS,QAAQ;GAC5C,UAAU;IACR,QAAQ,WAAW,IAAI;GACzB;EACF;EAEA,CAAC,CAAC,WAAW,aAAa,QAAQ,OAA4B,WAAW;EACzE,MAAM,CAAC,EAAC,YAAY,eAAc,QAAQ,OACxC,YACF;EACA,MAAM,CAAC,EAAC,gBAAgB,mBAAkB,QAAQ,OAE/C,gBAAgB;EAEnB,MAAM,UAAW,YAAY,WAAY,GAAA,CAAI,QAAQ,CAAC;EACtD,MAAM,gBAAiB,gBAAgB,WAAY,GAAA,CAAI,QAAQ,CAAC;EAGhE,GAAG,OAAO,cAAc,KAAK,IAAI,OAAO,OAAO,aAAa,cAAc;EAG1E,MAAM,SAAS,mBAAmB,OAAO;EACzC,GAAG,QAAQ,yBAAyB,QAAQ,EAAC,OAAM,CAAC;EACpD,IAAI,wBAAwB,KAAA,GAExB;QAAA,KAAK,IAAI,KAAK,OAAO,GAAG,EAAE,CAAC,EAAE,UAAU,QAAQ,KAAK,MAC3B,kBAAkB,qBAAqB;IAChE,GAAG,OAAO,qCAAqC,MAAM;IACrD,MAAM,KAAK,YAAY,IAAI;IAC3B,QAAQ,WAAW,IAAI;IAIvB,QAAQ,WAAW,KAAK;IACxB,IAAI;KACF,QAAQ,OAAO,oBAAoB;KACnC,QAAQ,KAAK,QAAQ;KACrB,YAAY,SAAS,QAAQ;KAC7B,MAAM,KAAK,YAAY,IAAI;KAC3B,GAAG,OAAO,qBAAqB,KAAK,GAAG,KAAK;IAC9C,SAAS,GAAG;KAGV,IAAI,EAAE,aAAa,eAAe,EAAE,SAAS,gBAC3C,MAAM;KAER,GAAG,OAAO,sBAAsB,KAAK,2BAA2B,CAAC;IACnE,UAAU;KACR,QAAQ,WAAW,IAAI;KACvB,QAAQ,WAAW,KAAK;IAC1B;GACF;;EAGF,MAAM,eAAe,IAAI,SAAS,OAAO;EAEzC,MAAM,UAAU,gBAAgB;EAChC,aAAa,SAAS,OAAO;EAE7B,QAAQ,OAAO,oBAAoB;EACnC,GAAG,OAAO,aAAa,MAAM;CAC/B,UAAU;EACR,QAAQ,MAAM;CAChB;CACA,OAAO;EAAC;EAAM;EAAS;CAAQ;AACjC;AAMA,eAAe,eACb,IACA,SACA,MACA;CACA,GAAG,OAAO,WAAW,QAAQ,KAAK,MAAM,KAAK,MAAM;CACnD,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,KAAK;EAC1B,IAAI;GACF,QAAQ,OAAO,kBAAkB,MAAM;GACvC;EACF,SAAS,GAAG;GACV,GAAG,OAAO,iCAAiC,KAAK,YAAY,IAAI,EAAE,IAAI,CAAC;GACvE,MAAM;EACR;EACA,MAAM,MAAM,GAAG;CACjB;CACA,MAAM;AACR;;;;;;;;;;;AAYA,SAAgB,kBAAgC;CAC9C,OAAO;EACL,aAAa;EACb,eAAe;EACf,mBAAmB;CACrB;AACF;AAEA,SAAgB,aACd,IACA,MACA,gBACA;CACA,GAAG,OAAO,cAAc,KAAK,SAAS;CAEtC,QAAQ,MAAR;EACE,KAAK,UACH,OAAO,QAAQ,IAAI,gBAAgB,OAAO,IAAI;EAEhD,KAAK,gBAAgB;GAGnB,MAAM,EAAC,SAAQ;GACf,MAAM,eAAe,gBAAgB,MAAM,IAAI;GAC/C,aAAa,YAAY;GAEzB,MAAM,QAAQ,KAAK,IAAI;GACvB,GAAG,OAAO,WAAW,KAAK,MAAM,cAAc;GAC9C,MAAM,UAAU,IAAI,SAAS,IAAI,IAAI;GACrC,QAAQ,QAAQ,eAAe,CAAC,CAAC,IAAI,YAAY;GACjD,QAAQ,MAAM;GACd,GAAG,OAAO,kBAAkB,KAAK,IAAI,IAAI,MAAM,KAAK;GAEpD,OAAO,QAAQ,IAAI;IAAC,GAAG;IAAgB,MAAM;GAAY,GAAG,QAAQ,IAAI;EAC1E;EAEA,KAAK,WACH,OAAO,QAAQ,IAAI,gBAAgB,QAAQ,IAAI;EAEjD,SACE,MAAM,IAAI,MAAM,uBAAuB,MAAM;CACjD;AACF;AAEA,SAAgB,qBACd,IACA,YACA,QACA;CACA,wBAAwB,IAAI,QAAQ,UAAU;AAChD;AAIA,SAAgB,wBACd,IACA,YACA,UACA;CACA,WAAW,cAAc,aAAa,YAAY;EAChD,MAAM,eAAe,SAAS,UAAU;EAExC,WAAW,GAAG,eAAe;GAC3B,GAAG,QAAQ,yCAAyC,WAAW,KAAK;GACpE,aAAa,OAAO;EACtB,CAAC;EAED,WAAW,MAAM,OAAO,cACtB,IAAI;GACF,WAAW,KAAmB,CAAC,UAAU,GAAG,CAAC;EAC/C,SAAS,GAAG;GAUV,GARE,aAAa,SACb,UAAU,KAGV,EAAE,SAAS,2BACP,SACA,QAEC,GACL,4CAA4C,WAAW,IAAI,IAAI,OAAO,CAAC,KACvE,CACF;EACF;CAEJ,CAAC;AACH;;;;;;AAOA,SAAgB,mBACd,KACA,QACA,UACU;CACV,MAAM,WAAW,IAAI,SAAS;CAC9B,OAAO,cAA4B,WAAU,QAAO;EAClD,WAAW,GAAG;EACd,SAAc,kBAAkB,GAAG;CACrC,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,YAAY,KAAiB,QAAgB;CAC3D,OAAO,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;AAC/B"}
|