@rocicorp/zero 1.10.0-canary.8 → 1.10.0-canary.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration-lite.js +5 -3
- package/out/zero-cache/src/db/migration-lite.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +8 -2
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js +11 -5
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/package.json +1 -1
package/out/zero/package.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
var package_default = {
|
|
2
2
|
name: "@rocicorp/zero",
|
|
3
|
-
version: "1.10.0-canary.
|
|
3
|
+
version: "1.10.0-canary.9",
|
|
4
4
|
description: "Zero is a web framework for serverless web development.",
|
|
5
5
|
homepage: "https://zero.rocicorp.dev",
|
|
6
6
|
bugs: { "url": "https://bugs.rocicorp.dev" },
|
package/out/zero/package.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.10.0-canary.
|
|
1
|
+
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.10.0-canary.9\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"license\": \"Apache-2.0\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"bin\": {\n \"analyze-query\": \"./out/zero/src/analyze-query.js\",\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\",\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\"\n },\n \"files\": [\n \"out\",\n \"!*.tsbuildinfo\"\n ],\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./analyze\": {\n \"types\": \"./out/zero/src/analyze.d.ts\",\n \"default\": \"./out/zero/src/analyze.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/kysely\": {\n \"types\": \"./out/zero/src/adapters/kysely.d.ts\",\n \"default\": \"./out/zero/src/adapters/kysely.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"oxfmt .\",\n \"check-format\": \"oxfmt --check .\",\n \"lint\": \"oxlint --quiet --config ../../oxlint.config.ts src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\",\n \"fmt\": \"oxfmt .\",\n \"check-fmt\": \"oxfmt --check .\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.3.11\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.39.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.0.0\",\n \"@fastify/websocket\": \"^11.0.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/api-logs\": \"^0.218.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.76.0\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"^0.218.0\",\n \"@opentelemetry/resources\": \"^2.7.1\",\n \"@opentelemetry/sdk-metrics\": \"^2.7.1\",\n \"@opentelemetry/sdk-node\": \"^0.218.0\",\n \"@opentelemetry/sdk-trace-node\": \"^2.7.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.41.1\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^1.0.4\",\n \"@rocicorp/logger\": \"^6.1.0\",\n \"@rocicorp/resolver\": \"^1.0.2\",\n \"@rocicorp/zero-sqlite3\": \"^1.1.4\",\n \"@standard-schema/spec\": \"^1.0.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.5.12\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk-template\": \"^1.1.0\",\n \"chokidar\": \"^4.0.1\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.1\",\n \"command-line-usage\": \"^7.0.3\",\n \"compare-utf8\": \"^0.2.0\",\n \"defu\": \"^6.1.4\",\n \"eventemitter3\": \"^5.0.1\",\n \"fastify\": \"^5.0.0\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.9.3\",\n \"js-xxhash\": \"^4.0.0\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.0\",\n \"nanoid\": \"^5.1.2\",\n \"oxfmt\": \"^0.45.0\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"semver\": \"^7.5.4\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.18.1\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"@vitest/runner\": \"^4.1.6\",\n \"analyze-query\": \"workspace:*\",\n \"ast-to-zql\": \"workspace:*\",\n \"expo-sqlite\": \">=15\",\n \"oxc-parser\": \"^0.129.0\",\n \"replicache\": \"workspace:*\",\n \"shared\": \"workspace:*\",\n \"syncpack\": \"^14.3.0\",\n \"typedoc\": \"^0.28.17\",\n \"typedoc-plugin-markdown\": \"^4.10.0\",\n \"typescript\": \"~6.0.2\",\n \"vite\": \"^8.0.13\",\n \"vitest\": \"^4.1.6\",\n \"zero-cache\": \"workspace:*\",\n \"zero-client\": \"workspace:*\",\n \"zero-pg\": \"workspace:*\",\n \"zero-react\": \"workspace:*\",\n \"zero-server\": \"workspace:*\",\n \"zero-solid\": \"workspace:*\",\n \"zqlite\": \"workspace:*\"\n },\n \"engines\": {\n \"node\": \">=22\"\n },\n \"packageManager\": \"pnpm@11.11.0\"\n}\n"],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migration-lite.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/db/migration-lite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,KAAK,EAAC,QAAQ,IAAI,EAAE,EAAC,MAAM,2BAA2B,CAAC;AAG9D,KAAK,UAAU,GAAG,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAEpE;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG;IACtB;;;;OAIG;IACH,aAAa,CAAC,EAAE,UAAU,CAAC;IAE3B;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IAEzB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,MAAM,uBAAuB,GAAG;IACpC,CAAC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,SAAS,EACzB,uBAAuB,EAAE,uBAAuB,GAC/C,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"migration-lite.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/db/migration-lite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAIjD,OAAO,KAAK,CAAC,MAAM,+BAA+B,CAAC;AACnD,OAAO,KAAK,EAAC,QAAQ,IAAI,EAAE,EAAC,MAAM,2BAA2B,CAAC;AAG9D,KAAK,UAAU,GAAG,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAEpE;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG;IACtB;;;;OAIG;IACH,aAAa,CAAC,EAAE,UAAU,CAAC;IAE3B;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,UAAU,CAAC;IAEzB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,MAAM,uBAAuB,GAAG;IACpC,CAAC,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,cAAc,EAAE,SAAS,EACzB,uBAAuB,EAAE,uBAAuB,GAC/C,OAAO,CAAC,IAAI,CAAC,CAiIf;AAaD,eAAO,MAAM,cAAc;IACzB;;;;;;OAMG;;IAGH;;;;;;OAMG;;IAGH;;;;OAIG;;aAEH,CAAC;AAGH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAG5D,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,EAAE,GAAG,cAAc,CAmBxD"}
|
|
@@ -46,9 +46,11 @@ async function runSchemaMigrations(log, debugName, dbPath, setupMigration, incre
|
|
|
46
46
|
return versions;
|
|
47
47
|
});
|
|
48
48
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
const start = performance.now();
|
|
50
|
+
db.pragma("optimize");
|
|
51
|
+
const elapsed = performance.now() - start;
|
|
52
|
+
log.info?.(`OPTIMIZE completed (${elapsed.toFixed(2)}ms)`);
|
|
53
|
+
}
|
|
52
54
|
db.pragma("synchronous = NORMAL");
|
|
53
55
|
db.unsafeMode(false);
|
|
54
56
|
assert(versions.dataVersion === codeVersion, () => `Final dataVersion (${versions.dataVersion}) does not match codeVersion (${codeVersion})`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migration-lite.js","names":[],"sources":["../../../../../zero-cache/src/db/migration-lite.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {Database as Db} from '../../../zqlite/src/db.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\n\ntype Operations = (log: LogContext, tx: Db) => Promise<void> | void;\n\n/**\n * Encapsulates the logic for setting up or upgrading to a new schema. After the\n * Migration code successfully completes, {@link runSchemaMigrations}\n * will update the schema version and commit the transaction.\n */\nexport type Migration = {\n /**\n * Perform database operations that create or alter table structure. This is\n * called at most once during lifetime of the application. If a `migrateData()`\n * operation is defined, that will be performed after `migrateSchema()` succeeds.\n */\n migrateSchema?: Operations;\n\n /**\n * Perform database operations to migrate data to the new schema. This is\n * called after `migrateSchema()` (if defined), and may be called again\n * to re-migrate data after the server was rolled back to an earlier version,\n * and rolled forward again.\n *\n * Consequently, the logic in `migrateData()` must be idempotent.\n */\n migrateData?: Operations;\n\n /**\n * Sets the `minSafeVersion` to the specified value, prohibiting running\n * any earlier code versions.\n */\n minSafeVersion?: number;\n};\n\n/**\n * Mapping of incremental migrations to move from the previous old code\n * version to next one. Versions must be non-zero.\n *\n * The schema resulting from performing incremental migrations should be\n * equivalent to that of the `setupMigration` on a blank database.\n *\n * The highest destinationVersion of this map denotes the current\n * \"code version\", and is also used as the destination version when\n * running the initial setup migration on a blank database.\n */\nexport type IncrementalMigrationMap = {\n [destinationVersion: number]: Migration;\n};\n\n/**\n * Ensures that the schema is compatible with the current code, updating and\n * migrating the schema if necessary.\n */\nexport async function runSchemaMigrations(\n log: LogContext,\n debugName: string,\n dbPath: string,\n setupMigration: Migration,\n incrementalMigrationMap: IncrementalMigrationMap,\n): Promise<void> {\n const start = Date.now();\n log = log.withContext(\n 'initSchema',\n randInt(0, Number.MAX_SAFE_INTEGER).toString(36),\n );\n const db = new Database(log, dbPath);\n\n try {\n const versionMigrations = sorted(incrementalMigrationMap);\n assert(\n versionMigrations.length,\n `Must specify a at least one version migration`,\n );\n assert(\n versionMigrations[0][0] > 0,\n `Versions must be non-zero positive numbers`,\n );\n // oxlint-disable-next-line typescript/no-non-null-assertion\n const codeVersion = versionMigrations.at(-1)![0];\n log.info?.(\n `Checking schema for compatibility with ${debugName} at schema v${codeVersion}`,\n );\n\n let versions = await runTransaction(log, db, tx => {\n const versions = getVersionHistory(tx);\n if (codeVersion < versions.minSafeVersion) {\n throw new Error(\n `Cannot run ${debugName} at schema v${codeVersion} because rollback limit is v${versions.minSafeVersion}`,\n );\n }\n\n if (versions.dataVersion > codeVersion) {\n log.info?.(\n `Data is at v${versions.dataVersion}. Resetting to v${codeVersion}`,\n );\n return updateVersionHistory(log, tx, versions, codeVersion);\n }\n return versions;\n });\n if (versions.dataVersion < codeVersion) {\n db.unsafeMode(true); // Enables journal_mode = OFF\n db.pragma('locking_mode = EXCLUSIVE');\n db.pragma('foreign_keys = OFF');\n db.pragma('journal_mode = OFF');\n db.pragma('synchronous = OFF');\n // Unfortunately, AUTO_VACUUM is not compatible with BEGIN CONCURRENT,\n // so it is not an option for the replica file.\n // https://sqlite.org/forum/forumpost/25f183416a\n // db.pragma('auto_vacuum = INCREMENTAL');\n\n const migrations =\n versions.dataVersion === 0\n ? // For the empty database v0, only run the setup migration.\n ([[codeVersion, setupMigration]] as const)\n : versionMigrations;\n\n for (const [dest, migration] of migrations) {\n if (versions.dataVersion < dest) {\n log.info?.(\n `Migrating schema from v${versions.dataVersion} to v${dest}`,\n );\n void log.flush(); // Flush logs before each migration to help debug crash-y migrations.\n\n versions = await runTransaction(log, db, async tx => {\n // Fetch meta from within the transaction to make the migration atomic.\n let versions = getVersionHistory(tx);\n if (versions.dataVersion < dest) {\n versions = await runMigration(log, tx, versions, dest, migration);\n assert(\n versions.dataVersion === dest,\n () =>\n `Migration did not reach target version: expected ${dest}, got ${versions.dataVersion}`,\n );\n }\n return versions;\n });\n }\n }\n\n db.exec('ANALYZE main');\n log.info?.('ANALYZE completed');\n } else {\n // Run optimize whenever opening an sqlite db file as recommended in\n // https://www.sqlite.org/pragma.html#pragma_optimize\n // It is important to run the same initialization steps as is done\n // in the view-syncer (i.e. when preparing database for serving\n // replication) so that any corruption detected in the view-syncer is\n // similarly detected in the change-streamer, facilitating an eventual\n // recovery by resyncing the replica anew.\n db.pragma('optimize = 0x10002');\n // Do not run `quick_check` here. It scans the full database and can\n // dominate startup time for large replicas or slower disks.\n }\n\n db.pragma('synchronous = NORMAL');\n db.unsafeMode(false);\n\n assert(\n versions.dataVersion === codeVersion,\n () =>\n `Final dataVersion (${versions.dataVersion}) does not match codeVersion (${codeVersion})`,\n );\n log.info?.(\n `Running ${debugName} at schema v${codeVersion} (${\n Date.now() - start\n } ms)`,\n );\n } catch (e) {\n if (e instanceof AbortError) {\n // AbortErrors (e.g. AutoResetSignal) are not failures. They are the\n // expected mechanism for signaling that the replica needs a reset (e.g.\n // corruption or an incompatible schema version) so that the caller can\n // resync anew. Logging them at `error` would trip error-based alerting\n // for a normal, self-healing path.\n log.warn?.('Replica requires reset; resync will follow', e);\n } else {\n log.error?.('Error in ensureSchemaMigrated', e);\n }\n throw e;\n } finally {\n db.close();\n void log.flush(); // Flush the logs but do not block server progress on it.\n }\n}\n\nfunction sorted(\n incrementalMigrationMap: IncrementalMigrationMap,\n): [number, Migration][] {\n const versionMigrations: [number, Migration][] = [];\n for (const [v, m] of Object.entries(incrementalMigrationMap)) {\n versionMigrations.push([Number(v), m]);\n }\n return versionMigrations.sort(([a], [b]) => a - b);\n}\n\n// Exposed for tests.\nexport const versionHistory = v.object({\n /**\n * The `schemaVersion` is highest code version that has ever been run\n * on the database, and is used to delineate the structure of the tables\n * in the database. A schemaVersion only moves forward; rolling back to\n * an earlier (safe) code version does not revert schema changes that\n * have already been applied.\n */\n schemaVersion: v.number(),\n\n /**\n * The data version is the code version of the latest server that ran.\n * Note that this may be less than the schemaVersion in the case that\n * a server is rolled back to an earlier version after a schema change.\n * In such a case, data (but not schema), may need to be re-migrated\n * when rolling forward again.\n */\n dataVersion: v.number(),\n\n /**\n * The minimum code version that is safe to run. This is used when\n * a schema migration is not backwards compatible with an older version\n * of the code.\n */\n minSafeVersion: v.number(),\n});\n\n// Exposed for tests.\nexport type VersionHistory = v.Infer<typeof versionHistory>;\n\n// Exposed for tests\nexport function getVersionHistory(db: Db): VersionHistory {\n // Note: The `lock` column transparently ensures that at most one row exists.\n db.prepare(\n `\n CREATE TABLE IF NOT EXISTS \"_zero.versionHistory\" (\n dataVersion INTEGER NOT NULL,\n schemaVersion INTEGER NOT NULL,\n minSafeVersion INTEGER NOT NULL,\n\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n `,\n ).run();\n const result = db\n .prepare(\n 'SELECT dataVersion, schemaVersion, minSafeVersion FROM \"_zero.versionHistory\"',\n )\n .get() as VersionHistory;\n return result ?? {dataVersion: 0, schemaVersion: 0, minSafeVersion: 0};\n}\n\nfunction updateVersionHistory(\n log: LogContext,\n db: Db,\n prev: VersionHistory,\n newVersion: number,\n minSafeVersion?: number,\n): VersionHistory {\n assert(newVersion > 0, 'newVersion must be positive');\n const meta = {\n ...prev,\n dataVersion: newVersion,\n // The schemaVersion never moves backwards.\n schemaVersion: Math.max(newVersion, prev.schemaVersion),\n minSafeVersion: getMinSafeVersion(log, prev, minSafeVersion),\n } satisfies VersionHistory;\n\n db.prepare(\n `\n INSERT INTO \"_zero.versionHistory\" (dataVersion, schemaVersion, minSafeVersion, lock)\n VALUES (@dataVersion, @schemaVersion, @minSafeVersion, 1)\n ON CONFLICT (lock) DO UPDATE\n SET dataVersion=@dataVersion,\n schemaVersion=@schemaVersion,\n minSafeVersion=@minSafeVersion\n `,\n ).run(meta);\n\n return meta;\n}\n\nasync function runMigration(\n log: LogContext,\n tx: Db,\n versions: VersionHistory,\n destinationVersion: number,\n migration: Migration,\n): Promise<VersionHistory> {\n if (versions.schemaVersion < destinationVersion) {\n await migration.migrateSchema?.(log, tx);\n }\n if (versions.dataVersion < destinationVersion) {\n await migration.migrateData?.(log, tx);\n }\n return updateVersionHistory(\n log,\n tx,\n versions,\n destinationVersion,\n migration.minSafeVersion,\n );\n}\n\n/**\n * Bumps the rollback limit [[toAtLeast]] the specified version.\n * Leaves the rollback limit unchanged if it is equal or greater.\n */\nfunction getMinSafeVersion(\n log: LogContext,\n current: VersionHistory,\n proposedSafeVersion?: number,\n): number {\n if (proposedSafeVersion === undefined) {\n return current.minSafeVersion;\n }\n if (current.minSafeVersion >= proposedSafeVersion) {\n // The rollback limit must never move backwards.\n log.debug?.(\n `rollback limit is already at ${current.minSafeVersion}, ` +\n `don't need to bump to ${proposedSafeVersion}`,\n );\n return current.minSafeVersion;\n }\n log.info?.(\n `bumping rollback limit from ${current.minSafeVersion} to ${proposedSafeVersion}`,\n );\n return proposedSafeVersion;\n}\n\n// Note: We use a custom transaction wrapper (instead of db.begin(...)) in order\n// to support async operations within the transaction.\nasync function runTransaction<T>(\n log: LogContext,\n db: Db,\n tx: (db: Db) => Promise<T> | T,\n): Promise<T> {\n db.prepare('BEGIN EXCLUSIVE').run();\n try {\n const result = await tx(db);\n db.prepare('COMMIT').run();\n return result;\n } catch (e) {\n if (e instanceof AbortError) {\n // AbortErrors (e.g. AutoResetSignal) are an expected, self-healing\n // control-flow signal, not a failure. Log at `warn` so the rollback is\n // still visible without tripping error-based alerting.\n log.warn?.('Aborted transaction for reset signal', e);\n } else {\n log.error?.('Aborted transaction due to error', e);\n }\n try {\n db.prepare('ROLLBACK').run();\n } catch (rollbackError) {\n log.error?.('Unable to rollback transaction', rollbackError);\n const combinedError = new Error(\n `Transaction failed and rollback also failed: operation error = ${String(\n e,\n )}; rollback error = ${String(rollbackError)}`,\n );\n combinedError.cause = e;\n throw combinedError;\n }\n throw e;\n }\n}\n"],"mappings":";;;;;;;;;;AA2DA,eAAsB,oBACpB,KACA,WACA,QACA,gBACA,yBACe;CACf,MAAM,QAAQ,KAAK,IAAI;CACvB,MAAM,IAAI,YACR,cACA,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE,CACjD;CACA,MAAM,KAAK,IAAI,SAAS,KAAK,MAAM;CAEnC,IAAI;EACF,MAAM,oBAAoB,OAAO,uBAAuB;EACxD,OACE,kBAAkB,QAClB,+CACF;EACA,OACE,kBAAkB,GAAG,KAAK,GAC1B,4CACF;EAEA,MAAM,cAAc,kBAAkB,GAAG,EAAE,EAAG;EAC9C,IAAI,OACF,0CAA0C,UAAU,cAAc,aACpE;EAEA,IAAI,WAAW,MAAM,eAAe,KAAK,KAAI,OAAM;GACjD,MAAM,WAAW,kBAAkB,EAAE;GACrC,IAAI,cAAc,SAAS,gBACzB,MAAM,IAAI,MACR,cAAc,UAAU,cAAc,YAAY,8BAA8B,SAAS,gBAC3F;GAGF,IAAI,SAAS,cAAc,aAAa;IACtC,IAAI,OACF,eAAe,SAAS,YAAY,kBAAkB,aACxD;IACA,OAAO,qBAAqB,KAAK,IAAI,UAAU,WAAW;GAC5D;GACA,OAAO;EACT,CAAC;EACD,IAAI,SAAS,cAAc,aAAa;GACtC,GAAG,WAAW,IAAI;GAClB,GAAG,OAAO,0BAA0B;GACpC,GAAG,OAAO,oBAAoB;GAC9B,GAAG,OAAO,oBAAoB;GAC9B,GAAG,OAAO,mBAAmB;GAM7B,MAAM,aACJ,SAAS,gBAAgB,IAEpB,CAAC,CAAC,aAAa,cAAc,CAAC,IAC/B;GAEN,KAAK,MAAM,CAAC,MAAM,cAAc,YAC9B,IAAI,SAAS,cAAc,MAAM;IAC/B,IAAI,OACF,0BAA0B,SAAS,YAAY,OAAO,MACxD;IACA,IAAS,MAAM;IAEf,WAAW,MAAM,eAAe,KAAK,IAAI,OAAM,OAAM;KAEnD,IAAI,WAAW,kBAAkB,EAAE;KACnC,IAAI,SAAS,cAAc,MAAM;MAC/B,WAAW,MAAM,aAAa,KAAK,IAAI,UAAU,MAAM,SAAS;MAChE,OACE,SAAS,gBAAgB,YAEvB,oDAAoD,KAAK,QAAQ,SAAS,aAC9E;KACF;KACA,OAAO;IACT,CAAC;GACH;GAGF,GAAG,KAAK,cAAc;GACtB,IAAI,OAAO,mBAAmB;EAChC,OAQE,GAAG,OAAO,oBAAoB;EAKhC,GAAG,OAAO,sBAAsB;EAChC,GAAG,WAAW,KAAK;EAEnB,OACE,SAAS,gBAAgB,mBAEvB,sBAAsB,SAAS,YAAY,gCAAgC,YAAY,EAC3F;EACA,IAAI,OACF,WAAW,UAAU,cAAc,YAAY,IAC7C,KAAK,IAAI,IAAI,MACd,KACH;CACF,SAAS,GAAG;EACV,IAAI,aAAa,YAMf,IAAI,OAAO,8CAA8C,CAAC;OAE1D,IAAI,QAAQ,iCAAiC,CAAC;EAEhD,MAAM;CACR,UAAU;EACR,GAAG,MAAM;EACT,IAAS,MAAM;CACjB;AACF;AAEA,SAAS,OACP,yBACuB;CACvB,MAAM,oBAA2C,CAAC;CAClD,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,uBAAuB,GACzD,kBAAkB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;CAEvC,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC;AACnD;AAG8B,eAAE,OAAO;;;;;;;;CAQrC,eAAe,eAAE,OAAO;;;;;;;;CASxB,aAAa,eAAE,OAAO;;;;;;CAOtB,gBAAgB,eAAE,OAAO;AAC3B,CAAC;AAMD,SAAgB,kBAAkB,IAAwB;CAExD,GAAG,QACD;;;;;;;;GASF,EAAE,IAAI;CAMN,OALe,GACZ,QACC,iFACF,EACC,IACI,KAAU;EAAC,aAAa;EAAG,eAAe;EAAG,gBAAgB;CAAC;AACvE;AAEA,SAAS,qBACP,KACA,IACA,MACA,YACA,gBACgB;CAChB,OAAO,aAAa,GAAG,6BAA6B;CACpD,MAAM,OAAO;EACX,GAAG;EACH,aAAa;EAEb,eAAe,KAAK,IAAI,YAAY,KAAK,aAAa;EACtD,gBAAgB,kBAAkB,KAAK,MAAM,cAAc;CAC7D;CAEA,GAAG,QACD;;;;;;;GAQF,EAAE,IAAI,IAAI;CAEV,OAAO;AACT;AAEA,eAAe,aACb,KACA,IACA,UACA,oBACA,WACyB;CACzB,IAAI,SAAS,gBAAgB,oBAC3B,MAAM,UAAU,gBAAgB,KAAK,EAAE;CAEzC,IAAI,SAAS,cAAc,oBACzB,MAAM,UAAU,cAAc,KAAK,EAAE;CAEvC,OAAO,qBACL,KACA,IACA,UACA,oBACA,UAAU,cACZ;AACF;;;;;AAMA,SAAS,kBACP,KACA,SACA,qBACQ;CACR,IAAI,wBAAwB,KAAA,GAC1B,OAAO,QAAQ;CAEjB,IAAI,QAAQ,kBAAkB,qBAAqB;EAEjD,IAAI,QACF,gCAAgC,QAAQ,eAAe,0BAC5B,qBAC7B;EACA,OAAO,QAAQ;CACjB;CACA,IAAI,OACF,+BAA+B,QAAQ,eAAe,MAAM,qBAC9D;CACA,OAAO;AACT;AAIA,eAAe,eACb,KACA,IACA,IACY;CACZ,GAAG,QAAQ,iBAAiB,EAAE,IAAI;CAClC,IAAI;EACF,MAAM,SAAS,MAAM,GAAG,EAAE;EAC1B,GAAG,QAAQ,QAAQ,EAAE,IAAI;EACzB,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aAAa,YAIf,IAAI,OAAO,wCAAwC,CAAC;OAEpD,IAAI,QAAQ,oCAAoC,CAAC;EAEnD,IAAI;GACF,GAAG,QAAQ,UAAU,EAAE,IAAI;EAC7B,SAAS,eAAe;GACtB,IAAI,QAAQ,kCAAkC,aAAa;GAC3D,MAAM,gCAAgB,IAAI,MACxB,kEAAkE,OAChE,CACF,EAAE,qBAAqB,OAAO,aAAa,GAC7C;GACA,cAAc,QAAQ;GACtB,MAAM;EACR;EACA,MAAM;CACR;AACF"}
|
|
1
|
+
{"version":3,"file":"migration-lite.js","names":[],"sources":["../../../../../zero-cache/src/db/migration-lite.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../shared/src/abort-error.ts';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {randInt} from '../../../shared/src/rand.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport type {Database as Db} from '../../../zqlite/src/db.ts';\nimport {Database} from '../../../zqlite/src/db.ts';\n\ntype Operations = (log: LogContext, tx: Db) => Promise<void> | void;\n\n/**\n * Encapsulates the logic for setting up or upgrading to a new schema. After the\n * Migration code successfully completes, {@link runSchemaMigrations}\n * will update the schema version and commit the transaction.\n */\nexport type Migration = {\n /**\n * Perform database operations that create or alter table structure. This is\n * called at most once during lifetime of the application. If a `migrateData()`\n * operation is defined, that will be performed after `migrateSchema()` succeeds.\n */\n migrateSchema?: Operations;\n\n /**\n * Perform database operations to migrate data to the new schema. This is\n * called after `migrateSchema()` (if defined), and may be called again\n * to re-migrate data after the server was rolled back to an earlier version,\n * and rolled forward again.\n *\n * Consequently, the logic in `migrateData()` must be idempotent.\n */\n migrateData?: Operations;\n\n /**\n * Sets the `minSafeVersion` to the specified value, prohibiting running\n * any earlier code versions.\n */\n minSafeVersion?: number;\n};\n\n/**\n * Mapping of incremental migrations to move from the previous old code\n * version to next one. Versions must be non-zero.\n *\n * The schema resulting from performing incremental migrations should be\n * equivalent to that of the `setupMigration` on a blank database.\n *\n * The highest destinationVersion of this map denotes the current\n * \"code version\", and is also used as the destination version when\n * running the initial setup migration on a blank database.\n */\nexport type IncrementalMigrationMap = {\n [destinationVersion: number]: Migration;\n};\n\n/**\n * Ensures that the schema is compatible with the current code, updating and\n * migrating the schema if necessary.\n */\nexport async function runSchemaMigrations(\n log: LogContext,\n debugName: string,\n dbPath: string,\n setupMigration: Migration,\n incrementalMigrationMap: IncrementalMigrationMap,\n): Promise<void> {\n const start = Date.now();\n log = log.withContext(\n 'initSchema',\n randInt(0, Number.MAX_SAFE_INTEGER).toString(36),\n );\n const db = new Database(log, dbPath);\n\n try {\n const versionMigrations = sorted(incrementalMigrationMap);\n assert(\n versionMigrations.length,\n `Must specify a at least one version migration`,\n );\n assert(\n versionMigrations[0][0] > 0,\n `Versions must be non-zero positive numbers`,\n );\n // oxlint-disable-next-line typescript/no-non-null-assertion\n const codeVersion = versionMigrations.at(-1)![0];\n log.info?.(\n `Checking schema for compatibility with ${debugName} at schema v${codeVersion}`,\n );\n\n let versions = await runTransaction(log, db, tx => {\n const versions = getVersionHistory(tx);\n if (codeVersion < versions.minSafeVersion) {\n throw new Error(\n `Cannot run ${debugName} at schema v${codeVersion} because rollback limit is v${versions.minSafeVersion}`,\n );\n }\n\n if (versions.dataVersion > codeVersion) {\n log.info?.(\n `Data is at v${versions.dataVersion}. Resetting to v${codeVersion}`,\n );\n return updateVersionHistory(log, tx, versions, codeVersion);\n }\n return versions;\n });\n if (versions.dataVersion < codeVersion) {\n db.unsafeMode(true); // Enables journal_mode = OFF\n db.pragma('locking_mode = EXCLUSIVE');\n db.pragma('foreign_keys = OFF');\n db.pragma('journal_mode = OFF');\n db.pragma('synchronous = OFF');\n // Unfortunately, AUTO_VACUUM is not compatible with BEGIN CONCURRENT,\n // so it is not an option for the replica file.\n // https://sqlite.org/forum/forumpost/25f183416a\n // db.pragma('auto_vacuum = INCREMENTAL');\n\n const migrations =\n versions.dataVersion === 0\n ? // For the empty database v0, only run the setup migration.\n ([[codeVersion, setupMigration]] as const)\n : versionMigrations;\n\n for (const [dest, migration] of migrations) {\n if (versions.dataVersion < dest) {\n log.info?.(\n `Migrating schema from v${versions.dataVersion} to v${dest}`,\n );\n void log.flush(); // Flush logs before each migration to help debug crash-y migrations.\n\n versions = await runTransaction(log, db, async tx => {\n // Fetch meta from within the transaction to make the migration atomic.\n let versions = getVersionHistory(tx);\n if (versions.dataVersion < dest) {\n versions = await runMigration(log, tx, versions, dest, migration);\n assert(\n versions.dataVersion === dest,\n () =>\n `Migration did not reach target version: expected ${dest}, got ${versions.dataVersion}`,\n );\n }\n return versions;\n });\n }\n }\n\n // Note: This used to be ANALYZE but that takes a long time for large\n // dbs. The SQLite docs generally recommend optimize instead.\n //\n // https://sqlite.org/pragma.html#pragma_analysis_limit\n //\n // Beginning with SQLite version 3.46.0 (2024-05-23), the recommended\n // way of running ANALYZE is with the PRAGMA optimize command.\n // The PRAGMA optimize will automatically set a reasonable, temporary\n // analysis limit that ensures that the PRAGMA optimize command will\n // finish quickly even on enormous databases. Applications that use\n // the PRAGMA optimize instead of running ANALYZE directly do not need\n // to set an analysis limit.\n const start = performance.now();\n db.pragma('optimize');\n const elapsed = performance.now() - start;\n log.info?.(`OPTIMIZE completed (${elapsed.toFixed(2)}ms)`);\n }\n\n // Do not run `quick_check` here. It scans the full database and can\n // dominate startup time for large replicas or slower disks.\n db.pragma('synchronous = NORMAL');\n db.unsafeMode(false);\n\n assert(\n versions.dataVersion === codeVersion,\n () =>\n `Final dataVersion (${versions.dataVersion}) does not match codeVersion (${codeVersion})`,\n );\n log.info?.(\n `Running ${debugName} at schema v${codeVersion} (${\n Date.now() - start\n } ms)`,\n );\n } catch (e) {\n if (e instanceof AbortError) {\n // AbortErrors (e.g. AutoResetSignal) are not failures. They are the\n // expected mechanism for signaling that the replica needs a reset (e.g.\n // corruption or an incompatible schema version) so that the caller can\n // resync anew. Logging them at `error` would trip error-based alerting\n // for a normal, self-healing path.\n log.warn?.('Replica requires reset; resync will follow', e);\n } else {\n log.error?.('Error in ensureSchemaMigrated', e);\n }\n throw e;\n } finally {\n db.close();\n void log.flush(); // Flush the logs but do not block server progress on it.\n }\n}\n\nfunction sorted(\n incrementalMigrationMap: IncrementalMigrationMap,\n): [number, Migration][] {\n const versionMigrations: [number, Migration][] = [];\n for (const [v, m] of Object.entries(incrementalMigrationMap)) {\n versionMigrations.push([Number(v), m]);\n }\n return versionMigrations.sort(([a], [b]) => a - b);\n}\n\n// Exposed for tests.\nexport const versionHistory = v.object({\n /**\n * The `schemaVersion` is highest code version that has ever been run\n * on the database, and is used to delineate the structure of the tables\n * in the database. A schemaVersion only moves forward; rolling back to\n * an earlier (safe) code version does not revert schema changes that\n * have already been applied.\n */\n schemaVersion: v.number(),\n\n /**\n * The data version is the code version of the latest server that ran.\n * Note that this may be less than the schemaVersion in the case that\n * a server is rolled back to an earlier version after a schema change.\n * In such a case, data (but not schema), may need to be re-migrated\n * when rolling forward again.\n */\n dataVersion: v.number(),\n\n /**\n * The minimum code version that is safe to run. This is used when\n * a schema migration is not backwards compatible with an older version\n * of the code.\n */\n minSafeVersion: v.number(),\n});\n\n// Exposed for tests.\nexport type VersionHistory = v.Infer<typeof versionHistory>;\n\n// Exposed for tests\nexport function getVersionHistory(db: Db): VersionHistory {\n // Note: The `lock` column transparently ensures that at most one row exists.\n db.prepare(\n `\n CREATE TABLE IF NOT EXISTS \"_zero.versionHistory\" (\n dataVersion INTEGER NOT NULL,\n schemaVersion INTEGER NOT NULL,\n minSafeVersion INTEGER NOT NULL,\n\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n `,\n ).run();\n const result = db\n .prepare(\n 'SELECT dataVersion, schemaVersion, minSafeVersion FROM \"_zero.versionHistory\"',\n )\n .get() as VersionHistory;\n return result ?? {dataVersion: 0, schemaVersion: 0, minSafeVersion: 0};\n}\n\nfunction updateVersionHistory(\n log: LogContext,\n db: Db,\n prev: VersionHistory,\n newVersion: number,\n minSafeVersion?: number,\n): VersionHistory {\n assert(newVersion > 0, 'newVersion must be positive');\n const meta = {\n ...prev,\n dataVersion: newVersion,\n // The schemaVersion never moves backwards.\n schemaVersion: Math.max(newVersion, prev.schemaVersion),\n minSafeVersion: getMinSafeVersion(log, prev, minSafeVersion),\n } satisfies VersionHistory;\n\n db.prepare(\n `\n INSERT INTO \"_zero.versionHistory\" (dataVersion, schemaVersion, minSafeVersion, lock)\n VALUES (@dataVersion, @schemaVersion, @minSafeVersion, 1)\n ON CONFLICT (lock) DO UPDATE\n SET dataVersion=@dataVersion,\n schemaVersion=@schemaVersion,\n minSafeVersion=@minSafeVersion\n `,\n ).run(meta);\n\n return meta;\n}\n\nasync function runMigration(\n log: LogContext,\n tx: Db,\n versions: VersionHistory,\n destinationVersion: number,\n migration: Migration,\n): Promise<VersionHistory> {\n if (versions.schemaVersion < destinationVersion) {\n await migration.migrateSchema?.(log, tx);\n }\n if (versions.dataVersion < destinationVersion) {\n await migration.migrateData?.(log, tx);\n }\n return updateVersionHistory(\n log,\n tx,\n versions,\n destinationVersion,\n migration.minSafeVersion,\n );\n}\n\n/**\n * Bumps the rollback limit [[toAtLeast]] the specified version.\n * Leaves the rollback limit unchanged if it is equal or greater.\n */\nfunction getMinSafeVersion(\n log: LogContext,\n current: VersionHistory,\n proposedSafeVersion?: number,\n): number {\n if (proposedSafeVersion === undefined) {\n return current.minSafeVersion;\n }\n if (current.minSafeVersion >= proposedSafeVersion) {\n // The rollback limit must never move backwards.\n log.debug?.(\n `rollback limit is already at ${current.minSafeVersion}, ` +\n `don't need to bump to ${proposedSafeVersion}`,\n );\n return current.minSafeVersion;\n }\n log.info?.(\n `bumping rollback limit from ${current.minSafeVersion} to ${proposedSafeVersion}`,\n );\n return proposedSafeVersion;\n}\n\n// Note: We use a custom transaction wrapper (instead of db.begin(...)) in order\n// to support async operations within the transaction.\nasync function runTransaction<T>(\n log: LogContext,\n db: Db,\n tx: (db: Db) => Promise<T> | T,\n): Promise<T> {\n db.prepare('BEGIN EXCLUSIVE').run();\n try {\n const result = await tx(db);\n db.prepare('COMMIT').run();\n return result;\n } catch (e) {\n if (e instanceof AbortError) {\n // AbortErrors (e.g. AutoResetSignal) are an expected, self-healing\n // control-flow signal, not a failure. Log at `warn` so the rollback is\n // still visible without tripping error-based alerting.\n log.warn?.('Aborted transaction for reset signal', e);\n } else {\n log.error?.('Aborted transaction due to error', e);\n }\n try {\n db.prepare('ROLLBACK').run();\n } catch (rollbackError) {\n log.error?.('Unable to rollback transaction', rollbackError);\n const combinedError = new Error(\n `Transaction failed and rollback also failed: operation error = ${String(\n e,\n )}; rollback error = ${String(rollbackError)}`,\n );\n combinedError.cause = e;\n throw combinedError;\n }\n throw e;\n }\n}\n"],"mappings":";;;;;;;;;;AA2DA,eAAsB,oBACpB,KACA,WACA,QACA,gBACA,yBACe;CACf,MAAM,QAAQ,KAAK,IAAI;CACvB,MAAM,IAAI,YACR,cACA,QAAQ,GAAG,OAAO,gBAAgB,EAAE,SAAS,EAAE,CACjD;CACA,MAAM,KAAK,IAAI,SAAS,KAAK,MAAM;CAEnC,IAAI;EACF,MAAM,oBAAoB,OAAO,uBAAuB;EACxD,OACE,kBAAkB,QAClB,+CACF;EACA,OACE,kBAAkB,GAAG,KAAK,GAC1B,4CACF;EAEA,MAAM,cAAc,kBAAkB,GAAG,EAAE,EAAG;EAC9C,IAAI,OACF,0CAA0C,UAAU,cAAc,aACpE;EAEA,IAAI,WAAW,MAAM,eAAe,KAAK,KAAI,OAAM;GACjD,MAAM,WAAW,kBAAkB,EAAE;GACrC,IAAI,cAAc,SAAS,gBACzB,MAAM,IAAI,MACR,cAAc,UAAU,cAAc,YAAY,8BAA8B,SAAS,gBAC3F;GAGF,IAAI,SAAS,cAAc,aAAa;IACtC,IAAI,OACF,eAAe,SAAS,YAAY,kBAAkB,aACxD;IACA,OAAO,qBAAqB,KAAK,IAAI,UAAU,WAAW;GAC5D;GACA,OAAO;EACT,CAAC;EACD,IAAI,SAAS,cAAc,aAAa;GACtC,GAAG,WAAW,IAAI;GAClB,GAAG,OAAO,0BAA0B;GACpC,GAAG,OAAO,oBAAoB;GAC9B,GAAG,OAAO,oBAAoB;GAC9B,GAAG,OAAO,mBAAmB;GAM7B,MAAM,aACJ,SAAS,gBAAgB,IAEpB,CAAC,CAAC,aAAa,cAAc,CAAC,IAC/B;GAEN,KAAK,MAAM,CAAC,MAAM,cAAc,YAC9B,IAAI,SAAS,cAAc,MAAM;IAC/B,IAAI,OACF,0BAA0B,SAAS,YAAY,OAAO,MACxD;IACA,IAAS,MAAM;IAEf,WAAW,MAAM,eAAe,KAAK,IAAI,OAAM,OAAM;KAEnD,IAAI,WAAW,kBAAkB,EAAE;KACnC,IAAI,SAAS,cAAc,MAAM;MAC/B,WAAW,MAAM,aAAa,KAAK,IAAI,UAAU,MAAM,SAAS;MAChE,OACE,SAAS,gBAAgB,YAEvB,oDAAoD,KAAK,QAAQ,SAAS,aAC9E;KACF;KACA,OAAO;IACT,CAAC;GACH;GAeF,MAAM,QAAQ,YAAY,IAAI;GAC9B,GAAG,OAAO,UAAU;GACpB,MAAM,UAAU,YAAY,IAAI,IAAI;GACpC,IAAI,OAAO,uBAAuB,QAAQ,QAAQ,CAAC,EAAE,IAAI;EAC3D;EAIA,GAAG,OAAO,sBAAsB;EAChC,GAAG,WAAW,KAAK;EAEnB,OACE,SAAS,gBAAgB,mBAEvB,sBAAsB,SAAS,YAAY,gCAAgC,YAAY,EAC3F;EACA,IAAI,OACF,WAAW,UAAU,cAAc,YAAY,IAC7C,KAAK,IAAI,IAAI,MACd,KACH;CACF,SAAS,GAAG;EACV,IAAI,aAAa,YAMf,IAAI,OAAO,8CAA8C,CAAC;OAE1D,IAAI,QAAQ,iCAAiC,CAAC;EAEhD,MAAM;CACR,UAAU;EACR,GAAG,MAAM;EACT,IAAS,MAAM;CACjB;AACF;AAEA,SAAS,OACP,yBACuB;CACvB,MAAM,oBAA2C,CAAC;CAClD,KAAK,MAAM,CAAC,GAAG,MAAM,OAAO,QAAQ,uBAAuB,GACzD,kBAAkB,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;CAEvC,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC;AACnD;AAG8B,eAAE,OAAO;;;;;;;;CAQrC,eAAe,eAAE,OAAO;;;;;;;;CASxB,aAAa,eAAE,OAAO;;;;;;CAOtB,gBAAgB,eAAE,OAAO;AAC3B,CAAC;AAMD,SAAgB,kBAAkB,IAAwB;CAExD,GAAG,QACD;;;;;;;;GASF,EAAE,IAAI;CAMN,OALe,GACZ,QACC,iFACF,EACC,IACI,KAAU;EAAC,aAAa;EAAG,eAAe;EAAG,gBAAgB;CAAC;AACvE;AAEA,SAAS,qBACP,KACA,IACA,MACA,YACA,gBACgB;CAChB,OAAO,aAAa,GAAG,6BAA6B;CACpD,MAAM,OAAO;EACX,GAAG;EACH,aAAa;EAEb,eAAe,KAAK,IAAI,YAAY,KAAK,aAAa;EACtD,gBAAgB,kBAAkB,KAAK,MAAM,cAAc;CAC7D;CAEA,GAAG,QACD;;;;;;;GAQF,EAAE,IAAI,IAAI;CAEV,OAAO;AACT;AAEA,eAAe,aACb,KACA,IACA,UACA,oBACA,WACyB;CACzB,IAAI,SAAS,gBAAgB,oBAC3B,MAAM,UAAU,gBAAgB,KAAK,EAAE;CAEzC,IAAI,SAAS,cAAc,oBACzB,MAAM,UAAU,cAAc,KAAK,EAAE;CAEvC,OAAO,qBACL,KACA,IACA,UACA,oBACA,UAAU,cACZ;AACF;;;;;AAMA,SAAS,kBACP,KACA,SACA,qBACQ;CACR,IAAI,wBAAwB,KAAA,GAC1B,OAAO,QAAQ;CAEjB,IAAI,QAAQ,kBAAkB,qBAAqB;EAEjD,IAAI,QACF,gCAAgC,QAAQ,eAAe,0BAC5B,qBAC7B;EACA,OAAO,QAAQ;CACjB;CACA,IAAI,OACF,+BAA+B,QAAQ,eAAe,MAAM,qBAC9D;CACA,OAAO;AACT;AAIA,eAAe,eACb,KACA,IACA,IACY;CACZ,GAAG,QAAQ,iBAAiB,EAAE,IAAI;CAClC,IAAI;EACF,MAAM,SAAS,MAAM,GAAG,EAAE;EAC1B,GAAG,QAAQ,QAAQ,EAAE,IAAI;EACzB,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aAAa,YAIf,IAAI,OAAO,wCAAwC,CAAC;OAEpD,IAAI,QAAQ,oCAAoC,CAAC;EAEnD,IAAI;GACF,GAAG,QAAQ,UAAU,EAAE,IAAI;EAC7B,SAAS,eAAe;GACtB,IAAI,QAAQ,kCAAkC,aAAa;GAC3D,MAAM,gCAAgB,IAAI,MACxB,kEAAkE,OAChE,CACF,EAAE,qBAAqB,OAAO,aAAa,GAC7C;GACA,cAAc,QAAQ;GACtB,MAAM;EACR;EACA,MAAM;CACR;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replica-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replica-schema.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,iCAAiC,CAAC;AAG9D,OAAO,EAEL,KAAK,uBAAuB,EAE7B,MAAM,+BAA+B,CAAC;AAgBvC,wBAAsB,WAAW,CAC/B,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,GAC3D,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,UAAU,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,iBAoBf;AAkBD,eAAO,MAAM,+BAA+B,0UAW3C,CAAC;AAEF,eAAO,MAAM,oBAAoB,8XAUhC,CAAC;AAEF,eAAO,MAAM,8BAA8B,sMAO1C,CAAC;AAKF,eAAO,MAAM,2BAA2B,0BAA0B,CAAC;AAQnE,eAAO,MAAM,4BAA4B,8aAaxC,CAAC;AA4BF,eAAO,MAAM,yBAAyB,EAAE,uBA+MvC,CAAC;AAGF,eAAO,MAAM,sBAAsB,QAEwB,CAAC"}
|
|
@@ -4,18 +4,24 @@ import { listTables } from "../../../db/lite-tables.js";
|
|
|
4
4
|
import { CREATE_BACKFILLING_TABLE, populateBackfillingFromColumnMetadata } from "../../replicator/schema/backfilling.js";
|
|
5
5
|
import { CREATE_RUNTIME_EVENTS_TABLE, recordEvent } from "../../replicator/schema/replication-state.js";
|
|
6
6
|
import { isSQLiteCorruption, logSQLiteCorruptionDiagnostics } from "../../../db/sqlite-corruption.js";
|
|
7
|
+
import { deleteLiteDB } from "../../../db/delete-lite-db.js";
|
|
7
8
|
import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
|
|
8
9
|
import { runSchemaMigrations } from "../../../db/migration-lite.js";
|
|
10
|
+
import { existsSync, renameSync } from "node:fs";
|
|
9
11
|
//#region ../zero-cache/src/services/change-source/common/replica-schema.ts
|
|
10
12
|
async function initReplica(log, debugName, dbPath, initialSync) {
|
|
13
|
+
const isInitialSync = !existsSync(dbPath);
|
|
14
|
+
const migrationPath = isInitialSync ? `${dbPath}.tmp` : dbPath;
|
|
15
|
+
if (isInitialSync) deleteLiteDB(migrationPath);
|
|
11
16
|
const setupMigration = {
|
|
12
17
|
migrateSchema: (log, tx) => initialSync(log, tx),
|
|
13
18
|
minSafeVersion: 1
|
|
14
19
|
};
|
|
15
20
|
try {
|
|
16
|
-
await runSchemaMigrations(log, debugName,
|
|
21
|
+
await runSchemaMigrations(log, debugName, migrationPath, setupMigration, schemaVersionMigrationMap);
|
|
22
|
+
if (isInitialSync) renameSync(migrationPath, dbPath);
|
|
17
23
|
} catch (e) {
|
|
18
|
-
throwAutoResetForCorruption(log, debugName,
|
|
24
|
+
throwAutoResetForCorruption(log, debugName, migrationPath, e);
|
|
19
25
|
}
|
|
20
26
|
}
|
|
21
27
|
async function upgradeReplica(log, debugName, dbPath) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n runSchemaMigrations,\n type IncrementalMigrationMap,\n type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n CREATE_BACKFILLING_TABLE,\n populateBackfillingFromColumnMetadata,\n} from '../../replicator/schema/backfilling.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n CREATE_RUNTIME_EVENTS_TABLE,\n recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n const setupMigration: Migration = {\n migrateSchema: (log, tx) => initialSync(log, tx),\n minSafeVersion: 1,\n };\n\n try {\n await runSchemaMigrations(\n log,\n debugName,\n dbPath,\n setupMigration,\n schemaVersionMigrationMap,\n );\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, dbPath, e);\n }\n}\n\nexport async function upgradeReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n) {\n try {\n await runSchemaMigrations(\n log,\n debugName,\n dbPath,\n // setupMigration should never be invoked\n {\n migrateSchema: () => {\n throw new Error(\n 'This should only be called for already synced replicas',\n );\n },\n },\n schemaVersionMigrationMap,\n );\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, dbPath, e);\n }\n}\n\nfunction throwAutoResetForCorruption(\n log: LogContext,\n debugName: string,\n dbPath: string,\n e: unknown,\n): never {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n throw new AutoResetSignal(\n `replica database appears corrupt: ${String(e)}`,\n {cause: e},\n );\n }\n throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n CREATE TABLE \"_zero.column_metadata\" (\n table_name TEXT NOT NULL,\n column_name TEXT NOT NULL,\n upstream_type TEXT NOT NULL,\n is_not_null INTEGER NOT NULL,\n is_enum INTEGER NOT NULL,\n is_array INTEGER NOT NULL,\n character_max_length INTEGER,\n PRIMARY KEY (table_name, column_name)\n );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n CREATE TABLE \"_zero.changeLog2\" (\n \"stateVersion\" TEXT NOT NULL,\n \"pos\" INT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"rowKey\" TEXT NOT NULL,\n \"op\" TEXT NOT NULL,\n PRIMARY KEY(\"stateVersion\", \"pos\"),\n UNIQUE(\"table\", \"rowKey\")\n );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n CREATE TABLE \"_zero.tableMetadata\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n`;\n\n// Deliberately shadows the same names in `replicator/change-log-db.ts`: these\n// are the v14 replica's, that module's are the change-log database's, and the\n// two are frozen apart.\nexport const V14_CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nconst V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n '_zero.changeLogStream_writeTimeMs';\n\n// The change-log stream table, as migration 14 created it and migration 16\n// drops it. Frozen at its v14 shape: a migration is history, so this must not\n// be kept in sync with the live definition in `replicator/change-log-db.ts`,\n// which now describes a table in a different database.\nexport const CREATE_V14_CHANGE_LOG_STREAM = /*sql*/ `\n CREATE TABLE \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\n \"watermark\" TEXT NOT NULL,\n \"pos\" INTEGER NOT NULL,\n \"change\" TEXT NOT NULL,\n \"precommit\" TEXT,\n \"writeTimeMs\" INTEGER,\n PRIMARY KEY (\"watermark\", \"pos\")\n );\n\n CREATE INDEX \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n ON \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\n// Idempotent because migrateData may run again after a rollback followed by a\n// roll-forward, and both rows go in with one statement so a partial seed is\n// not reachable.\nconst SEED_V14_CHANGE_LOG_STREAM = /*sql*/ `\n INSERT INTO \"${V14_CHANGE_LOG_STREAM_TABLE}\"\n (\"watermark\", \"pos\", \"change\", \"precommit\", \"writeTimeMs\")\n VALUES\n (@stateVersion, 0, '{\"tag\":\"begin\"}', NULL, NULL),\n (@stateVersion, 1, '{\"tag\":\"commit\"}', @stateVersion, @writeTimeMs)\n ON CONFLICT (\"watermark\", \"pos\") DO NOTHING\n`;\n\nfunction seedV14ChangeLogStream(db: Database): void {\n const state = db\n .prepare(/*sql*/ `\n SELECT \"stateVersion\", \"writeTimeMs\" FROM \"_zero.replicationState\"\n `)\n .get<{stateVersion: string; writeTimeMs: number | null} | undefined>();\n assert(state !== undefined, 'replication state must be initialized');\n assert(\n state.writeTimeMs !== null,\n 'replication state writeTimeMs must be initialized',\n );\n db.prepare(SEED_V14_CHANGE_LOG_STREAM).run(state);\n}\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n // There's no incremental migration from v1. Just reset the replica.\n 4: {\n migrateSchema: () => {\n throw new AutoResetSignal('upgrading replica to new schema');\n },\n minSafeVersion: 3,\n },\n\n 5: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n },\n migrateData: (_, db) => {\n recordEvent(db, 'upgrade');\n },\n },\n\n // Revised in the migration to v8 because the v6 code was incomplete.\n 6: {},\n\n 7: {\n migrateSchema: (_, db) => {\n // Note: The original \"changeLog\" table is kept so that the replica file\n // is compatible with older zero-caches. However, it is truncated for\n // space savings (since historic changes were never read).\n db.exec(`DELETE FROM \"_zero.changeLog\"`);\n // First version of changeLog2\n db.exec(CREATE_V7_CHANGE_LOG);\n },\n },\n\n 8: {\n migrateSchema: (_, db) => {\n const tableExists = db\n .prepare(\n `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n )\n .get();\n\n if (!tableExists) {\n db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n }\n },\n migrateData: (_, db) => {\n // Re-populate the ColumnMetadataStore; the original migration\n // at v6 was incomplete, as covered replicas migrated from earlier\n // versions but did not initialize the table for new replicas.\n db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n const tables = listTables(db, false, false);\n populateFromExistingTables(db, tables);\n },\n },\n\n 9: {\n migrateSchema: (_, db) => {\n db.exec(\n /*sql*/ `\n ALTER TABLE \"_zero.changeLog2\" \n ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n ALTER TABLE \"_zero.column_metadata\"\n ADD COLUMN backfill TEXT;\n ` + CREATE_V9_TABLE_METADATA_TABLE,\n );\n },\n },\n\n 10: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.replicationConfig\" \n ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n `);\n },\n },\n\n 11: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n -- Removing the NOT NULL constraint from \"metadata\" requires copying\n -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"upstreamMetadata\" TEXT;\n UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n `);\n },\n },\n\n 12: {\n migrateSchema: (_, db) => {\n // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n // It is needed for backwards compatibility.\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"metadata\" TEXT;\n `);\n },\n\n migrateData: (_, db) => {\n // For rollback then roll forward, re-copy anything written to metadata.\n db.exec(/*sql*/ `\n UPDATE \"_zero.tableMetadata\" \n SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n \"metadata\" = NULL;\n `);\n },\n },\n\n 13: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n `);\n },\n\n migrateData: (_, db) => {\n db.exec(/*sql*/ `\n UPDATE \"_zero.replicationState\" \n SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n },\n },\n\n // Deliberately left as it shipped, even though v16 immediately undoes it.\n // Rewriting it to a no-op would strand replicas already at v14: they roll\n // back to v14 code, which expects the table this created.\n 14: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_V14_CHANGE_LOG_STREAM);\n },\n\n migrateData: (_, db) => {\n seedV14ChangeLogStream(db);\n },\n },\n\n 15: {\n migrateSchema: (_, db) => {\n // Make writeTimeMs NOT NULL. In SQLite the only way to do this is to\n // create a new table, copy the data over, delete the old table, and\n // rename. This is cheap because it's a single row table.\n db.exec(/*sql*/ `\n CREATE TABLE \"_zero.replicationState2\" (\n stateVersion TEXT NOT NULL,\n writeTimeMs INTEGER NOT NULL,\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n\n INSERT INTO \"_zero.replicationState2\" (stateVersion, writeTimeMs, lock)\n SELECT stateVersion, writeTimeMs, lock FROM \"_zero.replicationState\";\n\n DROP TABLE \"_zero.replicationState\";\n\n ALTER TABLE \"_zero.replicationState2\" RENAME TO \"_zero.replicationState\";\n `);\n },\n },\n\n // The change log moved out of the replica and into its own database\n // (`${replicaFile}-change-log`), so the replica's copy is dead weight in\n // every file the backup ships and every follower downloads. Nothing reads it\n // as of this version.\n //\n // `IF EXISTS` because a fresh replica never runs the incremental migrations\n // at all — its setup migration creates the current schema directly, which no\n // longer includes this table — and because a rollback to v14 followed by a\n // roll-forward re-runs `migrateData` while skipping `migrateSchema`.\n //\n // The freed pages are reclaimed by the existing `vacuumIntervalHours`\n // trigger in `workers/replicator.ts`; no special handling here.\n //\n // No `minSafeVersion`: an older zero-cache still runs against a v16 replica,\n // and with `sqliteChangeLogMode=off` (the default) it never looks for this\n // table. Bumping the rollback limit would turn every rollback into a hard\n // startup failure to protect a configuration that is not enabled anywhere.\n 16: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n DROP INDEX IF EXISTS \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\";\n DROP TABLE IF EXISTS \"${V14_CHANGE_LOG_STREAM_TABLE}\";\n `);\n },\n },\n\n // `_zero.column_metadata.backfill` records which columns are backfilling but\n // is keyed by lite table name, which has no inverse: it cannot be turned back\n // into the `BackfillRequest`s that a change log initialized from this replica\n // would have to send. This table carries the upstream identity alongside the\n // same state. See `replicator/schema/backfilling.ts`.\n //\n // No `minSafeVersion`: an older zero-cache runs fine against a v17 replica,\n // since nothing at v16 reads the new table. Its writes to\n // `column_metadata.backfill` are what `migrateData` rebuilds from when rolling\n // forward again, so a rollback costs nothing but the re-seed.\n 17: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_BACKFILLING_TABLE);\n },\n\n migrateData: (lc, db) => {\n populateBackfillingFromColumnMetadata(lc, db);\n },\n },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;AAwBA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QACA,gBACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAYtD,IAAa,8BAA8B;AAC3C,IAAM,yCACJ;AAMF,IAAa,+BAAuC;kBAClC,4BAA4B;;;;;;;;;kBAS5B,uCAAuC;UAC/C,4BAA4B;;;AAOtC,IAAM,6BAAqC;iBAC1B,4BAA4B;;;;;;;AAQ7C,SAAS,uBAAuB,IAAoB;CAClD,MAAM,QAAQ,GACX,QAAgB;;KAEhB,EACA,IAAoE;CACvE,OAAO,UAAU,KAAA,GAAW,uCAAuC;CACnE,OACE,MAAM,gBAAgB,MACtB,mDACF;CACA,GAAG,QAAQ,0BAA0B,EAAE,IAAI,KAAK;AAClD;AAEA,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;CAKA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,4BAA4B;EACtC;EAEA,cAAc,GAAG,OAAO;GACtB,uBAAuB,EAAE;EAC3B;CACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAa;;;;;;;;;;;;;OAaf;CACH,EACF;CAmBA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;gCACU,uCAAuC;gCACvC,4BAA4B;OACrD;CACH,EACF;CAYA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,wBAAwB;EAClC;EAEA,cAAc,IAAI,OAAO;GACvB,sCAAsC,IAAI,EAAE;EAC9C;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}
|
|
1
|
+
{"version":3,"file":"replica-schema.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/common/replica-schema.ts"],"sourcesContent":["import {existsSync, renameSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport {assert} from '../../../../../shared/src/asserts.ts';\nimport type {Database} from '../../../../../zqlite/src/db.ts';\nimport {deleteLiteDB} from '../../../db/delete-lite-db.ts';\nimport {listTables} from '../../../db/lite-tables.ts';\nimport {\n runSchemaMigrations,\n type IncrementalMigrationMap,\n type Migration,\n} from '../../../db/migration-lite.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../../db/sqlite-corruption.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n CREATE_BACKFILLING_TABLE,\n populateBackfillingFromColumnMetadata,\n} from '../../replicator/schema/backfilling.ts';\nimport {populateFromExistingTables} from '../../replicator/schema/column-metadata.ts';\nimport {\n CREATE_RUNTIME_EVENTS_TABLE,\n recordEvent,\n} from '../../replicator/schema/replication-state.ts';\n\nexport async function initReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n initialSync: (lc: LogContext, tx: Database) => Promise<void>,\n): Promise<void> {\n const isInitialSync = !existsSync(dbPath);\n const migrationPath = isInitialSync ? `${dbPath}.tmp` : dbPath;\n if (isInitialSync) {\n deleteLiteDB(migrationPath);\n }\n\n const setupMigration: Migration = {\n migrateSchema: (log, tx) => initialSync(log, tx),\n minSafeVersion: 1,\n };\n\n try {\n await runSchemaMigrations(\n log,\n debugName,\n migrationPath,\n setupMigration,\n schemaVersionMigrationMap,\n );\n if (isInitialSync) {\n renameSync(migrationPath, dbPath);\n }\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, migrationPath, e);\n }\n}\n\nexport async function upgradeReplica(\n log: LogContext,\n debugName: string,\n dbPath: string,\n) {\n try {\n await runSchemaMigrations(\n log,\n debugName,\n dbPath,\n // setupMigration should never be invoked\n {\n migrateSchema: () => {\n throw new Error(\n 'This should only be called for already synced replicas',\n );\n },\n },\n schemaVersionMigrationMap,\n );\n } catch (e) {\n throwAutoResetForCorruption(log, debugName, dbPath, e);\n }\n}\n\nfunction throwAutoResetForCorruption(\n log: LogContext,\n debugName: string,\n dbPath: string,\n e: unknown,\n): never {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(log, debugName, dbPath, e);\n throw new AutoResetSignal(\n `replica database appears corrupt: ${String(e)}`,\n {cause: e},\n );\n }\n throw e;\n}\n\nexport const CREATE_V6_COLUMN_METADATA_TABLE = /*sql*/ `\n CREATE TABLE \"_zero.column_metadata\" (\n table_name TEXT NOT NULL,\n column_name TEXT NOT NULL,\n upstream_type TEXT NOT NULL,\n is_not_null INTEGER NOT NULL,\n is_enum INTEGER NOT NULL,\n is_array INTEGER NOT NULL,\n character_max_length INTEGER,\n PRIMARY KEY (table_name, column_name)\n );\n`;\n\nexport const CREATE_V7_CHANGE_LOG = /*sql*/ `\n CREATE TABLE \"_zero.changeLog2\" (\n \"stateVersion\" TEXT NOT NULL,\n \"pos\" INT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"rowKey\" TEXT NOT NULL,\n \"op\" TEXT NOT NULL,\n PRIMARY KEY(\"stateVersion\", \"pos\"),\n UNIQUE(\"table\", \"rowKey\")\n );\n`;\n\nexport const CREATE_V9_TABLE_METADATA_TABLE = /*sql*/ `\n CREATE TABLE \"_zero.tableMetadata\" (\n \"schema\" TEXT NOT NULL,\n \"table\" TEXT NOT NULL,\n \"metadata\" TEXT NOT NULL,\n PRIMARY KEY (\"schema\", \"table\")\n );\n`;\n\n// Deliberately shadows the same names in `replicator/change-log-db.ts`: these\n// are the v14 replica's, that module's are the change-log database's, and the\n// two are frozen apart.\nexport const V14_CHANGE_LOG_STREAM_TABLE = '_zero.changeLogStream';\nconst V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX =\n '_zero.changeLogStream_writeTimeMs';\n\n// The change-log stream table, as migration 14 created it and migration 16\n// drops it. Frozen at its v14 shape: a migration is history, so this must not\n// be kept in sync with the live definition in `replicator/change-log-db.ts`,\n// which now describes a table in a different database.\nexport const CREATE_V14_CHANGE_LOG_STREAM = /*sql*/ `\n CREATE TABLE \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\n \"watermark\" TEXT NOT NULL,\n \"pos\" INTEGER NOT NULL,\n \"change\" TEXT NOT NULL,\n \"precommit\" TEXT,\n \"writeTimeMs\" INTEGER,\n PRIMARY KEY (\"watermark\", \"pos\")\n );\n\n CREATE INDEX \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\"\n ON \"${V14_CHANGE_LOG_STREAM_TABLE}\" (\"writeTimeMs\", \"watermark\")\n WHERE \"writeTimeMs\" IS NOT NULL;\n`;\n\n// Idempotent because migrateData may run again after a rollback followed by a\n// roll-forward, and both rows go in with one statement so a partial seed is\n// not reachable.\nconst SEED_V14_CHANGE_LOG_STREAM = /*sql*/ `\n INSERT INTO \"${V14_CHANGE_LOG_STREAM_TABLE}\"\n (\"watermark\", \"pos\", \"change\", \"precommit\", \"writeTimeMs\")\n VALUES\n (@stateVersion, 0, '{\"tag\":\"begin\"}', NULL, NULL),\n (@stateVersion, 1, '{\"tag\":\"commit\"}', @stateVersion, @writeTimeMs)\n ON CONFLICT (\"watermark\", \"pos\") DO NOTHING\n`;\n\nfunction seedV14ChangeLogStream(db: Database): void {\n const state = db\n .prepare(/*sql*/ `\n SELECT \"stateVersion\", \"writeTimeMs\" FROM \"_zero.replicationState\"\n `)\n .get<{stateVersion: string; writeTimeMs: number | null} | undefined>();\n assert(state !== undefined, 'replication state must be initialized');\n assert(\n state.writeTimeMs !== null,\n 'replication state writeTimeMs must be initialized',\n );\n db.prepare(SEED_V14_CHANGE_LOG_STREAM).run(state);\n}\n\nexport const schemaVersionMigrationMap: IncrementalMigrationMap = {\n // There's no incremental migration from v1. Just reset the replica.\n 4: {\n migrateSchema: () => {\n throw new AutoResetSignal('upgrading replica to new schema');\n },\n minSafeVersion: 3,\n },\n\n 5: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_RUNTIME_EVENTS_TABLE);\n },\n migrateData: (_, db) => {\n recordEvent(db, 'upgrade');\n },\n },\n\n // Revised in the migration to v8 because the v6 code was incomplete.\n 6: {},\n\n 7: {\n migrateSchema: (_, db) => {\n // Note: The original \"changeLog\" table is kept so that the replica file\n // is compatible with older zero-caches. However, it is truncated for\n // space savings (since historic changes were never read).\n db.exec(`DELETE FROM \"_zero.changeLog\"`);\n // First version of changeLog2\n db.exec(CREATE_V7_CHANGE_LOG);\n },\n },\n\n 8: {\n migrateSchema: (_, db) => {\n const tableExists = db\n .prepare(\n `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_zero.column_metadata'`,\n )\n .get();\n\n if (!tableExists) {\n db.exec(CREATE_V6_COLUMN_METADATA_TABLE);\n }\n },\n migrateData: (_, db) => {\n // Re-populate the ColumnMetadataStore; the original migration\n // at v6 was incomplete, as covered replicas migrated from earlier\n // versions but did not initialize the table for new replicas.\n db.exec(/*sql*/ `DELETE FROM \"_zero.column_metadata\"`);\n\n const tables = listTables(db, false, false);\n populateFromExistingTables(db, tables);\n },\n },\n\n 9: {\n migrateSchema: (_, db) => {\n db.exec(\n /*sql*/ `\n ALTER TABLE \"_zero.changeLog2\" \n ADD COLUMN \"backfillingColumnVersions\" TEXT DEFAULT '{}';\n ALTER TABLE \"_zero.column_metadata\"\n ADD COLUMN backfill TEXT;\n ` + CREATE_V9_TABLE_METADATA_TABLE,\n );\n },\n },\n\n 10: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.replicationConfig\" \n ADD COLUMN \"initialSyncContext\" TEXT DEFAULT '{}';\n `);\n },\n },\n\n 11: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"minRowVersion\" TEXT NOT NULL DEFAULT '00';\n\n -- Removing the NOT NULL constraint from \"metadata\" requires copying\n -- the column. We piggyback the rename to \"upstreamMetadata\" here.\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"upstreamMetadata\" TEXT;\n UPDATE \"_zero.tableMetadata\" SET \"upstreamMetadata\" = \"metadata\";\n ALTER TABLE \"_zero.tableMetadata\" DROP \"metadata\";\n `);\n },\n },\n\n 12: {\n migrateSchema: (_, db) => {\n // Bring back the \"metadata\" column removed in v11, but as a NULL-able column.\n // It is needed for backwards compatibility.\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.tableMetadata\"\n ADD COLUMN \"metadata\" TEXT;\n `);\n },\n\n migrateData: (_, db) => {\n // For rollback then roll forward, re-copy anything written to metadata.\n db.exec(/*sql*/ `\n UPDATE \"_zero.tableMetadata\" \n SET \"upstreamMetadata\" = COALESCE(\"metadata\", \"upstreamMetadata\"),\n \"metadata\" = NULL;\n `);\n },\n },\n\n 13: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n ALTER TABLE \"_zero.replicationState\" ADD COLUMN writeTimeMs INTEGER;\n `);\n },\n\n migrateData: (_, db) => {\n db.exec(/*sql*/ `\n UPDATE \"_zero.replicationState\" \n SET writeTimeMs = COALESCE(writeTimeMs, unixepoch('subsec') * 1000)`);\n },\n },\n\n // Deliberately left as it shipped, even though v16 immediately undoes it.\n // Rewriting it to a no-op would strand replicas already at v14: they roll\n // back to v14 code, which expects the table this created.\n 14: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_V14_CHANGE_LOG_STREAM);\n },\n\n migrateData: (_, db) => {\n seedV14ChangeLogStream(db);\n },\n },\n\n 15: {\n migrateSchema: (_, db) => {\n // Make writeTimeMs NOT NULL. In SQLite the only way to do this is to\n // create a new table, copy the data over, delete the old table, and\n // rename. This is cheap because it's a single row table.\n db.exec(/*sql*/ `\n CREATE TABLE \"_zero.replicationState2\" (\n stateVersion TEXT NOT NULL,\n writeTimeMs INTEGER NOT NULL,\n lock INTEGER PRIMARY KEY DEFAULT 1 CHECK (lock=1)\n );\n\n INSERT INTO \"_zero.replicationState2\" (stateVersion, writeTimeMs, lock)\n SELECT stateVersion, writeTimeMs, lock FROM \"_zero.replicationState\";\n\n DROP TABLE \"_zero.replicationState\";\n\n ALTER TABLE \"_zero.replicationState2\" RENAME TO \"_zero.replicationState\";\n `);\n },\n },\n\n // The change log moved out of the replica and into its own database\n // (`${replicaFile}-change-log`), so the replica's copy is dead weight in\n // every file the backup ships and every follower downloads. Nothing reads it\n // as of this version.\n //\n // `IF EXISTS` because a fresh replica never runs the incremental migrations\n // at all — its setup migration creates the current schema directly, which no\n // longer includes this table — and because a rollback to v14 followed by a\n // roll-forward re-runs `migrateData` while skipping `migrateSchema`.\n //\n // The freed pages are reclaimed by the existing `vacuumIntervalHours`\n // trigger in `workers/replicator.ts`; no special handling here.\n //\n // No `minSafeVersion`: an older zero-cache still runs against a v16 replica,\n // and with `sqliteChangeLogMode=off` (the default) it never looks for this\n // table. Bumping the rollback limit would turn every rollback into a hard\n // startup failure to protect a configuration that is not enabled anywhere.\n 16: {\n migrateSchema: (_, db) => {\n db.exec(/*sql*/ `\n DROP INDEX IF EXISTS \"${V14_CHANGE_LOG_STREAM_WRITE_TIME_INDEX}\";\n DROP TABLE IF EXISTS \"${V14_CHANGE_LOG_STREAM_TABLE}\";\n `);\n },\n },\n\n // `_zero.column_metadata.backfill` records which columns are backfilling but\n // is keyed by lite table name, which has no inverse: it cannot be turned back\n // into the `BackfillRequest`s that a change log initialized from this replica\n // would have to send. This table carries the upstream identity alongside the\n // same state. See `replicator/schema/backfilling.ts`.\n //\n // No `minSafeVersion`: an older zero-cache runs fine against a v17 replica,\n // since nothing at v16 reads the new table. Its writes to\n // `column_metadata.backfill` are what `migrateData` rebuilds from when rolling\n // forward again, so a rollback costs nothing but the re-seed.\n 17: {\n migrateSchema: (_, db) => {\n db.exec(CREATE_BACKFILLING_TABLE);\n },\n\n migrateData: (lc, db) => {\n populateBackfillingFromColumnMetadata(lc, db);\n },\n },\n};\n\n// Referenced in tests.\nexport const CURRENT_SCHEMA_VERSION = Object.keys(\n schemaVersionMigrationMap,\n).reduce((prev, curr) => Math.max(prev, parseInt(curr)), 0);\n"],"mappings":";;;;;;;;;;;AA0BA,eAAsB,YACpB,KACA,WACA,QACA,aACe;CACf,MAAM,gBAAgB,CAAC,WAAW,MAAM;CACxC,MAAM,gBAAgB,gBAAgB,GAAG,OAAO,QAAQ;CACxD,IAAI,eACF,aAAa,aAAa;CAG5B,MAAM,iBAA4B;EAChC,gBAAgB,KAAK,OAAO,YAAY,KAAK,EAAE;EAC/C,gBAAgB;CAClB;CAEA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,eACA,gBACA,yBACF;EACA,IAAI,eACF,WAAW,eAAe,MAAM;CAEpC,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,eAAe,CAAC;CAC9D;AACF;AAEA,eAAsB,eACpB,KACA,WACA,QACA;CACA,IAAI;EACF,MAAM,oBACJ,KACA,WACA,QAEA,EACE,qBAAqB;GACnB,MAAM,IAAI,MACR,wDACF;EACF,EACF,GACA,yBACF;CACF,SAAS,GAAG;EACV,4BAA4B,KAAK,WAAW,QAAQ,CAAC;CACvD;AACF;AAEA,SAAS,4BACP,KACA,WACA,QACA,GACO;CACP,IAAI,mBAAmB,CAAC,GAAG;EACzB,+BAA+B,KAAK,WAAW,QAAQ,CAAC;EACxD,MAAM,IAAI,gBACR,qCAAqC,OAAO,CAAC,KAC7C,EAAC,OAAO,EAAC,CACX;CACF;CACA,MAAM;AACR;AAEA,IAAa,kCAA0C;;;;;;;;;;;;AAavD,IAAa,uBAA+B;;;;;;;;;;;AAY5C,IAAa,iCAAyC;;;;;;;;AAYtD,IAAa,8BAA8B;AAC3C,IAAM,yCACJ;AAMF,IAAa,+BAAuC;kBAClC,4BAA4B;;;;;;;;;kBAS5B,uCAAuC;UAC/C,4BAA4B;;;AAOtC,IAAM,6BAAqC;iBAC1B,4BAA4B;;;;;;;AAQ7C,SAAS,uBAAuB,IAAoB;CAClD,MAAM,QAAQ,GACX,QAAgB;;KAEhB,EACA,IAAoE;CACvE,OAAO,UAAU,KAAA,GAAW,uCAAuC;CACnE,OACE,MAAM,gBAAgB,MACtB,mDACF;CACA,GAAG,QAAQ,0BAA0B,EAAE,IAAI,KAAK;AAClD;AAEA,IAAa,4BAAqD;CAEhE,GAAG;EACD,qBAAqB;GACnB,MAAM,IAAI,gBAAgB,iCAAiC;EAC7D;EACA,gBAAgB;CAClB;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,2BAA2B;EACrC;EACA,cAAc,GAAG,OAAO;GACtB,YAAY,IAAI,SAAS;EAC3B;CACF;CAGA,GAAG,CAAC;CAEJ,GAAG,EACD,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAK,+BAA+B;EAEvC,GAAG,KAAK,oBAAoB;CAC9B,EACF;CAEA,GAAG;EACD,gBAAgB,GAAG,OAAO;GAOxB,IAAI,CANgB,GACjB,QACC,qFACF,EACC,IAEE,GACH,GAAG,KAAK,+BAA+B;EAE3C;EACA,cAAc,GAAG,OAAO;GAItB,GAAG,KAAa,qCAAqC;GAGrD,2BAA2B,IADZ,WAAW,IAAI,OAAO,KACN,CAAM;EACvC;CACF;CAEA,GAAG,EACD,gBAAgB,GAAG,OAAO;EACxB,GAAG,KACO;;;;;UAKN,8BACJ;CACF,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;OAGf;CACH,EACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;;;;;;;;;;OAUf;CACH,EACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GAGxB,GAAG,KAAa;;;OAGf;EACH;EAEA,cAAc,GAAG,OAAO;GAEtB,GAAG,KAAa;;;;OAIf;EACH;CACF;CAEA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAa;;OAEf;EACH;EAEA,cAAc,GAAG,OAAO;GACtB,GAAG,KAAa;;8EAEwD;EAC1E;CACF;CAKA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,4BAA4B;EACtC;EAEA,cAAc,GAAG,OAAO;GACtB,uBAAuB,EAAE;EAC3B;CACF;CAEA,IAAI,EACF,gBAAgB,GAAG,OAAO;EAIxB,GAAG,KAAa;;;;;;;;;;;;;OAaf;CACH,EACF;CAmBA,IAAI,EACF,gBAAgB,GAAG,OAAO;EACxB,GAAG,KAAa;gCACU,uCAAuC;gCACvC,4BAA4B;OACrD;CACH,EACF;CAYA,IAAI;EACF,gBAAgB,GAAG,OAAO;GACxB,GAAG,KAAK,wBAAwB;EAClC;EAEA,cAAc,IAAI,OAAO;GACvB,sCAAsC,IAAI,EAAE;EAC9C;CACF;AACF;AAGsC,OAAO,KAC3C,yBACF,EAAE,QAAQ,MAAM,SAAS,KAAK,IAAI,MAAM,SAAS,IAAI,CAAC,GAAG,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"litestream3-prometheus-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"litestream3-prometheus-poller.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAMjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAOnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,qBAAqB,CAAC;AAE3D,eAAO,MAAM,iBAAiB,QAAS,CAAC;AAExC;;;GAGG;AACH,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,wBAAwB,QAAc,CAAC;AAIpD;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,gCAAgC,QAAc,CAAC;AAG5D;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;AAEtD;;;;;;;;;;;;;;;GAeG;AACH,qBAAa,2BAA2B;;gBAwCpC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,eAAe,EAAE,MAAM,EACvB,iBAAiB,EAAE,mBAAmB;IAexC;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC;IA8BlC,QAAQ,CAAC,+BAA+B,sBAgBtC;CAmSH"}
|
|
@@ -2,6 +2,7 @@ import { assert } from "../../../../shared/src/asserts.js";
|
|
|
2
2
|
import { must } from "../../../../shared/src/must.js";
|
|
3
3
|
import { getOrCreateCounter } from "../../observability/metrics.js";
|
|
4
4
|
import { RunningState, UnrecoverableError } from "../running-state.js";
|
|
5
|
+
import { versionFromLexi } from "../../types/lexi-version.js";
|
|
5
6
|
import { Subscription } from "../../types/subscription.js";
|
|
6
7
|
import { litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs } from "../litestream/metrics.js";
|
|
7
8
|
import { statSync } from "node:fs";
|
|
@@ -160,11 +161,16 @@ var Litestream3PrometheusPoller = class {
|
|
|
160
161
|
const watermark = metric.labels?.watermark;
|
|
161
162
|
const name = metric.labels?.name;
|
|
162
163
|
const time = /* @__PURE__ */ new Date(parseFloat(metric.value) * 1e3);
|
|
163
|
-
if (watermark)
|
|
164
|
-
watermark
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
164
|
+
if (watermark) try {
|
|
165
|
+
versionFromLexi(watermark);
|
|
166
|
+
yield {
|
|
167
|
+
watermark,
|
|
168
|
+
time,
|
|
169
|
+
name
|
|
170
|
+
};
|
|
171
|
+
} catch (e) {
|
|
172
|
+
this.#lc.warn?.(`ignoring invalid watermark ${watermark}`, { metric }, e);
|
|
173
|
+
}
|
|
168
174
|
}
|
|
169
175
|
}
|
|
170
176
|
async #checkWatermarks() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n litestreamBackupVerificationDuration,\n litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n readonly #lc: LogContext;\n readonly #replicaFile: string;\n readonly #backupURL: string;\n readonly #metricsEndpoint: string;\n readonly #state = new RunningState('litestream3-prometheus-poller');\n\n readonly #stream: Subscription<BackedUpWatermark>;\n readonly #watermarks = new Map<string, Date>();\n\n readonly #verifyBackupState: BackupStateVerifier;\n readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n description:\n 'Number of change-log purges blocked because the actual backup state ' +\n '(as listed from the replica destination) could not be verified, or ' +\n 'is older than the backup progress claimed by litestream metrics. ' +\n 'A steadily increasing value indicates a wedged or failing backup. ' +\n 'The \"backup-wedged\" reason is emitted once just before the process ' +\n 'shuts itself down due to a persistently stale backup.',\n });\n\n #lastWatermark: string = '';\n #latestBackupTime: Date | null = null;\n // The time of the most recent object actually verified in the backup\n // destination, or `null` if none has ever been verified. Only assigned from a\n // successful verifyBackupState() and never cleared, so a non-null value also\n // means a restorable backup has been confirmed to exist (which is what gates\n // the snapshot `status` signal and satisfies the initial-backup deadline).\n #lastVerifiedUploadTime: Date | null = null;\n // Epoch ms at which the actual backup state was first observed to be\n // continuously behind the watermark litestream claims to have backed up,\n // or `null` while the backup is keeping up. Reset whenever a purge is\n // confirmed against a real upload.\n #backupStaleSince: number | null = null;\n // Epoch ms at which run() started, used to bound how long to wait for the\n // first restorable backup to appear. `null` until run() is called.\n #runStartTime: number | null = null;\n #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n backupURL: string,\n metricsEndpoint: string,\n verifyBackupState: BackupStateVerifier,\n ) {\n this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n this.#replicaFile = replicaFile;\n this.#backupURL = backupURL;\n this.#metricsEndpoint = metricsEndpoint;\n this.#verifyBackupState = verifyBackupState;\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n clearInterval(this.#checkMetricsTimer);\n this.#state.stop(this.#lc);\n },\n });\n }\n\n /**\n * Starts polling for watermarks and pushes new ones to returned stream.\n * Must only be called once.\n */\n start(): Source<BackedUpWatermark> {\n assert(this.#runStartTime === null, `Already called start()`);\n this.#runStartTime = Date.now();\n this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n this.#checkMetricsTimer = setInterval(\n this.checkVerifyAndPublishWatermarks,\n CHECK_INTERVAL_MS,\n );\n return this.#stream;\n }\n\n /**\n * Verifies whether a restorable backup actually exists in the destination\n * (as opposed to what litestream metrics merely claim). On success the\n * verified upload time is cached — seeding the cleanup-verification fast path\n * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n * Returns false if no backup is listable yet (e.g. the initial snapshot is\n * still uploading).\n */\n async #confirmRestorableBackup(): Promise<boolean> {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n return true;\n } catch (e) {\n this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n return false;\n }\n }\n\n // Exported for testing\n readonly checkVerifyAndPublishWatermarks = async () => {\n try {\n await this.#checkWatermarks();\n } catch (e) {\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n }\n try {\n await this.#publishVerifiedWatermarks();\n } catch (e) {\n this.#lc.warn?.(`error scheduling cleanup`, e);\n }\n try {\n await this.#checkInitialBackupDeadline();\n } catch (e) {\n this.#lc.warn?.(`error checking initial backup deadline`, e);\n }\n };\n\n async *#fetchWatermarks(): AsyncGenerator<{\n watermark: string;\n time: Date;\n name?: string | undefined;\n }> {\n const metricsEndpoint = this.#metricsEndpoint;\n const signal = this.#state.signal;\n let resp;\n try {\n resp = await fetch(metricsEndpoint, {signal});\n } catch (e) {\n if (signal.aborted) {\n // not an error.\n return;\n }\n // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n // log them and skip the watermark check until the next interval.\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n return;\n }\n if (!resp.ok) {\n this.#lc.warn?.(\n `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n );\n return;\n }\n\n const families = parsePrometheusTextFormat(await resp.text());\n for (const family of families) {\n if (\n family.type === 'GAUGE' &&\n family.name === 'litestream_replica_progress'\n ) {\n for (const metric of family.metrics) {\n const watermark = metric.labels?.watermark;\n const name = metric.labels?.name;\n const time = new Date(parseFloat(metric.value) * 1000);\n\n if (watermark) {\n yield {watermark, time, name};\n }\n }\n }\n }\n }\n\n async #checkWatermarks() {\n for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n this.#lc.info?.(\n `replicated watermark=${watermark} to ${name}` +\n ` at ${time.toISOString()}.`,\n );\n this.#watermarks.set(watermark, time);\n this.#latestBackupTime = time;\n }\n }\n return this.#latestBackupTime;\n }\n\n async #publishVerifiedWatermarks() {\n const maxBackup = this.#maxBackupBefore(Date.now());\n if (!maxBackup) {\n return;\n }\n const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n // Purge guard: the watermarks (and their backup times) come from\n // litestream metrics, which are exported when litestream *believes*\n // an upload succeeded, and have been observed to advance even when\n // nothing was actually written to the backup destination. Purging the\n // change-log based on a falsely advancing watermark permanently breaks\n // the ability to restore + catch up. Before advancing the cleanup\n // watermark, verify it against the actual backup state: a claimed\n // backup time is only trusted if an object was actually uploaded to\n // the replica destination at (or after) that time, modulo clock skew.\n if (!this.#confirmedDurable(claimedTime)) {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n } catch (e) {\n this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n // Skipping the purge is safe: the change-log just grows.\n this.#lc.warn?.(\n `unable to verify backup state. skipping change-log cleanup ` +\n `up to watermark ${maxWatermark} ` +\n `(claimed backup time ${claimedTime.toISOString()})`,\n e,\n );\n return;\n }\n }\n // Watermarks whose backup time isn't yet confirmed durable remain in the\n // map and are re-evaluated at the next check.\n const lastUpload = must(this.#lastVerifiedUploadTime);\n const verifiedBackup = this.#maxBackupBefore(\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n );\n if (!verifiedBackup) {\n this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n const now = Date.now();\n if (this.#backupStaleSince === null) {\n this.#backupStaleSince = now;\n }\n const staleForMs = now - this.#backupStaleSince;\n this.#lc.warn?.(\n `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n `litestream claims it was backed up at ` +\n `${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded to ${this.#backupURL} was at ` +\n `${lastUpload.toISOString()}. ` +\n `The backup has been stale for ${staleForMs} ms.`,\n );\n if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n }\n return;\n }\n // The cleanup watermark advanced past a real upload, so the backup is\n // keeping up: clear the staleness clock.\n this.#backupStaleSince = null;\n const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n for (const watermark of this.#watermarks.keys()) {\n if (watermark <= verifiedWatermark) {\n this.#watermarks.delete(watermark);\n }\n }\n this.#stream.push({\n watermark: verifiedWatermark,\n backupTimeMs: backupTime.getTime(),\n });\n this.#lastWatermark = verifiedWatermark;\n }\n\n async #verifyBackupStateTimed(): Promise<Date> {\n const start = performance.now();\n let result: 'success' | 'error' = 'error';\n try {\n const ret = await this.#verifyBackupState();\n result = 'success';\n return ret;\n } finally {\n litestreamBackupVerificationDuration().recordMs(\n performance.now() - start,\n {\n ...litestreamMonitorMetricAttrs(\n this.#backupURL,\n 'legacy',\n 'replication_manager',\n ),\n result,\n },\n );\n }\n }\n\n /**\n * Called when the backup has been continuously stale for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n * backup, so the process is shut down by rejecting the {@link run()}\n * promise. The exit is non-zero and logged at ERROR for alerting; on\n * restart the monitor re-verifies, so a still-wedged backup keeps the\n * process down until the destination is fixed.\n */\n #shutDownOnWedgedBackup(\n staleForMs: number,\n claimedTime: Date,\n lastUpload: Date,\n ) {\n this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n const err = new UnrecoverableError(\n `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n `at ${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n `not advanced for ${staleForMs} ms (grace period ` +\n `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n `the backup; investigate the backup destination.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /**\n * On a fresh stack, fails loudly if no restorable backup has appeared within\n * a generous, replica-size-scaled grace period (see\n * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n * onto the replication-manager — the producer responsible for creating the\n * backup — instead of leaving view-syncers to wait (and the platform to keep\n * restarting them) while the real fault is here. Does nothing once a\n * restorable backup has been confirmed.\n */\n async #checkInitialBackupDeadline() {\n if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n return;\n }\n const elapsed = Date.now() - this.#runStartTime;\n if (elapsed < this.#initialBackupGraceMs()) {\n return;\n }\n // The deadline has elapsed without a confirmed backup. Do one definitive\n // check against the destination before giving up, in case a backup landed\n // but was never exercised by a reservation or a cleanup verification.\n if (await this.#confirmRestorableBackup()) {\n return;\n }\n this.#shutDownOnMissingInitialBackup(elapsed);\n }\n\n /**\n * The grace period for the first restorable backup, scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n * the minimum (one unit) if the replica file size cannot be determined.\n */\n #initialBackupGraceMs(): number {\n let bytes = 0;\n try {\n bytes = statSync(this.#replicaFile).size;\n } catch {\n // Replica file not present/readable yet; fall back to the minimum grace.\n }\n const units = Math.max(\n 1,\n Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n );\n return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n }\n\n /**\n * Called when no restorable backup has appeared within\n * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n * view-syncer can restore — and the fault is the backup pipeline, so the\n * process exits non-zero with a logged error for alerting, mirroring\n * {@link #shutDownOnWedgedBackup}.\n */\n #shutDownOnMissingInitialBackup(elapsedMs: number) {\n const err = new UnrecoverableError(\n `no restorable backup has appeared at ${this.#backupURL} within ` +\n `${elapsedMs} ms of startup (grace period ` +\n `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n `appears to be broken; shutting down so that the replication-manager ` +\n `is flagged as the cause rather than the view-syncers waiting on it. ` +\n `Investigate litestream replication to ${this.#backupURL}.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /**\n * Returns the newest watermark whose backup time is at or before `cutoff`\n * (epoch ms), or `''` if there is none.\n */\n #maxBackupBefore(\n cutoff: number,\n ): {watermark: string; backupTime: Date} | null {\n let max = '';\n let maxBackupTime: Date | undefined;\n for (const [watermark, backupTime] of this.#watermarks) {\n if (backupTime.getTime() <= cutoff && watermark > max) {\n max = watermark;\n maxBackupTime = backupTime;\n }\n }\n if (!maxBackupTime) {\n return null;\n }\n return {watermark: max, backupTime: maxBackupTime};\n }\n\n /**\n * Returns `true` if the actual backup state, as last verified against the\n * backup destination, confirms that data claimed to be backed up at\n * `claimedTime` is durable (i.e. an object was actually uploaded at or\n * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n * clock skew).\n */\n #confirmedDurable(claimedTime: Date): boolean {\n const lastUpload = this.#lastVerifiedUploadTime;\n return (\n lastUpload !== null &&\n claimedTime.getTime() <=\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n );\n }\n}\n"],"mappings":";;;;;;;;;AAeA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GAErD,IAAI,WACF,MAAM;IAAC;IAAW;IAAM;GAAI;EAEhC;CAGN;CAEA,MAAMW,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}
|
|
1
|
+
{"version":3,"file":"litestream3-prometheus-poller.js","names":["#lc","#replicaFile","#backupURL","#metricsEndpoint","#state","#stream","#watermarks","#verifyBackupState","#purgesBlocked","#checkMetricsTimer","#runStartTime","#confirmRestorableBackup","#lastVerifiedUploadTime","#verifyBackupStateTimed","#checkWatermarks","#publishVerifiedWatermarks","#checkInitialBackupDeadline","#fetchWatermarks","#lastWatermark","#latestBackupTime","#maxBackupBefore","#confirmedDurable","#backupStaleSince","#shutDownOnWedgedBackup","#initialBackupGraceMs","#shutDownOnMissingInitialBackup"],"sources":["../../../../../../zero-cache/src/services/change-streamer/litestream3-prometheus-poller.ts"],"sourcesContent":["import {statSync} from 'node:fs';\nimport type {LogContext} from '@rocicorp/logger';\nimport parsePrometheusTextFormat from 'parse-prometheus-text-format';\nimport {assert} from '../../../../shared/src/asserts.ts';\nimport {must} from '../../../../shared/src/must.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {versionFromLexi} from '../../types/lexi-version.ts';\nimport type {Source} from '../../types/streams.ts';\nimport {Subscription} from '../../types/subscription.ts';\nimport {\n litestreamBackupVerificationDuration,\n litestreamMonitorMetricAttrs,\n} from '../litestream/metrics.ts';\nimport {RunningState, UnrecoverableError} from '../running-state.ts';\nimport type {BackedUpWatermark} from './backup-monitor.ts';\n\nexport const CHECK_INTERVAL_MS = 10_000;\n\n/**\n * Allowance for clock skew between the machine reporting litestream metrics\n * and the timestamps reported by the backup destination (e.g. S3).\n */\nexport const BACKUP_VERIFICATION_SLACK_MS = 60_000;\n\n/**\n * How long the actual backup state may remain *continuously* behind the\n * backup progress claimed by litestream metrics before the backup is\n * considered genuinely wedged and the process is shut down.\n *\n * A wedged backup is dangerous: litestream believes it is making durable\n * progress when it is not, which can silently corrupt the backup (the WAL\n * format only makes *some* such gaps detectable). Once that is confirmed to\n * be the persistent state, continuing to run is worse than crashing, so the\n * process exits loudly (non-zero, with a logged error) and the wedged backup\n * destination becomes the priority to investigate.\n *\n * The change-log is conservatively *not* purged for the entire time the\n * backup is stale, so the only cost of a generous grace period is unbounded\n * change-log growth. We therefore err well on the side of slack: the backup\n * must stay stuck across many {@link CHECK_INTERVAL_MS} checks before we tear\n * the server down, so that a transient hiccup (a slow or briefly unreachable\n * destination, or litestream restarting) does not trigger a shutdown. The\n * staleness clock is reset the moment a purge is confirmed against a real\n * upload.\n */\nexport const WEDGED_SHUTDOWN_GRACE_MS = 15 * 60_000; // 15 minutes\n\nconst GiB = 1024 ** 3;\n\n/**\n * How long the BackupMonitor waits for the *first* restorable backup to appear\n * before concluding that the backup pipeline is broken and shutting the process\n * down (see {@link BackupMonitor.#shutDownOnMissingInitialBackup}).\n *\n * On a fresh stack the first backup is not restorable until the initial\n * Postgres->replica sync completes and litestream finishes uploading the\n * initial snapshot — a multipart upload that is not listable in the destination\n * until it is committed. The producer cannot observe that sub-snapshot progress,\n * so this is necessarily a generous timeout. It is scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance (~1 hour per\n * 50GB) so that it never fires before the platform would restart a view-syncer\n * that is waiting on the backup.\n */\nexport const INITIAL_BACKUP_GRACE_MS_PER_UNIT = 60 * 60_000; // 1 hour ...\nconst INITIAL_BACKUP_GRACE_UNIT_BYTES = 50 * GiB; // ... per 50 GiB\n\n/**\n * How often the BackupMonitor re-checks whether the first restorable backup has\n * appeared while a view-syncer holds a snapshot reservation open during a cold\n * start.\n */\nexport const RESTORABLE_BACKUP_POLL_INTERVAL_MS = 10_000;\n\n/**\n * Returns the time of the most recent object actually uploaded to the\n * backup replica destination (e.g. as determined by listing the snapshots\n * and WAL segments in S3). Rejects if the backup state cannot be determined.\n *\n * See `getLastBackupTime()` in `../litestream/commands.ts` for the\n * production implementation.\n */\nexport type BackupStateVerifier = () => Promise<Date>;\n\n/**\n * The Litestream3PrometheusPoller polls the litestream \"/metrics\" endpoint to\n * track the watermark (label) value of the `litestream_replica_progress` gauge\n * and emits verified watermarks to a stream of {@link BackedUpWatermark}s.\n *\n * See: https://github.com/rocicorp/litestream/pull/3\n *\n * Because the watermarks reported by litestream metrics reflect what\n * litestream *believes* has been backed up (which has been observed to\n * diverge from reality when uploads silently fail), the cleanup watermark is\n * only published after verifying it against the actual backup state in the\n * replica destination via a {@link BackupStateVerifier}. If the\n * actual backup state stays behind the claimed progress for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}, the backup is treated as wedged and the\n * process is shut down rather than risk corrupting the backup.\n */\nexport class Litestream3PrometheusPoller {\n readonly #lc: LogContext;\n readonly #replicaFile: string;\n readonly #backupURL: string;\n readonly #metricsEndpoint: string;\n readonly #state = new RunningState('litestream3-prometheus-poller');\n\n readonly #stream: Subscription<BackedUpWatermark>;\n readonly #watermarks = new Map<string, Date>();\n\n readonly #verifyBackupState: BackupStateVerifier;\n readonly #purgesBlocked = getOrCreateCounter('replica', 'purge_blocked', {\n description:\n 'Number of change-log purges blocked because the actual backup state ' +\n '(as listed from the replica destination) could not be verified, or ' +\n 'is older than the backup progress claimed by litestream metrics. ' +\n 'A steadily increasing value indicates a wedged or failing backup. ' +\n 'The \"backup-wedged\" reason is emitted once just before the process ' +\n 'shuts itself down due to a persistently stale backup.',\n });\n\n #lastWatermark: string = '';\n #latestBackupTime: Date | null = null;\n // The time of the most recent object actually verified in the backup\n // destination, or `null` if none has ever been verified. Only assigned from a\n // successful verifyBackupState() and never cleared, so a non-null value also\n // means a restorable backup has been confirmed to exist (which is what gates\n // the snapshot `status` signal and satisfies the initial-backup deadline).\n #lastVerifiedUploadTime: Date | null = null;\n // Epoch ms at which the actual backup state was first observed to be\n // continuously behind the watermark litestream claims to have backed up,\n // or `null` while the backup is keeping up. Reset whenever a purge is\n // confirmed against a real upload.\n #backupStaleSince: number | null = null;\n // Epoch ms at which run() started, used to bound how long to wait for the\n // first restorable backup to appear. `null` until run() is called.\n #runStartTime: number | null = null;\n #checkMetricsTimer: NodeJS.Timeout | undefined;\n\n constructor(\n lc: LogContext,\n replicaFile: string,\n backupURL: string,\n metricsEndpoint: string,\n verifyBackupState: BackupStateVerifier,\n ) {\n this.#lc = lc.withContext('component', 'litestream3-prometheus-poller');\n this.#replicaFile = replicaFile;\n this.#backupURL = backupURL;\n this.#metricsEndpoint = metricsEndpoint;\n this.#verifyBackupState = verifyBackupState;\n this.#stream = Subscription.create<BackedUpWatermark>({\n cleanup: () => {\n clearInterval(this.#checkMetricsTimer);\n this.#state.stop(this.#lc);\n },\n });\n }\n\n /**\n * Starts polling for watermarks and pushes new ones to returned stream.\n * Must only be called once.\n */\n start(): Source<BackedUpWatermark> {\n assert(this.#runStartTime === null, `Already called start()`);\n this.#runStartTime = Date.now();\n this.#lc.info?.(`monitoring backups at ${this.#metricsEndpoint}`);\n this.#checkMetricsTimer = setInterval(\n this.checkVerifyAndPublishWatermarks,\n CHECK_INTERVAL_MS,\n );\n return this.#stream;\n }\n\n /**\n * Verifies whether a restorable backup actually exists in the destination\n * (as opposed to what litestream metrics merely claim). On success the\n * verified upload time is cached — seeding the cleanup-verification fast path\n * ({@link #confirmedDurable}) — and the initial-backup deadline is satisfied.\n * Returns false if no backup is listable yet (e.g. the initial snapshot is\n * still uploading).\n */\n async #confirmRestorableBackup(): Promise<boolean> {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n return true;\n } catch (e) {\n this.#lc.info?.(`backup not yet restorable at ${this.#backupURL}`, e);\n return false;\n }\n }\n\n // Exported for testing\n readonly checkVerifyAndPublishWatermarks = async () => {\n try {\n await this.#checkWatermarks();\n } catch (e) {\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n }\n try {\n await this.#publishVerifiedWatermarks();\n } catch (e) {\n this.#lc.warn?.(`error scheduling cleanup`, e);\n }\n try {\n await this.#checkInitialBackupDeadline();\n } catch (e) {\n this.#lc.warn?.(`error checking initial backup deadline`, e);\n }\n };\n\n async *#fetchWatermarks(): AsyncGenerator<{\n watermark: string;\n time: Date;\n name?: string | undefined;\n }> {\n const metricsEndpoint = this.#metricsEndpoint;\n const signal = this.#state.signal;\n let resp;\n try {\n resp = await fetch(metricsEndpoint, {signal});\n } catch (e) {\n if (signal.aborted) {\n // not an error.\n return;\n }\n // Treat exceptions from fetch (e.g. network errors) as non-fatal, and simply\n // log them and skip the watermark check until the next interval.\n this.#lc.warn?.(`unable to fetch metrics at ${this.#metricsEndpoint}`, e);\n return;\n }\n if (!resp.ok) {\n this.#lc.warn?.(\n `unable to fetch metrics at ${this.#metricsEndpoint}: ${await resp.text()}`,\n );\n return;\n }\n\n const families = parsePrometheusTextFormat(await resp.text());\n for (const family of families) {\n if (\n family.type === 'GAUGE' &&\n family.name === 'litestream_replica_progress'\n ) {\n for (const metric of family.metrics) {\n const watermark = metric.labels?.watermark;\n const name = metric.labels?.name;\n const time = new Date(parseFloat(metric.value) * 1000);\n if (watermark) {\n try {\n versionFromLexi(watermark); // Sanity check: Ensure that it is valid.\n yield {watermark, time, name};\n } catch (e) {\n this.#lc.warn?.(\n `ignoring invalid watermark ${watermark}`,\n {metric},\n e,\n );\n }\n }\n }\n }\n }\n }\n\n async #checkWatermarks() {\n for await (const {watermark, name, time} of this.#fetchWatermarks()) {\n if (watermark > this.#lastWatermark && !this.#watermarks.has(watermark)) {\n this.#lc.info?.(\n `replicated watermark=${watermark} to ${name}` +\n ` at ${time.toISOString()}.`,\n );\n this.#watermarks.set(watermark, time);\n this.#latestBackupTime = time;\n }\n }\n return this.#latestBackupTime;\n }\n\n async #publishVerifiedWatermarks() {\n const maxBackup = this.#maxBackupBefore(Date.now());\n if (!maxBackup) {\n return;\n }\n const {watermark: maxWatermark, backupTime: claimedTime} = maxBackup;\n // Purge guard: the watermarks (and their backup times) come from\n // litestream metrics, which are exported when litestream *believes*\n // an upload succeeded, and have been observed to advance even when\n // nothing was actually written to the backup destination. Purging the\n // change-log based on a falsely advancing watermark permanently breaks\n // the ability to restore + catch up. Before advancing the cleanup\n // watermark, verify it against the actual backup state: a claimed\n // backup time is only trusted if an object was actually uploaded to\n // the replica destination at (or after) that time, modulo clock skew.\n if (!this.#confirmedDurable(claimedTime)) {\n try {\n this.#lastVerifiedUploadTime = await this.#verifyBackupStateTimed();\n } catch (e) {\n this.#purgesBlocked.add(1, {reason: 'verification-failed'});\n // Skipping the purge is safe: the change-log just grows.\n this.#lc.warn?.(\n `unable to verify backup state. skipping change-log cleanup ` +\n `up to watermark ${maxWatermark} ` +\n `(claimed backup time ${claimedTime.toISOString()})`,\n e,\n );\n return;\n }\n }\n // Watermarks whose backup time isn't yet confirmed durable remain in the\n // map and are re-evaluated at the next check.\n const lastUpload = must(this.#lastVerifiedUploadTime);\n const verifiedBackup = this.#maxBackupBefore(\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS,\n );\n if (!verifiedBackup) {\n this.#purgesBlocked.add(1, {reason: 'backup-stale'});\n const now = Date.now();\n if (this.#backupStaleSince === null) {\n this.#backupStaleSince = now;\n }\n const staleForMs = now - this.#backupStaleSince;\n this.#lc.warn?.(\n `blocked change-log cleanup up to watermark ${maxWatermark}: ` +\n `litestream claims it was backed up at ` +\n `${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded to ${this.#backupURL} was at ` +\n `${lastUpload.toISOString()}. ` +\n `The backup has been stale for ${staleForMs} ms.`,\n );\n if (staleForMs >= WEDGED_SHUTDOWN_GRACE_MS) {\n this.#shutDownOnWedgedBackup(staleForMs, claimedTime, lastUpload);\n }\n return;\n }\n // The cleanup watermark advanced past a real upload, so the backup is\n // keeping up: clear the staleness clock.\n this.#backupStaleSince = null;\n const {watermark: verifiedWatermark, backupTime} = verifiedBackup;\n for (const watermark of this.#watermarks.keys()) {\n if (watermark <= verifiedWatermark) {\n this.#watermarks.delete(watermark);\n }\n }\n this.#stream.push({\n watermark: verifiedWatermark,\n backupTimeMs: backupTime.getTime(),\n });\n this.#lastWatermark = verifiedWatermark;\n }\n\n async #verifyBackupStateTimed(): Promise<Date> {\n const start = performance.now();\n let result: 'success' | 'error' = 'error';\n try {\n const ret = await this.#verifyBackupState();\n result = 'success';\n return ret;\n } finally {\n litestreamBackupVerificationDuration().recordMs(\n performance.now() - start,\n {\n ...litestreamMonitorMetricAttrs(\n this.#backupURL,\n 'legacy',\n 'replication_manager',\n ),\n result,\n },\n );\n }\n }\n\n /**\n * Called when the backup has been continuously stale for longer than\n * {@link WEDGED_SHUTDOWN_GRACE_MS}. Continuing to run risks corrupting the\n * backup, so the process is shut down by rejecting the {@link run()}\n * promise. The exit is non-zero and logged at ERROR for alerting; on\n * restart the monitor re-verifies, so a still-wedged backup keeps the\n * process down until the destination is fixed.\n */\n #shutDownOnWedgedBackup(\n staleForMs: number,\n claimedTime: Date,\n lastUpload: Date,\n ) {\n this.#purgesBlocked.add(1, {reason: 'backup-wedged'});\n const err = new UnrecoverableError(\n `backup at ${this.#backupURL} is wedged: litestream claims a backup ` +\n `at ${claimedTime.toISOString()}, but the last object actually ` +\n `uploaded was at ${lastUpload.toISOString()}, and the backup has ` +\n `not advanced for ${staleForMs} ms (grace period ` +\n `${WEDGED_SHUTDOWN_GRACE_MS} ms). Shutting down to avoid corrupting ` +\n `the backup; investigate the backup destination.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /**\n * On a fresh stack, fails loudly if no restorable backup has appeared within\n * a generous, replica-size-scaled grace period (see\n * {@link INITIAL_BACKUP_GRACE_MS_PER_UNIT}). This moves the \"give up\" decision\n * onto the replication-manager — the producer responsible for creating the\n * backup — instead of leaving view-syncers to wait (and the platform to keep\n * restarting them) while the real fault is here. Does nothing once a\n * restorable backup has been confirmed.\n */\n async #checkInitialBackupDeadline() {\n if (this.#lastVerifiedUploadTime !== null || this.#runStartTime === null) {\n return;\n }\n const elapsed = Date.now() - this.#runStartTime;\n if (elapsed < this.#initialBackupGraceMs()) {\n return;\n }\n // The deadline has elapsed without a confirmed backup. Do one definitive\n // check against the destination before giving up, in case a backup landed\n // but was never exercised by a reservation or a cleanup verification.\n if (await this.#confirmRestorableBackup()) {\n return;\n }\n this.#shutDownOnMissingInitialBackup(elapsed);\n }\n\n /**\n * The grace period for the first restorable backup, scaled by replica size to\n * mirror the platform's storage-scaled startup-probe allowance. Falls back to\n * the minimum (one unit) if the replica file size cannot be determined.\n */\n #initialBackupGraceMs(): number {\n let bytes = 0;\n try {\n bytes = statSync(this.#replicaFile).size;\n } catch {\n // Replica file not present/readable yet; fall back to the minimum grace.\n }\n const units = Math.max(\n 1,\n Math.ceil(bytes / INITIAL_BACKUP_GRACE_UNIT_BYTES),\n );\n return units * INITIAL_BACKUP_GRACE_MS_PER_UNIT;\n }\n\n /**\n * Called when no restorable backup has appeared within\n * {@link #initialBackupGraceMs}. Continuing to run is pointless — no\n * view-syncer can restore — and the fault is the backup pipeline, so the\n * process exits non-zero with a logged error for alerting, mirroring\n * {@link #shutDownOnWedgedBackup}.\n */\n #shutDownOnMissingInitialBackup(elapsedMs: number) {\n const err = new UnrecoverableError(\n `no restorable backup has appeared at ${this.#backupURL} within ` +\n `${elapsedMs} ms of startup (grace period ` +\n `${this.#initialBackupGraceMs()} ms). The initial backup pipeline ` +\n `appears to be broken; shutting down so that the replication-manager ` +\n `is flagged as the cause rather than the view-syncers waiting on it. ` +\n `Investigate litestream replication to ${this.#backupURL}.`,\n );\n this.#lc.error?.(err.message);\n this.#stream.fail(err);\n }\n\n /**\n * Returns the newest watermark whose backup time is at or before `cutoff`\n * (epoch ms), or `''` if there is none.\n */\n #maxBackupBefore(\n cutoff: number,\n ): {watermark: string; backupTime: Date} | null {\n let max = '';\n let maxBackupTime: Date | undefined;\n for (const [watermark, backupTime] of this.#watermarks) {\n if (backupTime.getTime() <= cutoff && watermark > max) {\n max = watermark;\n maxBackupTime = backupTime;\n }\n }\n if (!maxBackupTime) {\n return null;\n }\n return {watermark: max, backupTime: maxBackupTime};\n }\n\n /**\n * Returns `true` if the actual backup state, as last verified against the\n * backup destination, confirms that data claimed to be backed up at\n * `claimedTime` is durable (i.e. an object was actually uploaded at or\n * after `claimedTime`, allowing {@link BACKUP_VERIFICATION_SLACK_MS} of\n * clock skew).\n */\n #confirmedDurable(claimedTime: Date): boolean {\n const lastUpload = this.#lastVerifiedUploadTime;\n return (\n lastUpload !== null &&\n claimedTime.getTime() <=\n lastUpload.getTime() + BACKUP_VERIFICATION_SLACK_MS\n );\n }\n}\n"],"mappings":";;;;;;;;;;AAgBA,IAAa,oBAAoB;;;;;AAMjC,IAAa,+BAA+B;;;;;;;;;;;;;;;;;;;;;;AAuB5C,IAAa,2BAA2B,KAAK;AAE7C,IAAM,MAAM,QAAQ;;;;;;;;;;;;;;;AAgBpB,IAAa,mCAAmC,KAAK;AACrD,IAAM,kCAAkC,KAAK;;;;;;;;;;;;;;;;;AAmC7C,IAAa,8BAAb,MAAyC;CACvC;CACA;CACA;CACA;CACA,SAAkB,IAAI,aAAa,+BAA+B;CAElE;CACA,8BAAuB,IAAI,IAAkB;CAE7C;CACA,iBAA0B,mBAAmB,WAAW,iBAAiB,EACvE,aACE,uYAMJ,CAAC;CAED,iBAAyB;CACzB,oBAAiC;CAMjC,0BAAuC;CAKvC,oBAAmC;CAGnC,gBAA+B;CAC/B;CAEA,YACE,IACA,aACA,WACA,iBACA,mBACA;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,+BAA+B;EACtE,KAAKC,eAAe;EACpB,KAAKC,aAAa;EAClB,KAAKC,mBAAmB;EACxB,KAAKI,qBAAqB;EAC1B,KAAKF,UAAU,aAAa,OAA0B,EACpD,eAAe;GACb,cAAc,KAAKI,kBAAkB;GACrC,KAAKL,OAAO,KAAK,KAAKJ,GAAG;EAC3B,EACF,CAAC;CACH;;;;;CAMA,QAAmC;EACjC,OAAO,KAAKU,kBAAkB,MAAM,wBAAwB;EAC5D,KAAKA,gBAAgB,KAAK,IAAI;EAC9B,KAAKV,IAAI,OAAO,yBAAyB,KAAKG,kBAAkB;EAChE,KAAKM,qBAAqB,YACxB,KAAK,iCACL,iBACF;EACA,OAAO,KAAKJ;CACd;;;;;;;;;CAUA,MAAMM,2BAA6C;EACjD,IAAI;GACF,KAAKC,0BAA0B,MAAM,KAAKC,wBAAwB;GAClE,OAAO;EACT,SAAS,GAAG;GACV,KAAKb,IAAI,OAAO,gCAAgC,KAAKE,cAAc,CAAC;GACpE,OAAO;EACT;CACF;CAGA,kCAA2C,YAAY;EACrD,IAAI;GACF,MAAM,KAAKY,iBAAiB;EAC9B,SAAS,GAAG;GACV,KAAKd,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;EAC1E;EACA,IAAI;GACF,MAAM,KAAKY,2BAA2B;EACxC,SAAS,GAAG;GACV,KAAKf,IAAI,OAAO,4BAA4B,CAAC;EAC/C;EACA,IAAI;GACF,MAAM,KAAKgB,4BAA4B;EACzC,SAAS,GAAG;GACV,KAAKhB,IAAI,OAAO,0CAA0C,CAAC;EAC7D;CACF;CAEA,OAAOiB,mBAIJ;EACD,MAAM,kBAAkB,KAAKd;EAC7B,MAAM,SAAS,KAAKC,OAAO;EAC3B,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,iBAAiB,EAAC,OAAM,CAAC;EAC9C,SAAS,GAAG;GACV,IAAI,OAAO,SAET;GAIF,KAAKJ,IAAI,OAAO,8BAA8B,KAAKG,oBAAoB,CAAC;GACxE;EACF;EACA,IAAI,CAAC,KAAK,IAAI;GACZ,KAAKH,IAAI,OACP,8BAA8B,KAAKG,iBAAiB,IAAI,MAAM,KAAK,KAAK,GAC1E;GACA;EACF;EAEA,MAAM,WAAW,0BAA0B,MAAM,KAAK,KAAK,CAAC;EAC5D,KAAK,MAAM,UAAU,UACnB,IACE,OAAO,SAAS,WAChB,OAAO,SAAS,+BAEhB,KAAK,MAAM,UAAU,OAAO,SAAS;GACnC,MAAM,YAAY,OAAO,QAAQ;GACjC,MAAM,OAAO,OAAO,QAAQ;GAC5B,MAAM,uBAAO,IAAI,KAAK,WAAW,OAAO,KAAK,IAAI,GAAI;GACrD,IAAI,WACF,IAAI;IACF,gBAAgB,SAAS;IACzB,MAAM;KAAC;KAAW;KAAM;IAAI;GAC9B,SAAS,GAAG;IACV,KAAKH,IAAI,OACP,8BAA8B,aAC9B,EAAC,OAAM,GACP,CACF;GACF;EAEJ;CAGN;CAEA,MAAMc,mBAAmB;EACvB,WAAW,MAAM,EAAC,WAAW,MAAM,UAAS,KAAKG,iBAAiB,GAChE,IAAI,YAAY,KAAKC,kBAAkB,CAAC,KAAKZ,YAAY,IAAI,SAAS,GAAG;GACvE,KAAKN,IAAI,OACP,wBAAwB,UAAU,MAAM,KAAA,MAC/B,KAAK,YAAY,EAAE,EAC9B;GACA,KAAKM,YAAY,IAAI,WAAW,IAAI;GACpC,KAAKa,oBAAoB;EAC3B;EAEF,OAAO,KAAKA;CACd;CAEA,MAAMJ,6BAA6B;EACjC,MAAM,YAAY,KAAKK,iBAAiB,KAAK,IAAI,CAAC;EAClD,IAAI,CAAC,WACH;EAEF,MAAM,EAAC,WAAW,cAAc,YAAY,gBAAe;EAU3D,IAAI,CAAC,KAAKC,kBAAkB,WAAW,GACrC,IAAI;GACF,KAAKT,0BAA0B,MAAM,KAAKC,wBAAwB;EACpE,SAAS,GAAG;GACV,KAAKL,eAAe,IAAI,GAAG,EAAC,QAAQ,sBAAqB,CAAC;GAE1D,KAAKR,IAAI,OACP,8EACqB,aAAa,wBACR,YAAY,YAAY,EAAE,IACpD,CACF;GACA;EACF;EAIF,MAAM,aAAa,KAAK,KAAKY,uBAAuB;EACpD,MAAM,iBAAiB,KAAKQ,iBAC1B,WAAW,QAAQ,IAAI,4BACzB;EACA,IAAI,CAAC,gBAAgB;GACnB,KAAKZ,eAAe,IAAI,GAAG,EAAC,QAAQ,eAAc,CAAC;GACnD,MAAM,MAAM,KAAK,IAAI;GACrB,IAAI,KAAKc,sBAAsB,MAC7B,KAAKA,oBAAoB;GAE3B,MAAM,aAAa,MAAM,KAAKA;GAC9B,KAAKtB,IAAI,OACP,8CAA8C,aAAa,0CAEtD,YAAY,YAAY,EAAE,6CACd,KAAKE,WAAW,UAC5B,WAAW,YAAY,EAAE,kCACK,WAAW,KAChD;GACA,IAAI,cAAA,KACF,KAAKqB,wBAAwB,YAAY,aAAa,UAAU;GAElE;EACF;EAGA,KAAKD,oBAAoB;EACzB,MAAM,EAAC,WAAW,mBAAmB,eAAc;EACnD,KAAK,MAAM,aAAa,KAAKhB,YAAY,KAAK,GAC5C,IAAI,aAAa,mBACf,KAAKA,YAAY,OAAO,SAAS;EAGrC,KAAKD,QAAQ,KAAK;GAChB,WAAW;GACX,cAAc,WAAW,QAAQ;EACnC,CAAC;EACD,KAAKa,iBAAiB;CACxB;CAEA,MAAML,0BAAyC;EAC7C,MAAM,QAAQ,YAAY,IAAI;EAC9B,IAAI,SAA8B;EAClC,IAAI;GACF,MAAM,MAAM,MAAM,KAAKN,mBAAmB;GAC1C,SAAS;GACT,OAAO;EACT,UAAU;GACR,qCAAqC,EAAE,SACrC,YAAY,IAAI,IAAI,OACpB;IACE,GAAG,6BACD,KAAKL,YACL,UACA,qBACF;IACA;GACF,CACF;EACF;CACF;;;;;;;;;CAUA,wBACE,YACA,aACA,YACA;EACA,KAAKM,eAAe,IAAI,GAAG,EAAC,QAAQ,gBAAe,CAAC;EACpD,MAAM,MAAM,IAAI,mBACd,aAAa,KAAKN,WAAW,4CACrB,YAAY,YAAY,EAAE,iDACb,WAAW,YAAY,EAAE,wCACxB,WAAW,oBAC5B,yBAAyB,wFAEhC;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;;;;;;CAWA,MAAMW,8BAA8B;EAClC,IAAI,KAAKJ,4BAA4B,QAAQ,KAAKF,kBAAkB,MAClE;EAEF,MAAM,UAAU,KAAK,IAAI,IAAI,KAAKA;EAClC,IAAI,UAAU,KAAKc,sBAAsB,GACvC;EAKF,IAAI,MAAM,KAAKb,yBAAyB,GACtC;EAEF,KAAKc,gCAAgC,OAAO;CAC9C;;;;;;CAOA,wBAAgC;EAC9B,IAAI,QAAQ;EACZ,IAAI;GACF,QAAQ,SAAS,KAAKxB,YAAY,EAAE;EACtC,QAAQ,CAER;EAKA,OAJc,KAAK,IACjB,GACA,KAAK,KAAK,QAAQ,+BAA+B,CAE5C,IAAQ;CACjB;;;;;;;;CASA,gCAAgC,WAAmB;EACjD,MAAM,MAAM,IAAI,mBACd,wCAAwC,KAAKC,WAAW,UACnD,UAAU,+BACV,KAAKsB,sBAAsB,EAAE,kNAGS,KAAKtB,WAAW,EAC7D;EACA,KAAKF,IAAI,QAAQ,IAAI,OAAO;EAC5B,KAAKK,QAAQ,KAAK,GAAG;CACvB;;;;;CAMA,iBACE,QAC8C;EAC9C,IAAI,MAAM;EACV,IAAI;EACJ,KAAK,MAAM,CAAC,WAAW,eAAe,KAAKC,aACzC,IAAI,WAAW,QAAQ,KAAK,UAAU,YAAY,KAAK;GACrD,MAAM;GACN,gBAAgB;EAClB;EAEF,IAAI,CAAC,eACH,OAAO;EAET,OAAO;GAAC,WAAW;GAAK,YAAY;EAAa;CACnD;;;;;;;;CASA,kBAAkB,aAA4B;EAC5C,MAAM,aAAa,KAAKM;EACxB,OACE,eAAe,QACf,YAAY,QAAQ,KAClB,WAAW,QAAQ,IAAA;CAEzB;AACF"}
|