@rocicorp/zero 1.0.1-canary.0 → 1.2.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/_virtual/{_@oxc-project_runtime@0.115.0 → _@oxc-project_runtime@0.122.0}/helpers/usingCtx.js +1 -1
- package/out/replicache/src/mutation-recovery.js +0 -3
- package/out/zero/package.js +7 -6
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +2 -2
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +3 -3
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/services/analyze.js +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js +1 -1
- package/out/zero-cache/src/services/change-source/custom/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +37 -16
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/lsn.js +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +3 -3
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/life-cycle.d.ts.map +1 -1
- package/out/zero-cache/src/services/life-cycle.js +6 -2
- package/out/zero-cache/src/services/life-cycle.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +2 -2
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +10 -14
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.d.ts +4 -2
- package/out/zero-cache/src/services/replicator/replication-status.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replication-status.js +20 -7
- package/out/zero-cache/src/services/replicator/replication-status.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts +2 -2
- package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js +2 -2
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/running-state.d.ts +3 -1
- package/out/zero-cache/src/services/running-state.d.ts.map +1 -1
- package/out/zero-cache/src/services/running-state.js +4 -0
- package/out/zero-cache/src/services/running-state.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/inspect-handler.js +1 -1
- package/out/zero-cache/src/workers/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/workers/replicator.js +18 -3
- package/out/zero-cache/src/workers/replicator.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +15 -5
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts +32 -0
- package/out/zql/src/ivm/cap.d.ts.map +1 -0
- package/out/zql/src/ivm/cap.js +226 -0
- package/out/zql/src/ivm/cap.js.map +1 -0
- package/out/zql/src/ivm/join-utils.d.ts +2 -0
- package/out/zql/src/ivm/join-utils.d.ts.map +1 -1
- package/out/zql/src/ivm/join-utils.js +35 -1
- package/out/zql/src/ivm/join-utils.js.map +1 -1
- package/out/zql/src/ivm/join.d.ts.map +1 -1
- package/out/zql/src/ivm/join.js +6 -2
- package/out/zql/src/ivm/join.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +15 -2
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +69 -8
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/schema.d.ts +1 -1
- package/out/zql/src/ivm/schema.d.ts.map +1 -1
- package/out/zql/src/ivm/skip.d.ts.map +1 -1
- package/out/zql/src/ivm/skip.js +3 -0
- package/out/zql/src/ivm/skip.js.map +1 -1
- package/out/zql/src/ivm/source.d.ts +1 -1
- package/out/zql/src/ivm/source.d.ts.map +1 -1
- package/out/zql/src/ivm/take.d.ts +4 -1
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +4 -2
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +1 -0
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zqlite/src/query-builder.d.ts +1 -1
- package/out/zqlite/src/query-builder.d.ts.map +1 -1
- package/out/zqlite/src/query-builder.js +7 -2
- package/out/zqlite/src/query-builder.js.map +1 -1
- package/out/zqlite/src/table-source.d.ts +1 -1
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +15 -10
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +7 -6
- package/out/replicache/src/mutation-recovery.js.map +0 -1
package/out/zero/package.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
var package_default = {
|
|
2
2
|
name: "@rocicorp/zero",
|
|
3
|
-
version: "1.0
|
|
3
|
+
version: "1.2.0-canary.0",
|
|
4
4
|
description: "Zero is a web framework for serverless web development.",
|
|
5
5
|
author: "Rocicorp, Inc.",
|
|
6
6
|
repository: {
|
|
@@ -77,17 +77,18 @@ var package_default = {
|
|
|
77
77
|
},
|
|
78
78
|
devDependencies: {
|
|
79
79
|
"@op-engineering/op-sqlite": ">=15",
|
|
80
|
-
"@vitest/runner": "4.1.
|
|
80
|
+
"@vitest/runner": "4.1.2",
|
|
81
81
|
"analyze-query": "0.0.0",
|
|
82
82
|
"ast-to-zql": "0.0.0",
|
|
83
83
|
"expo-sqlite": ">=15",
|
|
84
84
|
"replicache": "15.2.1",
|
|
85
85
|
"shared": "0.0.0",
|
|
86
|
+
"syncpack": "^14.2.1",
|
|
86
87
|
"typedoc": "^0.28.17",
|
|
87
88
|
"typedoc-plugin-markdown": "^4.10.0",
|
|
88
89
|
"typescript": "~5.9.3",
|
|
89
|
-
"vite": "8.0.
|
|
90
|
-
"vitest": "4.1.
|
|
90
|
+
"vite": "8.0.3",
|
|
91
|
+
"vitest": "4.1.2",
|
|
91
92
|
"zero-cache": "0.0.0",
|
|
92
93
|
"zero-client": "0.0.0",
|
|
93
94
|
"zero-pg": "0.0.0",
|
|
@@ -98,8 +99,8 @@ var package_default = {
|
|
|
98
99
|
"zqlite": "0.0.0"
|
|
99
100
|
},
|
|
100
101
|
peerDependencies: {
|
|
101
|
-
"
|
|
102
|
-
"
|
|
102
|
+
"@op-engineering/op-sqlite": ">=15",
|
|
103
|
+
"expo-sqlite": ">=15"
|
|
103
104
|
},
|
|
104
105
|
peerDependenciesMeta: {
|
|
105
106
|
"expo-sqlite": { "optional": true },
|
package/out/zero/package.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.0
|
|
1
|
+
{"version":3,"file":"package.js","names":[],"sources":["../../package.json"],"sourcesContent":["{\n \"name\": \"@rocicorp/zero\",\n \"version\": \"1.2.0-canary.0\",\n \"description\": \"Zero is a web framework for serverless web development.\",\n \"author\": \"Rocicorp, Inc.\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/rocicorp/mono.git\",\n \"directory\": \"packages/zero\"\n },\n \"license\": \"Apache-2.0\",\n \"homepage\": \"https://zero.rocicorp.dev\",\n \"bugs\": {\n \"url\": \"https://bugs.rocicorp.dev\"\n },\n \"scripts\": {\n \"build\": \"node --experimental-strip-types --no-warnings tool/build.ts\",\n \"build:watch\": \"node --experimental-strip-types --no-warnings tool/build.ts --watch\",\n \"check-types\": \"tsc -p tsconfig.client.json && tsc -p tsconfig.server.json\",\n \"check-types:client:watch\": \"tsc -p tsconfig.client.json --watch\",\n \"check-types:server:watch\": \"tsc -p tsconfig.server.json --watch\",\n \"format\": \"prettier --write .\",\n \"check-format\": \"prettier --check .\",\n \"lint\": \"oxlint --type-aware src/\",\n \"docs\": \"node --experimental-strip-types --no-warnings tool/generate-docs.ts\",\n \"docs:server\": \"node --watch --experimental-strip-types --no-warnings tool/generate-docs.ts --server\"\n },\n \"dependencies\": {\n \"@badrap/valita\": \"0.3.11\",\n \"@databases/escape-identifier\": \"^1.0.3\",\n \"@databases/sql\": \"^3.3.0\",\n \"@dotenvx/dotenvx\": \"^1.39.0\",\n \"@drdgvhbh/postgres-error-codes\": \"^0.0.6\",\n \"@fastify/cors\": \"^10.0.0\",\n \"@fastify/websocket\": \"^11.0.0\",\n \"@google-cloud/precise-date\": \"^4.0.0\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/api-logs\": \"^0.203.0\",\n \"@opentelemetry/auto-instrumentations-node\": \"^0.62.0\",\n \"@opentelemetry/exporter-metrics-otlp-http\": \"^0.203.0\",\n \"@opentelemetry/resources\": \"^2.0.1\",\n \"@opentelemetry/sdk-metrics\": \"^2.0.1\",\n \"@opentelemetry/sdk-node\": \"^0.203.0\",\n \"@opentelemetry/sdk-trace-node\": \"^2.0.1\",\n \"@postgresql-typed/oids\": \"^0.2.0\",\n \"@rocicorp/lock\": \"^1.0.4\",\n \"@rocicorp/logger\": \"^5.4.0\",\n \"@rocicorp/resolver\": \"^1.0.2\",\n \"@rocicorp/zero-sqlite3\": \"^1.0.15\",\n \"@standard-schema/spec\": \"^1.0.0\",\n \"@types/basic-auth\": \"^1.1.8\",\n \"@types/ws\": \"^8.5.12\",\n \"basic-auth\": \"^2.0.1\",\n \"chalk\": \"^5.3.0\",\n \"chalk-template\": \"^1.1.0\",\n \"chokidar\": \"^4.0.1\",\n \"cloudevents\": \"^10.0.0\",\n \"command-line-args\": \"^6.0.1\",\n \"command-line-usage\": \"^7.0.3\",\n \"compare-utf8\": \"^0.1.1\",\n \"defu\": \"^6.1.4\",\n \"eventemitter3\": \"^5.0.1\",\n \"fastify\": \"^5.0.0\",\n \"is-in-subnet\": \"^4.0.1\",\n \"jose\": \"^5.9.3\",\n \"js-xxhash\": \"^4.0.0\",\n \"json-custom-numbers\": \"^3.1.1\",\n \"kasi\": \"^1.1.0\",\n \"nanoid\": \"^5.1.2\",\n \"parse-prometheus-text-format\": \"^1.1.1\",\n \"pg-format\": \"npm:pg-format-fix@^1.0.5\",\n \"postgres\": \"3.4.7\",\n \"prettier\": \"^3.8.1\",\n \"semver\": \"^7.5.4\",\n \"tsx\": \"^4.21.0\",\n \"url-pattern\": \"^1.0.3\",\n \"urlpattern-polyfill\": \"^10.1.0\",\n \"ws\": \"^8.18.1\"\n },\n \"devDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"@vitest/runner\": \"4.1.2\",\n \"analyze-query\": \"0.0.0\",\n \"ast-to-zql\": \"0.0.0\",\n \"expo-sqlite\": \">=15\",\n \"replicache\": \"15.2.1\",\n \"shared\": \"0.0.0\",\n \"syncpack\": \"^14.2.1\",\n \"typedoc\": \"^0.28.17\",\n \"typedoc-plugin-markdown\": \"^4.10.0\",\n \"typescript\": \"~5.9.3\",\n \"vite\": \"8.0.3\",\n \"vitest\": \"4.1.2\",\n \"zero-cache\": \"0.0.0\",\n \"zero-client\": \"0.0.0\",\n \"zero-pg\": \"0.0.0\",\n \"zero-protocol\": \"0.0.0\",\n \"zero-react\": \"0.0.0\",\n \"zero-server\": \"0.0.0\",\n \"zero-solid\": \"0.0.0\",\n \"zqlite\": \"0.0.0\"\n },\n \"peerDependencies\": {\n \"@op-engineering/op-sqlite\": \">=15\",\n \"expo-sqlite\": \">=15\"\n },\n \"peerDependenciesMeta\": {\n \"expo-sqlite\": {\n \"optional\": true\n },\n \"@op-engineering/op-sqlite\": {\n \"optional\": true\n }\n },\n \"type\": \"module\",\n \"main\": \"out/zero/src/zero.js\",\n \"module\": \"out/zero/src/zero.js\",\n \"types\": \"out/zero/src/zero.d.ts\",\n \"exports\": {\n \".\": {\n \"types\": \"./out/zero/src/zero.d.ts\",\n \"default\": \"./out/zero/src/zero.js\"\n },\n \"./bindings\": {\n \"types\": \"./out/zero/src/bindings.d.ts\",\n \"default\": \"./out/zero/src/bindings.js\"\n },\n \"./change-protocol/v0\": {\n \"types\": \"./out/zero/src/change-protocol/v0.d.ts\",\n \"default\": \"./out/zero/src/change-protocol/v0.js\"\n },\n \"./expo-sqlite\": {\n \"types\": \"./out/zero/src/expo-sqlite.d.ts\",\n \"default\": \"./out/zero/src/expo-sqlite.js\"\n },\n \"./op-sqlite\": {\n \"types\": \"./out/zero/src/op-sqlite.d.ts\",\n \"default\": \"./out/zero/src/op-sqlite.js\"\n },\n \"./pg\": {\n \"types\": \"./out/zero/src/pg.d.ts\",\n \"default\": \"./out/zero/src/pg.js\"\n },\n \"./react\": {\n \"types\": \"./out/zero/src/react.d.ts\",\n \"default\": \"./out/zero/src/react.js\"\n },\n \"./react-native\": {\n \"types\": \"./out/zero/src/react-native.d.ts\",\n \"default\": \"./out/zero/src/react-native.js\"\n },\n \"./server\": {\n \"types\": \"./out/zero/src/server.d.ts\",\n \"default\": \"./out/zero/src/server.js\"\n },\n \"./server/adapters/drizzle\": {\n \"types\": \"./out/zero/src/adapters/drizzle.d.ts\",\n \"default\": \"./out/zero/src/adapters/drizzle.js\"\n },\n \"./server/adapters/prisma\": {\n \"types\": \"./out/zero/src/adapters/prisma.d.ts\",\n \"default\": \"./out/zero/src/adapters/prisma.js\"\n },\n \"./server/adapters/pg\": {\n \"types\": \"./out/zero/src/adapters/pg.d.ts\",\n \"default\": \"./out/zero/src/adapters/pg.js\"\n },\n \"./server/adapters/postgresjs\": {\n \"types\": \"./out/zero/src/adapters/postgresjs.d.ts\",\n \"default\": \"./out/zero/src/adapters/postgresjs.js\"\n },\n \"./solid\": {\n \"types\": \"./out/zero/src/solid.d.ts\",\n \"default\": \"./out/zero/src/solid.js\"\n },\n \"./sqlite\": {\n \"types\": \"./out/zero/src/sqlite.d.ts\",\n \"default\": \"./out/zero/src/sqlite.js\"\n },\n \"./zqlite\": {\n \"types\": \"./out/zero/src/zqlite.d.ts\",\n \"default\": \"./out/zero/src/zqlite.js\"\n }\n },\n \"bin\": {\n \"zero-build-schema\": \"./out/zero/src/build-schema.js\",\n \"zero-cache\": \"./out/zero/src/cli.js\",\n \"zero-cache-dev\": \"./out/zero/src/zero-cache-dev.js\",\n \"zero-deploy-permissions\": \"./out/zero/src/deploy-permissions.js\",\n \"zero-out\": \"./out/zero/src/zero-out.js\",\n \"ast-to-zql\": \"./out/zero/src/ast-to-zql.js\",\n \"analyze-query\": \"./out/zero/src/analyze-query.js\",\n \"transform-query\": \"./out/zero/src/transform-query.js\"\n },\n \"engines\": {\n \"node\": \">=22\"\n },\n \"files\": [\n \"out\",\n \"!*.tsbuildinfo\"\n ]\n}"],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAyBA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"change-streamer.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"names":[],"mappings":"AAyBA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAK/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA8If"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { assert } from "../../../shared/src/asserts.js";
|
|
2
2
|
import { must } from "../../../shared/src/must.js";
|
|
3
|
-
import {
|
|
3
|
+
import { DatabaseInitError } from "../../../zqlite/src/db.js";
|
|
4
4
|
import { parentWorker, singleProcessMode } from "../types/processes.js";
|
|
5
5
|
import { exitAfter, runUntilKilled } from "../services/life-cycle.js";
|
|
6
6
|
import { getShardConfig } from "../types/shards.js";
|
|
@@ -43,7 +43,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
43
43
|
const context = getServerContext(config);
|
|
44
44
|
for (const first of [true, false]) try {
|
|
45
45
|
const { changeSource, subscriptionState } = upstream.type === "pg" ? await initializePostgresChangeSource(lc, upstream.db, shard, replica.file, initialSync, context, replicationLag.reportIntervalMs) : await initializeCustomChangeSource(lc, upstream.db, shard, replica.file, context);
|
|
46
|
-
changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource,
|
|
46
|
+
changeStreamer = await initializeStreamer(lc, shard, taskID, address, protocol, changeDB, changeSource, ReplicationStatusPublisher.forReplicaFile(replica.file), subscriptionState, autoReset ?? false, backPressureLimitHeapProportion, flowControlConsensusPaddingSeconds, setTimeout);
|
|
47
47
|
break;
|
|
48
48
|
} catch (e) {
|
|
49
49
|
if (first && e instanceof AutoResetSignal) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {
|
|
1
|
+
{"version":3,"file":"change-streamer.js","names":[],"sources":["../../../../../zero-cache/src/server/change-streamer.ts"],"sourcesContent":["import {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport {DatabaseInitError} from '../../../zqlite/src/db.ts';\nimport {getServerContext} from '../config/server-context.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {deleteLiteDB} from '../db/delete-lite-db.ts';\nimport {warmupConnections} from '../db/warmup.ts';\nimport {initEventSink, publishCriticalEvent} from '../observability/events.ts';\nimport {initializeCustomChangeSource} from '../services/change-source/custom/change-source.ts';\nimport {initializePostgresChangeSource} from '../services/change-source/pg/change-source.ts';\nimport {BackupMonitor} from '../services/change-streamer/backup-monitor.ts';\nimport {ChangeStreamerHttpServer} from '../services/change-streamer/change-streamer-http.ts';\nimport {initializeStreamer} from '../services/change-streamer/change-streamer-service.ts';\nimport type {ChangeStreamerService} from '../services/change-streamer/change-streamer.ts';\nimport {ReplicaMonitor} from '../services/change-streamer/replica-monitor.ts';\nimport {\n AutoResetSignal,\n CHANGE_STREAMER_APP_NAME,\n} from '../services/change-streamer/schema/tables.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n replicationStatusError,\n ReplicationStatusPublisher,\n} from '../services/replicator/replication-status.ts';\nimport {pgClient} from '../types/pg.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {createLogContext} from './logging.ts';\nimport {startOtelAuto} from './otel-start.ts';\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `parent startMs not specified`);\n const parentStartMs = parseInt(args[0]);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const {\n taskID,\n changeStreamer: {\n port,\n address,\n protocol,\n startupDelayMs,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n },\n upstream,\n change,\n replica,\n initialSync,\n litestream,\n } = config;\n\n startOtelAuto(createLogContext(config, {worker: 'change-streamer'}, false));\n const lc = createLogContext(config, {worker: 'change-streamer'}, true);\n initEventSink(lc, config);\n\n // Kick off DB connection warmup in the background.\n const changeDB = pgClient(\n lc,\n change.db,\n {\n max: change.maxConns,\n connection: {['application_name']: CHANGE_STREAMER_APP_NAME},\n },\n {sendStringAsJson: true},\n );\n void warmupConnections(lc, changeDB, 'change');\n\n const {autoReset, replicationLag} = config;\n const shard = getShardConfig(config);\n\n let changeStreamer: ChangeStreamerService | undefined;\n\n const context = getServerContext(config);\n\n for (const first of [true, false]) {\n try {\n // Note: This performs initial sync of the replica if necessary.\n const {changeSource, subscriptionState} =\n upstream.type === 'pg'\n ? await initializePostgresChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n initialSync,\n context,\n replicationLag.reportIntervalMs,\n )\n : await initializeCustomChangeSource(\n lc,\n upstream.db,\n shard,\n replica.file,\n context,\n );\n\n const replicationStatusPublisher =\n ReplicationStatusPublisher.forReplicaFile(replica.file);\n\n changeStreamer = await initializeStreamer(\n lc,\n shard,\n taskID,\n address,\n protocol,\n changeDB,\n changeSource,\n replicationStatusPublisher,\n subscriptionState,\n autoReset ?? false,\n backPressureLimitHeapProportion,\n flowControlConsensusPaddingSeconds,\n setTimeout,\n );\n break;\n } catch (e) {\n if (first && e instanceof AutoResetSignal) {\n lc.warn?.(`resetting replica ${replica.file}`, e);\n // TODO: Make deleteLiteDB work with litestream. It will probably have to be\n // a semantic wipe instead of a file delete.\n deleteLiteDB(replica.file);\n continue; // execute again with a fresh initial-sync\n }\n await publishCriticalEvent(\n lc,\n replicationStatusError(lc, 'Initializing', e),\n );\n if (e instanceof DatabaseInitError) {\n throw new Error(\n `Cannot open ZERO_REPLICA_FILE at \"${replica.file}\". Please check that the path is valid.`,\n {cause: e},\n );\n }\n throw e;\n }\n }\n // impossible: upstream must have advanced in order for replication to be stuck.\n assert(changeStreamer, `resetting replica did not advance replicaVersion`);\n\n const {backupURL, port: metricsPort} = litestream;\n const monitor = backupURL\n ? new BackupMonitor(\n lc,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n changeStreamer,\n // The time between when the zero-cache was started to when the\n // change-streamer is ready to start serves as the initial delay for\n // watermark cleanup (as it either includes a similar replica\n // restoration/preparation step, or an initial-sync, which\n // generally takes longer).\n //\n // Consider: Also account for permanent volumes?\n Date.now() - parentStartMs,\n )\n : new ReplicaMonitor(lc, replica.file, changeStreamer);\n\n const changeStreamerWebServer = new ChangeStreamerHttpServer(\n lc,\n config,\n {port, startupDelayMs},\n parent,\n changeStreamer,\n monitor instanceof BackupMonitor ? monitor : null,\n );\n\n parent.send(['ready', {ready: true}]);\n\n // Note: The changeStreamer itself is not started here; it is started by the\n // changeStreamerWebServer.\n return runUntilKilled(lc, parent, changeStreamerWebServer, monitor);\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAkCA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;AACf,QAAO,KAAK,SAAS,GAAG,+BAA+B;CACvD,MAAM,gBAAgB,SAAS,KAAK,GAAG;CAEvC,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;CAClE,MAAM,EACJ,QACA,gBAAgB,EACd,MACA,SACA,UACA,gBACA,iCACA,sCAEF,UACA,QACA,SACA,aACA,eACE;AAEJ,eAAc,iBAAiB,QAAQ,EAAC,QAAQ,mBAAkB,EAAE,MAAM,CAAC;CAC3E,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,mBAAkB,EAAE,KAAK;AACtE,eAAc,IAAI,OAAO;CAGzB,MAAM,WAAW,SACf,IACA,OAAO,IACP;EACE,KAAK,OAAO;EACZ,YAAY,GAAE,qBAAqB,0BAAyB;EAC7D,EACD,EAAC,kBAAkB,MAAK,CACzB;AACI,mBAAkB,IAAI,UAAU,SAAS;CAE9C,MAAM,EAAC,WAAW,mBAAkB;CACpC,MAAM,QAAQ,eAAe,OAAO;CAEpC,IAAI;CAEJ,MAAM,UAAU,iBAAiB,OAAO;AAExC,MAAK,MAAM,SAAS,CAAC,MAAM,MAAM,CAC/B,KAAI;EAEF,MAAM,EAAC,cAAc,sBACnB,SAAS,SAAS,OACd,MAAM,+BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,aACA,SACA,eAAe,iBAChB,GACD,MAAM,6BACJ,IACA,SAAS,IACT,OACA,QAAQ,MACR,QACD;AAKP,mBAAiB,MAAM,mBACrB,IACA,OACA,QACA,SACA,UACA,UACA,cATA,2BAA2B,eAAe,QAAQ,KAAK,EAWvD,mBACA,aAAa,OACb,iCACA,oCACA,WACD;AACD;UACO,GAAG;AACV,MAAI,SAAS,aAAa,iBAAiB;AACzC,MAAG,OAAO,qBAAqB,QAAQ,QAAQ,EAAE;AAGjD,gBAAa,QAAQ,KAAK;AAC1B;;AAEF,QAAM,qBACJ,IACA,uBAAuB,IAAI,gBAAgB,EAAE,CAC9C;AACD,MAAI,aAAa,kBACf,OAAM,IAAI,MACR,qCAAqC,QAAQ,KAAK,0CAClD,EAAC,OAAO,GAAE,CACX;AAEH,QAAM;;AAIV,QAAO,gBAAgB,mDAAmD;CAE1E,MAAM,EAAC,WAAW,MAAM,gBAAe;CACvC,MAAM,UAAU,YACZ,IAAI,cACF,IACA,WACA,oBAAoB,YAAY,WAChC,gBAQA,KAAK,KAAK,GAAG,cACd,GACD,IAAI,eAAe,IAAI,QAAQ,MAAM,eAAe;CAExD,MAAM,0BAA0B,IAAI,yBAClC,IACA,QACA;EAAC;EAAM;EAAe,EACtB,QACA,gBACA,mBAAmB,gBAAgB,UAAU,KAC9C;AAED,QAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AAIrC,QAAO,eAAe,IAAI,QAAQ,yBAAyB,QAAQ;;AAIrE,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../zero-cache/src/server/replicator.ts"],"names":[],"mappings":"AAcA,OAAO,EAGL,KAAK,MAAM,EACZ,MAAM,uBAAuB,CAAC;AAU/B,wBAA8B,SAAS,CACrC,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,IAAI,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,IAAI,CAAC,CA8Df"}
|
|
@@ -7,6 +7,7 @@ import { getShardConfig } from "../types/shards.js";
|
|
|
7
7
|
import { getNormalizedZeroConfig } from "../config/zero-config.js";
|
|
8
8
|
import { createLogContext } from "./logging.js";
|
|
9
9
|
import { initEventSink } from "../observability/events.js";
|
|
10
|
+
import { ReplicationStatusPublisher } from "../services/replicator/replication-status.js";
|
|
10
11
|
import { ChangeStreamerHttpClient } from "../services/change-streamer/change-streamer-http.js";
|
|
11
12
|
import { ThreadWriteWorkerClient } from "../services/replicator/write-worker-client.js";
|
|
12
13
|
import { getPragmaConfig, replicaFileModeSchema, setUpMessageHandlers, setupReplica } from "../workers/replicator.js";
|
|
@@ -24,8 +25,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
24
25
|
const workerName = `${mode}-replicator`;
|
|
25
26
|
const lc = createLogContext(config, { worker: workerName });
|
|
26
27
|
initEventSink(lc, config);
|
|
27
|
-
const
|
|
28
|
-
const dbPath = replica.name;
|
|
28
|
+
const dbPath = (await setupReplica(lc, fileMode, config.replica)).name;
|
|
29
29
|
const pragmas = getPragmaConfig(fileMode);
|
|
30
30
|
const workerClient = new ThreadWriteWorkerClient();
|
|
31
31
|
await workerClient.init(dbPath, mode, pragmas, config.log);
|
|
@@ -33,7 +33,7 @@ async function runWorker(parent, env, ...args) {
|
|
|
33
33
|
const shard = getShardConfig(config);
|
|
34
34
|
const { taskID, change, changeStreamer: { port, uri: changeStreamerURI = runningLocalChangeStreamer ? `http://localhost:${port}/` : void 0 } } = config;
|
|
35
35
|
const changeStreamer = new ChangeStreamerHttpClient(lc, shard, change.db, changeStreamerURI);
|
|
36
|
-
const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer,
|
|
36
|
+
const replicator = new ReplicatorService(lc, taskID, `${workerName}-${pid}`, mode, changeStreamer, workerClient, runningLocalChangeStreamer ? ReplicationStatusPublisher.forReplicaFile(dbPath) : null);
|
|
37
37
|
setUpMessageHandlers(lc, replicator, parent);
|
|
38
38
|
const running = runUntilKilled(lc, parent, replicator);
|
|
39
39
|
for await (const _ of replicator.subscribe()) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {pid} from 'node:process';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const workerName = `${mode}-replicator`;\n const lc = createLogContext(config, {worker: workerName});\n initEventSink(lc, config);\n\n const replica = await setupReplica(lc, fileMode, config.replica);\n\n // Create the write worker for async SQLite writes.\n const dbPath = replica.name;\n const pragmas = getPragmaConfig(fileMode);\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log);\n\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n
|
|
1
|
+
{"version":3,"file":"replicator.js","names":[],"sources":["../../../../../zero-cache/src/server/replicator.ts"],"sourcesContent":["import {pid} from 'node:process';\nimport {assert} from '../../../shared/src/asserts.ts';\nimport {must} from '../../../shared/src/must.ts';\nimport * as v from '../../../shared/src/valita.ts';\nimport {getNormalizedZeroConfig} from '../config/zero-config.ts';\nimport {initEventSink} from '../observability/events.ts';\nimport {ChangeStreamerHttpClient} from '../services/change-streamer/change-streamer-http.ts';\nimport {exitAfter, runUntilKilled} from '../services/life-cycle.ts';\nimport {ReplicationStatusPublisher} from '../services/replicator/replication-status.ts';\nimport {\n ReplicatorService,\n type ReplicatorMode,\n} from '../services/replicator/replicator.ts';\nimport {ThreadWriteWorkerClient} from '../services/replicator/write-worker-client.ts';\nimport {\n parentWorker,\n singleProcessMode,\n type Worker,\n} from '../types/processes.ts';\nimport {getShardConfig} from '../types/shards.ts';\nimport {\n getPragmaConfig,\n replicaFileModeSchema,\n setUpMessageHandlers,\n setupReplica,\n} from '../workers/replicator.ts';\nimport {createLogContext} from './logging.ts';\n\nexport default async function runWorker(\n parent: Worker,\n env: NodeJS.ProcessEnv,\n ...args: string[]\n): Promise<void> {\n assert(args.length > 0, `replicator mode not specified`);\n const fileMode = v.parse(args[0], replicaFileModeSchema);\n\n const config = getNormalizedZeroConfig({env, argv: args.slice(1)});\n const mode: ReplicatorMode = fileMode === 'backup' ? 'backup' : 'serving';\n const workerName = `${mode}-replicator`;\n const lc = createLogContext(config, {worker: workerName});\n initEventSink(lc, config);\n\n const replica = await setupReplica(lc, fileMode, config.replica);\n\n // Create the write worker for async SQLite writes.\n const dbPath = replica.name;\n const pragmas = getPragmaConfig(fileMode);\n const workerClient = new ThreadWriteWorkerClient();\n await workerClient.init(dbPath, mode, pragmas, config.log);\n\n const runningLocalChangeStreamer =\n config.changeStreamer.mode === 'dedicated' && !config.changeStreamer.uri;\n const shard = getShardConfig(config);\n const {\n taskID,\n change,\n changeStreamer: {\n port,\n uri: changeStreamerURI = runningLocalChangeStreamer\n ? `http://localhost:${port}/`\n : undefined,\n },\n } = config;\n const changeStreamer = new ChangeStreamerHttpClient(\n lc,\n shard,\n change.db,\n changeStreamerURI,\n );\n\n const replicator = new ReplicatorService(\n lc,\n taskID,\n `${workerName}-${pid}`,\n mode,\n changeStreamer,\n workerClient,\n runningLocalChangeStreamer\n ? // publish ReplicationStatusEvents from backup-replicator only\n ReplicationStatusPublisher.forReplicaFile(dbPath)\n : null,\n );\n\n setUpMessageHandlers(lc, replicator, parent);\n\n const running = runUntilKilled(lc, parent, replicator);\n\n // Signal readiness once the first ReplicaVersionReady notification is received.\n for await (const _ of replicator.subscribe()) {\n parent.send(['ready', {ready: true}]);\n break;\n }\n\n return running;\n}\n\n// fork()\nif (!singleProcessMode()) {\n void exitAfter(() =>\n runWorker(must(parentWorker), process.env, ...process.argv.slice(2)),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;AA4BA,eAA8B,UAC5B,QACA,KACA,GAAG,MACY;AACf,QAAO,KAAK,SAAS,GAAG,gCAAgC;CACxD,MAAM,WAAW,MAAQ,KAAK,IAAI,sBAAsB;CAExD,MAAM,SAAS,wBAAwB;EAAC;EAAK,MAAM,KAAK,MAAM,EAAE;EAAC,CAAC;CAClE,MAAM,OAAuB,aAAa,WAAW,WAAW;CAChE,MAAM,aAAa,GAAG,KAAK;CAC3B,MAAM,KAAK,iBAAiB,QAAQ,EAAC,QAAQ,YAAW,CAAC;AACzD,eAAc,IAAI,OAAO;CAKzB,MAAM,UAHU,MAAM,aAAa,IAAI,UAAU,OAAO,QAAQ,EAGzC;CACvB,MAAM,UAAU,gBAAgB,SAAS;CACzC,MAAM,eAAe,IAAI,yBAAyB;AAClD,OAAM,aAAa,KAAK,QAAQ,MAAM,SAAS,OAAO,IAAI;CAE1D,MAAM,6BACJ,OAAO,eAAe,SAAS,eAAe,CAAC,OAAO,eAAe;CACvE,MAAM,QAAQ,eAAe,OAAO;CACpC,MAAM,EACJ,QACA,QACA,gBAAgB,EACd,MACA,KAAK,oBAAoB,6BACrB,oBAAoB,KAAK,KACzB,KAAA,QAEJ;CACJ,MAAM,iBAAiB,IAAI,yBACzB,IACA,OACA,OAAO,IACP,kBACD;CAED,MAAM,aAAa,IAAI,kBACrB,IACA,QACA,GAAG,WAAW,GAAG,OACjB,MACA,gBACA,cACA,6BAEI,2BAA2B,eAAe,OAAO,GACjD,KACL;AAED,sBAAqB,IAAI,YAAY,OAAO;CAE5C,MAAM,UAAU,eAAe,IAAI,QAAQ,WAAW;AAGtD,YAAW,MAAM,KAAK,WAAW,WAAW,EAAE;AAC5C,SAAO,KAAK,CAAC,SAAS,EAAC,OAAO,MAAK,CAAC,CAAC;AACrC;;AAGF,QAAO;;AAIT,IAAI,CAAC,mBAAmB,CACjB,iBACH,UAAU,KAAK,aAAa,EAAE,QAAQ,KAAK,GAAG,QAAQ,KAAK,MAAM,EAAE,CAAC,CACrE"}
|
|
@@ -7,7 +7,7 @@ import { computeZqlSpecs, mustGetTableSpec } from "../db/lite-tables.js";
|
|
|
7
7
|
import { createSQLiteCostModel } from "../../../zqlite/src/sqlite-cost-model.js";
|
|
8
8
|
import { runAst } from "./run-ast.js";
|
|
9
9
|
import { explainQueries } from "../../../zqlite/src/explain-queries.js";
|
|
10
|
-
import { _usingCtx } from "../../../_virtual/_@oxc-project_runtime@0.
|
|
10
|
+
import { _usingCtx } from "../../../_virtual/_@oxc-project_runtime@0.122.0/helpers/usingCtx.js";
|
|
11
11
|
import { TimeSliceTimer } from "./view-syncer/view-syncer.js";
|
|
12
12
|
//#region ../zero-cache/src/services/analyze.ts
|
|
13
13
|
var TIME_SLICE_LAP_THRESHOLD_MS = 200;
|
|
@@ -95,7 +95,7 @@ async function initialSync(lc, shard, tx, upstreamURI, context) {
|
|
|
95
95
|
const processor = new ChangeProcessor(new StatementRunner(tx), "initial-sync", (_, err) => {
|
|
96
96
|
throw err;
|
|
97
97
|
});
|
|
98
|
-
const statusPublisher =
|
|
98
|
+
const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);
|
|
99
99
|
try {
|
|
100
100
|
let num = 0;
|
|
101
101
|
for await (const change of changes) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = [...shard.publications].sort();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = new ReplicationStatusPublisher(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n [...publications].sort(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,CAAC,GAAG,MAAM,aAAa,CAAC,MAAM;EAChD,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,YAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,IAAI,2BAA2B,GAAG;AAC1D,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,CAAC,GAAG,aAAa,CAAC,MAAM,EACxB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}
|
|
1
|
+
{"version":3,"file":"change-source.js","names":["#lc","#upstreamUri","#shard","#replicationConfig","#startStream"],"sources":["../../../../../../../zero-cache/src/services/change-source/custom/change-source.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {WebSocket} from 'ws';\nimport {assert, unreachable} from '../../../../../shared/src/asserts.ts';\nimport {\n stringify,\n type JSONObject,\n} from '../../../../../shared/src/bigint-json.ts';\nimport {deepEqual} from '../../../../../shared/src/json.ts';\nimport type {SchemaValue} from '../../../../../zero-schema/src/table-schema.ts';\nimport {Database} from '../../../../../zqlite/src/db.ts';\nimport {computeZqlSpecs} from '../../../db/lite-tables.ts';\nimport {StatementRunner} from '../../../db/statements.ts';\nimport type {ShardConfig, ShardID} from '../../../types/shards.ts';\nimport {stream} from '../../../types/streams.ts';\nimport {\n AutoResetSignal,\n type ReplicationConfig,\n} from '../../change-streamer/schema/tables.ts';\nimport {ChangeProcessor} from '../../replicator/change-processor.ts';\nimport {ReplicationStatusPublisher} from '../../replicator/replication-status.ts';\nimport {\n createReplicationStateTables,\n getSubscriptionState,\n initReplicationState,\n type SubscriptionState,\n} from '../../replicator/schema/replication-state.ts';\nimport type {ChangeSource, ChangeStream} from '../change-source.ts';\nimport {initReplica} from '../common/replica-schema.ts';\nimport {changeStreamMessageSchema} from '../protocol/current/downstream.ts';\nimport {\n type BackfillRequest,\n type ChangeSourceUpstream,\n} from '../protocol/current/upstream.ts';\n\n/** Server context to store with the initial sync metadata for debugging. */\nexport type ServerContext = JSONObject;\n\n/**\n * Initializes a Custom change source before streaming changes from the\n * corresponding logical replication stream.\n */\nexport async function initializeCustomChangeSource(\n lc: LogContext,\n upstreamURI: string,\n shard: ShardConfig,\n replicaDbFile: string,\n context: ServerContext,\n): Promise<{subscriptionState: SubscriptionState; changeSource: ChangeSource}> {\n await initReplica(\n lc,\n `replica-${shard.appID}-${shard.shardNum}`,\n replicaDbFile,\n (log, tx) => initialSync(log, shard, tx, upstreamURI, context),\n );\n\n const replica = new Database(lc, replicaDbFile);\n const subscriptionState = getSubscriptionState(new StatementRunner(replica));\n replica.close();\n\n if (shard.publications.length) {\n // Verify that the publications match what has been synced.\n const requested = [...shard.publications].sort();\n const replicated = subscriptionState.publications.sort();\n if (!deepEqual(requested, replicated)) {\n throw new Error(\n `Invalid ShardConfig. Requested publications [${requested}] do not match synced publications: [${replicated}]`,\n );\n }\n }\n\n const changeSource = new CustomChangeSource(\n lc,\n upstreamURI,\n shard,\n subscriptionState,\n );\n\n return {subscriptionState, changeSource};\n}\n\nclass CustomChangeSource implements ChangeSource {\n readonly #lc: LogContext;\n readonly #upstreamUri: string;\n readonly #shard: ShardID;\n readonly #replicationConfig: ReplicationConfig;\n\n constructor(\n lc: LogContext,\n upstreamUri: string,\n shard: ShardID,\n replicationConfig: ReplicationConfig,\n ) {\n this.#lc = lc.withContext('component', 'change-source');\n this.#upstreamUri = upstreamUri;\n this.#shard = shard;\n this.#replicationConfig = replicationConfig;\n }\n\n initialSync(): ChangeStream {\n return this.#startStream();\n }\n\n startLagReporter() {\n return null; // Not supported for custom sources\n }\n\n startStream(\n clientWatermark: string,\n backfillRequests: BackfillRequest[] = [],\n ): Promise<ChangeStream> {\n if (backfillRequests?.length) {\n throw new Error(\n 'backfill is yet not supported for custom change sources',\n );\n }\n return Promise.resolve(this.#startStream(clientWatermark));\n }\n\n #startStream(clientWatermark?: string): ChangeStream {\n const {publications, replicaVersion} = this.#replicationConfig;\n const {appID, shardNum} = this.#shard;\n const url = new URL(this.#upstreamUri);\n url.searchParams.set('appID', appID);\n url.searchParams.set('shardNum', String(shardNum));\n for (const pub of publications) {\n url.searchParams.append('publications', pub);\n }\n if (clientWatermark) {\n assert(\n replicaVersion.length,\n 'replicaVersion is required when clientWatermark is set',\n );\n url.searchParams.set('lastWatermark', clientWatermark);\n url.searchParams.set('replicaVersion', replicaVersion);\n }\n\n const ws = new WebSocket(url);\n const {instream, outstream} = stream(\n this.#lc,\n ws,\n changeStreamMessageSchema,\n // Upstream acks coalesce. If upstream exhibits back-pressure,\n // only the last ACK is kept / buffered.\n {coalesce: (curr: ChangeSourceUpstream) => curr},\n );\n return {changes: instream, acks: outstream};\n }\n}\n\n/**\n * Initial sync for a custom change source makes a request to the\n * change source endpoint with no `replicaVersion` or `lastWatermark`.\n * The initial transaction returned by the endpoint is treated as\n * the initial sync, and the commit watermark of that transaction\n * becomes the `replicaVersion` of the initialized replica.\n *\n * Note that this is equivalent to how the LSN of the Postgres WAL\n * at initial sync time is the `replicaVersion` (and starting\n * version for all initially-synced rows).\n */\nexport async function initialSync(\n lc: LogContext,\n shard: ShardConfig,\n tx: Database,\n upstreamURI: string,\n context: ServerContext,\n) {\n const {appID: id, publications} = shard;\n const changeSource = new CustomChangeSource(lc, upstreamURI, shard, {\n replicaVersion: '', // ignored for initialSync()\n publications,\n });\n const {changes} = changeSource.initialSync();\n\n createReplicationStateTables(tx);\n const processor = new ChangeProcessor(\n new StatementRunner(tx),\n 'initial-sync',\n (_, err) => {\n throw err;\n },\n );\n\n const statusPublisher = ReplicationStatusPublisher.forRunningTransaction(tx);\n try {\n let num = 0;\n for await (const change of changes) {\n const [tag] = change;\n switch (tag) {\n case 'begin': {\n const {commitWatermark} = change[2];\n lc.info?.(\n `initial sync of shard ${id} at replicaVersion ${commitWatermark}`,\n );\n statusPublisher.publish(\n lc,\n 'Initializing',\n `Copying upstream tables at version ${commitWatermark}`,\n 5000,\n );\n initReplicationState(\n tx,\n [...publications].sort(),\n commitWatermark,\n context,\n false,\n );\n processor.processMessage(lc, change);\n break;\n }\n case 'data':\n processor.processMessage(lc, change);\n if (++num % 1000 === 0) {\n lc.debug?.(`processed ${num} changes`);\n }\n break;\n case 'commit':\n processor.processMessage(lc, change);\n validateInitiallySyncedData(lc, tx, shard);\n lc.info?.(`finished initial-sync of ${num} changes`);\n return;\n\n case 'status':\n break; // Ignored\n // @ts-expect-error: falls through if the tag is not 'reset-required\n case 'control': {\n const {tag, message} = change[1];\n if (tag === 'reset-required') {\n throw new AutoResetSignal(\n message ?? 'auto-reset signaled by change source',\n );\n }\n }\n // falls through\n case 'rollback':\n throw new Error(\n `unexpected message during initial-sync: ${stringify(change)}`,\n );\n default:\n unreachable(change);\n }\n }\n throw new Error(\n `change source ${upstreamURI} closed before initial-sync completed`,\n );\n } catch (e) {\n await statusPublisher.publishAndThrowError(lc, 'Initializing', e);\n } finally {\n statusPublisher.stop();\n }\n}\n\n// Verify that the upstream tables expected by the sync logic\n// have been properly initialized.\nfunction getRequiredTables({\n appID,\n shardNum,\n}: ShardID): Record<string, Record<string, SchemaValue>> {\n return {\n [`${appID}_${shardNum}.clients`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n lastMutationID: {type: 'number'},\n userID: {type: 'string'},\n },\n [`${appID}_${shardNum}.mutations`]: {\n clientGroupID: {type: 'string'},\n clientID: {type: 'string'},\n mutationID: {type: 'number'},\n mutation: {type: 'json'},\n },\n [`${appID}.permissions`]: {\n permissions: {type: 'json'},\n hash: {type: 'string'},\n },\n };\n}\n\nfunction validateInitiallySyncedData(\n lc: LogContext,\n db: Database,\n shard: ShardID,\n) {\n const tables = computeZqlSpecs(lc, db, {includeBackfillingColumns: true});\n const required = getRequiredTables(shard);\n for (const [name, columns] of Object.entries(required)) {\n const table = tables.get(name)?.zqlSpec;\n if (!table) {\n throw new Error(\n `Upstream is missing the \"${name}\" table. (Found ${[\n ...tables.keys(),\n ]})` +\n `Please ensure that each table has a unique index over one ` +\n `or more non-null columns.`,\n );\n }\n for (const [col, {type}] of Object.entries(columns)) {\n const found = table[col];\n if (!found) {\n throw new Error(\n `Upstream \"${table}\" table is missing the \"${col}\" column`,\n );\n }\n if (found.type !== type) {\n throw new Error(\n `Upstream \"${table}.${col}\" column is a ${found.type} type but must be a ${type} type.`,\n );\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAyCA,eAAsB,6BACpB,IACA,aACA,OACA,eACA,SAC6E;AAC7E,OAAM,YACJ,IACA,WAAW,MAAM,MAAM,GAAG,MAAM,YAChC,gBACC,KAAK,OAAO,YAAY,KAAK,OAAO,IAAI,aAAa,QAAQ,CAC/D;CAED,MAAM,UAAU,IAAI,SAAS,IAAI,cAAc;CAC/C,MAAM,oBAAoB,qBAAqB,IAAI,gBAAgB,QAAQ,CAAC;AAC5E,SAAQ,OAAO;AAEf,KAAI,MAAM,aAAa,QAAQ;EAE7B,MAAM,YAAY,CAAC,GAAG,MAAM,aAAa,CAAC,MAAM;EAChD,MAAM,aAAa,kBAAkB,aAAa,MAAM;AACxD,MAAI,CAAC,UAAU,WAAW,WAAW,CACnC,OAAM,IAAI,MACR,gDAAgD,UAAU,uCAAuC,WAAW,GAC7G;;AAWL,QAAO;EAAC;EAAmB,cAPN,IAAI,mBACvB,IACA,aACA,OACA,kBACD;EAEuC;;AAG1C,IAAM,qBAAN,MAAiD;CAC/C;CACA;CACA;CACA;CAEA,YACE,IACA,aACA,OACA,mBACA;AACA,QAAA,KAAW,GAAG,YAAY,aAAa,gBAAgB;AACvD,QAAA,cAAoB;AACpB,QAAA,QAAc;AACd,QAAA,oBAA0B;;CAG5B,cAA4B;AAC1B,SAAO,MAAA,aAAmB;;CAG5B,mBAAmB;AACjB,SAAO;;CAGT,YACE,iBACA,mBAAsC,EAAE,EACjB;AACvB,MAAI,kBAAkB,OACpB,OAAM,IAAI,MACR,0DACD;AAEH,SAAO,QAAQ,QAAQ,MAAA,YAAkB,gBAAgB,CAAC;;CAG5D,aAAa,iBAAwC;EACnD,MAAM,EAAC,cAAc,mBAAkB,MAAA;EACvC,MAAM,EAAC,OAAO,aAAY,MAAA;EAC1B,MAAM,MAAM,IAAI,IAAI,MAAA,YAAkB;AACtC,MAAI,aAAa,IAAI,SAAS,MAAM;AACpC,MAAI,aAAa,IAAI,YAAY,OAAO,SAAS,CAAC;AAClD,OAAK,MAAM,OAAO,aAChB,KAAI,aAAa,OAAO,gBAAgB,IAAI;AAE9C,MAAI,iBAAiB;AACnB,UACE,eAAe,QACf,yDACD;AACD,OAAI,aAAa,IAAI,iBAAiB,gBAAgB;AACtD,OAAI,aAAa,IAAI,kBAAkB,eAAe;;EAGxD,MAAM,KAAK,IAAI,YAAU,IAAI;EAC7B,MAAM,EAAC,UAAU,cAAa,OAC5B,MAAA,IACA,IACA,2BAGA,EAAC,WAAW,SAA+B,MAAK,CACjD;AACD,SAAO;GAAC,SAAS;GAAU,MAAM;GAAU;;;;;;;;;;;;;;AAe/C,eAAsB,YACpB,IACA,OACA,IACA,aACA,SACA;CACA,MAAM,EAAC,OAAO,IAAI,iBAAgB;CAKlC,MAAM,EAAC,YAJc,IAAI,mBAAmB,IAAI,aAAa,OAAO;EAClE,gBAAgB;EAChB;EACD,CAAC,CAC6B,aAAa;AAE5C,8BAA6B,GAAG;CAChC,MAAM,YAAY,IAAI,gBACpB,IAAI,gBAAgB,GAAG,EACvB,iBACC,GAAG,QAAQ;AACV,QAAM;GAET;CAED,MAAM,kBAAkB,2BAA2B,sBAAsB,GAAG;AAC5E,KAAI;EACF,IAAI,MAAM;AACV,aAAW,MAAM,UAAU,SAAS;GAClC,MAAM,CAAC,OAAO;AACd,WAAQ,KAAR;IACE,KAAK,SAAS;KACZ,MAAM,EAAC,oBAAmB,OAAO;AACjC,QAAG,OACD,yBAAyB,GAAG,qBAAqB,kBAClD;AACD,qBAAgB,QACd,IACA,gBACA,sCAAsC,mBACtC,IACD;AACD,0BACE,IACA,CAAC,GAAG,aAAa,CAAC,MAAM,EACxB,iBACA,SACA,MACD;AACD,eAAU,eAAe,IAAI,OAAO;AACpC;;IAEF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,SAAI,EAAE,MAAM,QAAS,EACnB,IAAG,QAAQ,aAAa,IAAI,UAAU;AAExC;IACF,KAAK;AACH,eAAU,eAAe,IAAI,OAAO;AACpC,iCAA4B,IAAI,IAAI,MAAM;AAC1C,QAAG,OAAO,4BAA4B,IAAI,UAAU;AACpD;IAEF,KAAK,SACH;IAEF,KAAK,WAAW;KACd,MAAM,EAAC,KAAK,YAAW,OAAO;AAC9B,SAAI,QAAQ,iBACV,OAAM,IAAI,gBACR,WAAW,uCACZ;;IAIL,KAAK,WACH,OAAM,IAAI,MACR,2CAA2C,UAAU,OAAO,GAC7D;IACH,QACE,aAAY,OAAO;;;AAGzB,QAAM,IAAI,MACR,iBAAiB,YAAY,uCAC9B;UACM,GAAG;AACV,QAAM,gBAAgB,qBAAqB,IAAI,gBAAgB,EAAE;WACzD;AACR,kBAAgB,MAAM;;;AAM1B,SAAS,kBAAkB,EACzB,OACA,YACuD;AACvD,QAAO;GACJ,GAAG,MAAM,GAAG,SAAS,YAAY;GAChC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,gBAAgB,EAAC,MAAM,UAAS;GAChC,QAAQ,EAAC,MAAM,UAAS;GACzB;GACA,GAAG,MAAM,GAAG,SAAS,cAAc;GAClC,eAAe,EAAC,MAAM,UAAS;GAC/B,UAAU,EAAC,MAAM,UAAS;GAC1B,YAAY,EAAC,MAAM,UAAS;GAC5B,UAAU,EAAC,MAAM,QAAO;GACzB;GACA,GAAG,MAAM,gBAAgB;GACxB,aAAa,EAAC,MAAM,QAAO;GAC3B,MAAM,EAAC,MAAM,UAAS;GACvB;EACF;;AAGH,SAAS,4BACP,IACA,IACA,OACA;CACA,MAAM,SAAS,gBAAgB,IAAI,IAAI,EAAC,2BAA2B,MAAK,CAAC;CACzE,MAAM,WAAW,kBAAkB,MAAM;AACzC,MAAK,MAAM,CAAC,MAAM,YAAY,OAAO,QAAQ,SAAS,EAAE;EACtD,MAAM,QAAQ,OAAO,IAAI,KAAK,EAAE;AAChC,MAAI,CAAC,MACH,OAAM,IAAI,MACR,4BAA4B,KAAK,kBAAkB,CACjD,GAAG,OAAO,MAAM,CACjB,CAAC,sFAGH;AAEH,OAAK,MAAM,CAAC,KAAK,EAAC,WAAU,OAAO,QAAQ,QAAQ,EAAE;GACnD,MAAM,QAAQ,MAAM;AACpB,OAAI,CAAC,MACH,OAAM,IAAI,MACR,aAAa,MAAM,0BAA0B,IAAI,UAClD;AAEH,OAAI,MAAM,SAAS,KACjB,OAAM,IAAI,MACR,aAAa,MAAM,GAAG,IAAI,gBAAgB,MAAM,KAAK,sBAAsB,KAAK,QACjF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAejD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA2BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAsC7E;
|
|
1
|
+
{"version":3,"file":"change-source.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/change-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAejD,OAAO,KAAK,CAAC,MAAM,qCAAqC,CAAC;AAOzD,OAAO,KAAK,EAGV,kBAAkB,EACnB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,KAAK,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAEL,KAAK,WAAW,EAEjB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,2BAA2B,CAAC;AAEpD,OAAO,EAEL,KAAK,iBAAiB,EAEvB,MAAM,8CAA8C,CAAC;AACtD,OAAO,KAAK,EAAC,YAAY,EAAe,MAAM,qBAAqB,CAAC;AAEpE,OAAO,EAEL,KAAK,QAAQ,EACd,MAAM,wCAAwC,CAAC;AAchD,OAAO,KAAK,EAEV,mBAAmB,EAEpB,MAAM,mCAAmC,CAAC;AAG3C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAGV,eAAe,IAAI,gBAAgB,EACpC,MAAM,yCAAyC,CAAC;AA2BjD;;;;GAIG;AACH,wBAAsB,8BAA8B,CAClD,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,WAAW,EAClB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,kBAAkB,EAC/B,OAAO,EAAE,aAAa,EACtB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,YAAY,CAAA;CAAC,CAAC,CAsC7E;AAqaD,qBAAa,KAAM,YAAW,QAAQ;;gBAIxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;IAI9B,QAAQ,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI;IAgC3C,GAAG,CAAC,SAAS,EAAE,WAAW;CAoB3B;AAED,QAAA,MAAM,eAAe;;;;aAInB,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAitBxD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,gBAAgB,WAwB3E"}
|
|
@@ -12,7 +12,7 @@ import { upstreamSchema } from "../../../types/shards.js";
|
|
|
12
12
|
import { StatementRunner } from "../../../db/statements.js";
|
|
13
13
|
import { pgClient } from "../../../types/pg.js";
|
|
14
14
|
import { majorVersionFromString, majorVersionToString } from "../../../types/state-version.js";
|
|
15
|
-
import { fromBigInt, toStateVersionString } from "./lsn.js";
|
|
15
|
+
import { fromBigInt, toBigInt, toStateVersionString } from "./lsn.js";
|
|
16
16
|
import { UnsupportedColumnDefaultError, mapPostgresToLiteColumn } from "../../../db/pg-to-lite.js";
|
|
17
17
|
import { getSubscriptionStateAndContext } from "../../replicator/schema/replication-state.js";
|
|
18
18
|
import { runTx } from "../../../db/run-transaction.js";
|
|
@@ -103,7 +103,7 @@ var PostgresChangeSource = class {
|
|
|
103
103
|
this.#lagReporter = lagReportIntervalMs ? new LagReporter(lc.withContext("component", "lag-reporter"), shard, this.#db, lagReportIntervalMs) : null;
|
|
104
104
|
}
|
|
105
105
|
startLagReporter() {
|
|
106
|
-
return this.#lagReporter ? this.#lagReporter.initiateLagReport() : null;
|
|
106
|
+
return this.#lagReporter ? this.#lagReporter.initiateLagReport(true) : null;
|
|
107
107
|
}
|
|
108
108
|
async startStream(clientWatermark, backfillRequests = []) {
|
|
109
109
|
const { slot } = this.#replica;
|
|
@@ -126,6 +126,11 @@ var PostgresChangeSource = class {
|
|
|
126
126
|
* non-transactional messages with a downstream status message.
|
|
127
127
|
*/
|
|
128
128
|
const isTransactionalMessage = (lsn, msg) => {
|
|
129
|
+
if (msg.tag === "message" && msg.prefix === this.#lagReporter?.messagePrefix) {
|
|
130
|
+
changes.pushStatus(this.#lagReporter.processLagReport(msg));
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
this.#lagReporter?.checkCurrentLSN(lsn);
|
|
129
134
|
if (msg.tag === "keepalive") {
|
|
130
135
|
changes.pushStatus([
|
|
131
136
|
"status",
|
|
@@ -134,10 +139,6 @@ var PostgresChangeSource = class {
|
|
|
134
139
|
]);
|
|
135
140
|
return false;
|
|
136
141
|
}
|
|
137
|
-
if (msg.tag === "message" && msg.prefix === this.#lagReporter?.messagePrefix) {
|
|
138
|
-
changes.pushStatus(this.#lagReporter.processLagReport(msg));
|
|
139
|
-
return false;
|
|
140
|
-
}
|
|
141
142
|
return true;
|
|
142
143
|
};
|
|
143
144
|
(async () => {
|
|
@@ -311,7 +312,7 @@ var LagReporter = class LagReporter {
|
|
|
311
312
|
#db;
|
|
312
313
|
#lagIntervalMs;
|
|
313
314
|
#pgVersion;
|
|
314
|
-
#
|
|
315
|
+
#expectingLagReport = null;
|
|
315
316
|
#timer;
|
|
316
317
|
constructor(lc, shard, db, lagIntervalMs) {
|
|
317
318
|
this.#lc = lc;
|
|
@@ -327,35 +328,54 @@ var LagReporter = class LagReporter {
|
|
|
327
328
|
}
|
|
328
329
|
return this.#pgVersion;
|
|
329
330
|
}
|
|
330
|
-
async initiateLagReport(
|
|
331
|
+
async initiateLagReport(log = false) {
|
|
331
332
|
const pgVersion = this.#pgVersion ?? await this.#getPgVersion();
|
|
332
|
-
|
|
333
|
-
|
|
333
|
+
const now = Date.now();
|
|
334
|
+
const id = nanoid();
|
|
335
|
+
const lagReport = {
|
|
336
|
+
id,
|
|
337
|
+
lsn: 0n
|
|
338
|
+
};
|
|
339
|
+
this.#expectingLagReport = lagReport;
|
|
340
|
+
let lsn;
|
|
341
|
+
if (pgVersion >= 17e4) [{lsn}] = await this.#db`
|
|
334
342
|
SELECT pg_logical_emit_message(
|
|
335
343
|
false,
|
|
336
344
|
${this.messagePrefix},
|
|
337
345
|
json_build_object(
|
|
338
|
-
'id', ${
|
|
346
|
+
'id', ${id}::text,
|
|
339
347
|
'sendTimeMs', ${now}::int8,
|
|
340
348
|
'commitTimeMs', extract(epoch from now()) * 1000
|
|
341
349
|
)::text,
|
|
342
350
|
true
|
|
343
|
-
);
|
|
351
|
+
) as lsn;
|
|
344
352
|
`;
|
|
345
|
-
else await this.#db`
|
|
353
|
+
else [{lsn}] = await this.#db`
|
|
346
354
|
SELECT pg_logical_emit_message(
|
|
347
355
|
false,
|
|
348
356
|
${this.messagePrefix},
|
|
349
357
|
json_build_object(
|
|
350
|
-
'id', ${
|
|
358
|
+
'id', ${id}::text,
|
|
351
359
|
'sendTimeMs', ${now}::int8,
|
|
352
360
|
'commitTimeMs', extract(epoch from now()) * 1000
|
|
353
361
|
)::text
|
|
354
|
-
);
|
|
362
|
+
) as lsn;
|
|
355
363
|
`;
|
|
364
|
+
lagReport.lsn = toBigInt(lsn);
|
|
365
|
+
if (log) this.#lc.info?.(`initiated lag report at lsn ${lsn}`, {
|
|
366
|
+
id,
|
|
367
|
+
lsn
|
|
368
|
+
});
|
|
356
369
|
return { nextSendTimeMs: now };
|
|
357
370
|
}
|
|
371
|
+
checkCurrentLSN(lsn) {
|
|
372
|
+
if (this.#expectingLagReport?.lsn && lsn > this.#expectingLagReport.lsn) {
|
|
373
|
+
this.#lc.warn?.(`LSN ${fromBigInt(lsn)} is passed expected lag report ${fromBigInt(this.#expectingLagReport.lsn)}. Initiating new report.`);
|
|
374
|
+
this.#scheduleNextReport(0);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
358
377
|
#scheduleNextReport(delayMs) {
|
|
378
|
+
this.#expectingLagReport = null;
|
|
359
379
|
clearTimeout(this.#timer);
|
|
360
380
|
this.#timer = setTimeout(async () => {
|
|
361
381
|
try {
|
|
@@ -371,7 +391,8 @@ var LagReporter = class LagReporter {
|
|
|
371
391
|
const report = parseLogicalMessageContent(msg, lagReportSchema);
|
|
372
392
|
const now = Date.now();
|
|
373
393
|
const nextSendTimeMs = Math.max(now, report.sendTimeMs + this.#lagIntervalMs);
|
|
374
|
-
if (report.id === this.#
|
|
394
|
+
if (report.id === this.#expectingLagReport?.id) this.#scheduleNextReport(nextSendTimeMs - now);
|
|
395
|
+
else this.#lc.debug?.(`received extraneous lag report`, { report });
|
|
375
396
|
const { sendTimeMs, commitTimeMs } = report;
|
|
376
397
|
return [
|
|
377
398
|
"status",
|