@rocicorp/zero 1.11.0-canary.11 → 1.11.0-canary.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/shared/src/ring-buffer.d.ts +2 -0
- package/out/shared/src/ring-buffer.d.ts.map +1 -1
- package/out/shared/src/ring-buffer.js +4 -0
- package/out/shared/src/ring-buffer.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +26 -13
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +45 -6
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +117 -32
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +2 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +10 -17
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/schema/tables.js +15 -4
- package/out/zero-cache/src/services/change-streamer/schema/tables.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-message.d.ts +92 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-message.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-message.js +42 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-message.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts +7 -89
- package/out/zero-cache/src/services/change-streamer/snapshot.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot.js +26 -98
- package/out/zero-cache/src/services/change-streamer/snapshot.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/subscribe.d.ts +358 -0
- package/out/zero-cache/src/services/change-streamer/subscribe.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/subscribe.js +68 -0
- package/out/zero-cache/src/services/change-streamer/subscribe.js.map +1 -0
- package/out/zero-cache/src/services/litestream/commands.d.ts +2 -3
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +3 -2
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts +2 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/incremental-sync.js +13 -7
- package/out/zero-cache/src/services/replicator/incremental-sync.js.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts +2 -1
- package/out/zero-cache/src/services/replicator/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/replicator.js +2 -2
- package/out/zero-cache/src/services/replicator/replicator.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +19 -2
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +8 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts +8 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js +14 -4
- package/out/zero-cache/src/services/view-syncer/cvr-store.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts +6 -2
- package/out/zero-cache/src/services/view-syncer/cvr.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr.js +8 -4
- package/out/zero-cache/src/services/view-syncer/cvr.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js +13 -1
- package/out/zero-cache/src/services/view-syncer/pipeline-driver.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts +21 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/snapshotter.js +14 -6
- package/out/zero-cache/src/services/view-syncer/snapshotter.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +27 -4
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +33 -7
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js +93 -50
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +11 -0
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +56 -35
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/types/ws.js +1 -31
- package/out/zero-cache/src/types/ws.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +5 -5
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-protocol/src/inspect-down.d.ts +81 -81
- package/out/zql/src/builder/builder.d.ts.map +1 -1
- package/out/zql/src/builder/builder.js +5 -3
- package/out/zql/src/builder/builder.js.map +1 -1
- package/out/zql/src/ivm/cap.d.ts +2 -1
- package/out/zql/src/ivm/cap.d.ts.map +1 -1
- package/out/zql/src/ivm/cap.js +3 -0
- package/out/zql/src/ivm/cap.js.map +1 -1
- package/out/zql/src/ivm/exists.d.ts +2 -0
- package/out/zql/src/ivm/exists.d.ts.map +1 -1
- package/out/zql/src/ivm/exists.js +3 -0
- package/out/zql/src/ivm/exists.js.map +1 -1
- package/out/zql/src/ivm/fan-in.d.ts +5 -1
- package/out/zql/src/ivm/fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-in.js +7 -0
- package/out/zql/src/ivm/fan-in.js.map +1 -1
- package/out/zql/src/ivm/fan-out.d.ts +3 -0
- package/out/zql/src/ivm/fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/fan-out.js +4 -0
- package/out/zql/src/ivm/fan-out.js.map +1 -1
- package/out/zql/src/ivm/filter-operators.d.ts +2 -0
- package/out/zql/src/ivm/filter-operators.d.ts.map +1 -1
- package/out/zql/src/ivm/filter-operators.js +9 -0
- package/out/zql/src/ivm/filter-operators.js.map +1 -1
- package/out/zql/src/ivm/filter.d.ts +4 -1
- package/out/zql/src/ivm/filter.d.ts.map +1 -1
- package/out/zql/src/ivm/filter.js +3 -0
- package/out/zql/src/ivm/filter.js.map +1 -1
- package/out/zql/src/ivm/flipped-join.d.ts +3 -2
- package/out/zql/src/ivm/flipped-join.d.ts.map +1 -1
- package/out/zql/src/ivm/flipped-join.js +70 -16
- package/out/zql/src/ivm/flipped-join.js.map +1 -1
- package/out/zql/src/ivm/join-utils.d.ts +24 -7
- package/out/zql/src/ivm/join-utils.d.ts.map +1 -1
- package/out/zql/src/ivm/join-utils.js +87 -4
- package/out/zql/src/ivm/join-utils.js.map +1 -1
- package/out/zql/src/ivm/join.d.ts +3 -3
- package/out/zql/src/ivm/join.d.ts.map +1 -1
- package/out/zql/src/ivm/join.js +33 -58
- package/out/zql/src/ivm/join.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +33 -23
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/operator.d.ts +39 -10
- package/out/zql/src/ivm/operator.d.ts.map +1 -1
- package/out/zql/src/ivm/operator.js +9 -4
- package/out/zql/src/ivm/operator.js.map +1 -1
- package/out/zql/src/ivm/push-accumulated.d.ts +6 -1
- package/out/zql/src/ivm/push-accumulated.d.ts.map +1 -1
- package/out/zql/src/ivm/push-accumulated.js +47 -2
- package/out/zql/src/ivm/push-accumulated.js.map +1 -1
- package/out/zql/src/ivm/skip.d.ts +2 -1
- package/out/zql/src/ivm/skip.d.ts.map +1 -1
- package/out/zql/src/ivm/skip.js +3 -0
- package/out/zql/src/ivm/skip.js.map +1 -1
- package/out/zql/src/ivm/take-gate.d.ts +1 -0
- package/out/zql/src/ivm/take-gate.d.ts.map +1 -1
- package/out/zql/src/ivm/take-gate.js +3 -0
- package/out/zql/src/ivm/take-gate.js.map +1 -1
- package/out/zql/src/ivm/take.d.ts +2 -1
- package/out/zql/src/ivm/take.d.ts.map +1 -1
- package/out/zql/src/ivm/take.js +59 -40
- package/out/zql/src/ivm/take.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts +3 -0
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +56 -5
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zql/src/ivm/union-fan-out.d.ts +2 -1
- package/out/zql/src/ivm/union-fan-out.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-out.js +5 -0
- package/out/zql/src/ivm/union-fan-out.js.map +1 -1
- package/out/zql/src/query/measure-push-operator.d.ts +2 -1
- package/out/zql/src/query/measure-push-operator.d.ts.map +1 -1
- package/out/zql/src/query/measure-push-operator.js +7 -0
- package/out/zql/src/query/measure-push-operator.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAI3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAUhE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,
|
|
1
|
+
{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIrD,OAAO,KAAK,EAAC,UAAU,EAAW,MAAM,kBAAkB,CAAC;AAI3D,OAAO,EAAC,KAAK,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAUhE,OAAO,EAWL,KAAK,cAAc,EACpB,MAAM,cAAc,CAAC;AAEtB,MAAM,MAAM,kBAAkB,GAAG;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,WAAW,GACX,iBAAiB,GACjB,OAAO,CAAC;AAEZ,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,MAAM,EAAE,aAAa,CAAC;CACvB,CAAC;AAKF,qBAAa,uBAAwB,SAAQ,KAAK;IAChD,MAAM,CAAC,QAAQ,CAAC,IAAI,6BAA6B;IAEjD,YAAY,SAAS,EAAE,MAAM,GAAG,SAAS,EAExC;CACF;AAyFD,wBAAsB,UAAU,CAC9B,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,EACnB,kBAAkB,EAAE,kBAAkB,GAAG,SAAS,EAClD,IAAI,EAAE,cAAc,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,cAAc,CAAC,CAwIzB;AAoGD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,YAAY,CAuCd;AAQD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,gBAAgB,EACxB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,IAAI,CAAC,CAgBf;AA8DD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,WAAW,GAAG,KAAK,EAC5B,MAAM,EAAE,MAAM,GACb,IAAI,EAAE,CAoBR"}
|
|
@@ -47,7 +47,7 @@ function getLitestream(mode, config, replicaFile, logLevelOverride) {
|
|
|
47
47
|
}
|
|
48
48
|
};
|
|
49
49
|
}
|
|
50
|
-
async function tryRestore(lc, config, replicaFile, replicaConstraints, role) {
|
|
50
|
+
async function tryRestore(lc, config, replicaFile, replicaConstraints, role, signal) {
|
|
51
51
|
const { backupURL } = config;
|
|
52
52
|
const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);
|
|
53
53
|
let result = "error";
|
|
@@ -76,7 +76,8 @@ async function tryRestore(lc, config, replicaFile, replicaConstraints, role) {
|
|
|
76
76
|
"pipe",
|
|
77
77
|
"pipe"
|
|
78
78
|
],
|
|
79
|
-
windowsHide: true
|
|
79
|
+
windowsHide: true,
|
|
80
|
+
signal
|
|
80
81
|
});
|
|
81
82
|
let stdout = "";
|
|
82
83
|
let stderr = "";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n | 'success'\n | 'no_backup'\n | 'invalid_replica'\n | 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string | undefined;\n result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string | undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' | 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n backupURL,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n // Note: This assumes the default page size. If page size configuration\n // is added, this computation must be adjusted accordingly.\n minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n forceCheckpointThresholdMB,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n snapshotBackupIntervalHoursV5,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) ||\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n const snapshotIntervalHours = v5\n ? snapshotBackupIntervalHoursV5\n : snapshotBackupIntervalHours;\n // Disable truncate-page-n if forced checkpoints are enabled,\n // and otherwise use litestream's default.\n const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n const {backupURL} = config;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const replicaExistedBeforeRestore = existsSync(replicaFile);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n // Output AND capture the stdout and stderr, the former for observability\n // and the latter for error handling.\n proc.stdout.on('data', chunk => {\n process.stdout.write(chunk);\n stdout += chunk;\n });\n proc.stderr.on('data', chunk => {\n process.stderr.write(chunk);\n stderr += chunk;\n });\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints | undefined,\n) {\n let db: Database | undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' | 'error' | 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' | 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 ||\n created === undefined ||\n created === 'created' /* header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,+BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAC7C,MAAM,wBAAwB,KAC1B,gCACA;CAGJ,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,qBACF;IACC,sDAAsD,OACrD,wBAAwB,CAC1B;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {litestreamSocketPath} from './litestream-controller.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n | 'success'\n | 'no_backup'\n | 'invalid_replica'\n | 'error';\n\nexport type RestoreAttempt = {\n restored: boolean;\n backupURL: string | undefined;\n result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string | undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' | 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n backupURL,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n // Note: This assumes the default page size. If page size configuration\n // is added, this computation must be adjusted accordingly.\n minCheckpointPageCount = checkpointThresholdMB * 256, // 1 MiB / 4 KiB page size\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n forceCheckpointThresholdMB,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n snapshotBackupIntervalHoursV5,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) ||\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n const snapshotIntervalHours = v5\n ? snapshotBackupIntervalHoursV5\n : snapshotBackupIntervalHours;\n // Disable truncate-page-n if forced checkpoints are enabled,\n // and otherwise use litestream's default.\n const truncatePageN = forceCheckpointThresholdMB ? -1 : 121359;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n ['ZERO_LITESTREAM_TRUNCATE_PAGE_N']: String(truncatePageN),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['ZERO_LITESTREAM_SOCKET_PATH']: litestreamSocketPath(replicaFile),\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n role: LitestreamRole,\n signal?: AbortSignal,\n): Promise<RestoreAttempt> {\n const {backupURL} = config;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const replicaExistedBeforeRestore = existsSync(replicaFile);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, signal},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n // Output AND capture the stdout and stderr, the former for observability\n // and the latter for error handling.\n proc.stdout.on('data', chunk => {\n process.stdout.write(chunk);\n stdout += chunk;\n });\n proc.stderr.on('data', chunk => {\n process.stderr.write(chunk);\n stderr += chunk;\n });\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints | undefined,\n) {\n let db: Database | undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' | 'error' | 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' | 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 ||\n created === undefined ||\n created === 'created' /* header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAiDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBAGA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,4BACA,kCACA,kCACA,6BACA,+BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAC7C,MAAM,wBAAwB,KAC1B,gCACA;CAGJ,MAAM,gBAAgB,6BAA6B,KAAK;CAExD,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,oCAAoC,OAAO,aAAa;IACxD,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,qBACF;IACC,sDAAsD,OACrD,wBAAwB,CAC1B;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,gCAAgC,qBAAqB,WAAW;IAChE,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACA,QACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;GAAM;EAAM,CACpE;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { LogContext } from '@rocicorp/logger';
|
|
2
2
|
import type { Source } from '../../types/streams.ts';
|
|
3
|
+
import type { ReservationFollowup } from '../change-streamer/change-streamer-http.ts';
|
|
3
4
|
import { type ChangeStreamer } from '../change-streamer/change-streamer.ts';
|
|
4
5
|
import { type ReplicationStatusPublisher } from './replication-status.ts';
|
|
5
6
|
import type { ReplicaState, ReplicatorMode } from './replicator.ts';
|
|
@@ -13,7 +14,7 @@ import type { WriteWorkerClient } from './write-worker-client.ts';
|
|
|
13
14
|
*/
|
|
14
15
|
export declare class IncrementalSyncer {
|
|
15
16
|
#private;
|
|
16
|
-
constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, mode: ReplicatorMode, statusPublisher: ReplicationStatusPublisher | null);
|
|
17
|
+
constructor(lc: LogContext, taskID: string, id: string, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, mode: ReplicatorMode, statusPublisher: ReplicationStatusPublisher | null, initialConnection?: ReservationFollowup);
|
|
17
18
|
run(): Promise<void>;
|
|
18
19
|
subscribe(): Source<ReplicaState>;
|
|
19
20
|
stop(lc: LogContext, err?: unknown): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"incremental-sync.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAQ,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAG1D,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,EAGL,KAAK,cAAc,EACpB,MAAM,uCAAuC,CAAC;AAM/C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,YAAY,EAAE,cAAc,EAAC,MAAM,iBAAiB,CAAC;AAGlE,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAOhE;;;;;;GAMG;AACH,qBAAa,iBAAiB;;IAsB5B,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,IAAI,EAAE,cAAc,EACpB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAClD,iBAAiB,CAAC,EAAE,mBAAmB,EAYxC;IAEK,GAAG,kBA0OR;IA6BD,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAED,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,OAAO,QAIjC;CACF"}
|
|
@@ -26,9 +26,10 @@ var IncrementalSyncer = class {
|
|
|
26
26
|
#statusPublisher;
|
|
27
27
|
#notifier;
|
|
28
28
|
#reporter;
|
|
29
|
+
#initialConnection;
|
|
29
30
|
#state = new RunningState("IncrementalSyncer");
|
|
30
31
|
#replicationEvents = getOrCreateCounter("replication", "events", "Number of replication events processed");
|
|
31
|
-
constructor(lc, taskID, id, changeStreamer, worker, mode, statusPublisher) {
|
|
32
|
+
constructor(lc, taskID, id, changeStreamer, worker, mode, statusPublisher, initialConnection) {
|
|
32
33
|
this.#lc = lc;
|
|
33
34
|
this.#taskID = taskID;
|
|
34
35
|
this.#id = id;
|
|
@@ -38,6 +39,7 @@ var IncrementalSyncer = class {
|
|
|
38
39
|
this.#statusPublisher = statusPublisher;
|
|
39
40
|
this.#notifier = new Notifier();
|
|
40
41
|
this.#reporter = new ReplicationReportRecorder(lc);
|
|
42
|
+
this.#initialConnection = initialConnection;
|
|
41
43
|
}
|
|
42
44
|
async run() {
|
|
43
45
|
const lc = this.#lc;
|
|
@@ -58,16 +60,17 @@ var IncrementalSyncer = class {
|
|
|
58
60
|
let unregister = () => {};
|
|
59
61
|
let err;
|
|
60
62
|
try {
|
|
61
|
-
|
|
62
|
-
protocolVersion:
|
|
63
|
+
const ctx = {
|
|
64
|
+
protocolVersion: 7,
|
|
63
65
|
taskID: this.#taskID,
|
|
64
66
|
id: this.#id,
|
|
65
67
|
mode: this.#mode,
|
|
66
68
|
watermark,
|
|
67
|
-
replicaVersion
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
69
|
+
replicaVersion
|
|
70
|
+
};
|
|
71
|
+
const initialConnection = this.#initialConnection;
|
|
72
|
+
this.#initialConnection = void 0;
|
|
73
|
+
downstream = initialConnection ? await initialConnection.subscribe(ctx) : await this.#changeStreamer.subscribe(ctx);
|
|
71
74
|
unregister = this.#state.cancelOnStop(downstream);
|
|
72
75
|
this.#statusPublisher?.publish(lc, "Replicating", `Replicating from ${watermark}`);
|
|
73
76
|
let backfill;
|
|
@@ -93,6 +96,9 @@ var IncrementalSyncer = class {
|
|
|
93
96
|
this.#state.resetBackoff();
|
|
94
97
|
this.#replicationEvents.add(1);
|
|
95
98
|
switch (message[0]) {
|
|
99
|
+
case "reserved":
|
|
100
|
+
lc.warn?.(`unexpected snapshot reservation message received`, { message: message[0] });
|
|
101
|
+
break;
|
|
96
102
|
case "status": {
|
|
97
103
|
const { lagReport } = message[1];
|
|
98
104
|
if (lagReport) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport {\n errorTypeToReadableName,\n PROTOCOL_VERSION,\n type ChangeStreamer,\n type SizedDownstream,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n IndexingProgress,\n type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #lc: LogContext;\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #statusPublisher: ReplicationStatusPublisher | null;\n readonly #notifier: Notifier;\n readonly #reporter: ReplicationReportRecorder;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n statusPublisher: ReplicationStatusPublisher | null,\n ) {\n this.#lc = lc;\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#worker = worker;\n this.#mode = mode;\n this.#statusPublisher = statusPublisher;\n this.#notifier = new Notifier();\n this.#reporter = new ReplicationReportRecorder(lc);\n }\n\n async run() {\n const lc = this.#lc;\n let workerError: Error | undefined;\n this.#worker.onError(err => {\n workerError ??= err;\n this.#state.stop(lc, err);\n });\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n // This initial notification intentionally omits replicaReadyTimeMs because\n // it represents already-current state, not newly-unserved work.\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: initialWatermark,\n });\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<SizedDownstream> | undefined;\n let unregister = () => {};\n let err: unknown | undefined;\n\n try {\n downstream = await this.#changeStreamer.subscribe({\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n initial: watermark === initialWatermark,\n // The SQLite change log is written by the change-streamer itself, so\n // no replicator logs the change stream any more. The parameter stays\n // on the wire for change-streamers that still exclude a writer from\n // SQLite catchup.\n logsChangeStream: false,\n });\n unregister = this.#state.cancelOnStop(downstream);\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfill:\n | {status: DownloadStatus; table: string; columns: string[]}\n | undefined;\n let writeBatch: ChangeStreamData[] = [];\n let writeBatchSize = 0;\n let inTransaction = false;\n // Indexes created in the current run of consecutive index creations.\n let indexing: IndexingProgress | undefined;\n\n const publishBackfillStatus = (table: string) =>\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${table} table`,\n 3000,\n () =>\n backfill\n ? {\n downloadStatus: [\n {\n ...backfill.status,\n table: backfill.table,\n columns: backfill.columns,\n },\n ],\n }\n : {},\n );\n\n const flushWrites = async () => {\n if (writeBatch.length === 0) {\n return;\n }\n const batch = writeBatch;\n writeBatch = [];\n writeBatchSize = 0;\n\n const result = await this.#worker.processMessages(batch);\n this.#handleResult(lc, result);\n if (result?.completedBackfill) {\n backfill = undefined;\n }\n };\n\n for await (const {data: message, size} of downstream) {\n // Only reset backoff after receiving an actual message.\n this.#state.resetBackoff();\n\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'status': {\n const {lagReport} = message[1];\n if (lagReport) {\n const report: ReplicationReport = {\n nextSendTimeMs: lagReport.nextSendTimeMs,\n };\n if (lagReport.lastTimings) {\n report.lastTimings = {\n ...lagReport.lastTimings,\n replicateTimeMs: Date.now(),\n };\n }\n this.#reporter.record(report);\n }\n break;\n }\n case 'error': {\n // Signal from the replication-manager that the view-syncer must\n // shut down and restore a new backup from litestream.\n const {type, message: msg} = message[1];\n this.stop(\n lc,\n // Note: The AbortError indicates a clean / intentional shutdown.\n new AbortError(\n `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n ),\n );\n break;\n }\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status, relation} = msg;\n const first = !backfill;\n backfill = {\n status, // Update the current status\n table: relation.name,\n columns: [...relation.rowKey.columns, ...msg.columns],\n };\n if (first) {\n // Start publishing the status every 3 seconds.\n publishBackfillStatus(relation.name);\n }\n }\n\n if (msg.tag === 'create-index' && this.#statusPublisher) {\n // Creating an index on an existing table can take a long\n // time. Flush preceding changes so that the index creation\n // is processed (and timed) on its own.\n await flushWrites();\n const index = mapPostgresToLiteIndex(msg.spec);\n indexing ??= new IndexingProgress();\n indexing.start(index);\n this.#statusPublisher.publish(\n lc,\n 'Replicating',\n `Creating index ${index.name} on ${index.tableName}`,\n 3000,\n indexing.state,\n );\n // Exclude the time spent publishing the status.\n indexing.restartTimer();\n writeBatch.push(message as ChangeStreamData);\n writeBatchSize += size;\n await flushWrites();\n const elapsed = indexing.finish();\n lc.info?.(\n `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n );\n this.#statusPublisher.publish(\n lc,\n 'Replicating',\n `Created index ${index.name} on ${index.tableName}`,\n 0,\n indexing.state,\n );\n if (backfill) {\n // Resume reporting the progress of the ongoing backfill.\n publishBackfillStatus(backfill.table);\n }\n break;\n }\n indexing = undefined;\n\n const type = message[0];\n const invalidTransactionSequence =\n type === 'begin' ? inTransaction : !inTransaction;\n if (type === 'begin') {\n inTransaction = true;\n } else if (type === 'commit' || type === 'rollback') {\n inTransaction = false;\n }\n\n writeBatch.push(message as ChangeStreamData);\n writeBatchSize += size;\n if (\n // Waiting here ensures that consuming the commit, which ACKs\n // it upstream, only happens after the SQLite commit finishes.\n type === 'commit' ||\n type === 'rollback' ||\n // Promptly surface malformed transaction sequences instead\n // of leaving a partial batch waiting for another message.\n invalidTransactionSequence ||\n writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n writeBatchSize >= MAX_WORKER_BATCH_SIZE\n ) {\n await flushWrites();\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n this.#statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n if (workerError) {\n throw workerError;\n }\n }\n\n #handleResult(lc: LogContext, result: CommitResult | null) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: result.watermark,\n replicaReadyTimeMs: Date.now(),\n upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n });\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n // Abort any polling loop that the write worker may be in.\n this.#worker.abort();\n }\n}\n"],"mappings":";;;;;;;;;AA6BA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;;;;;;;;AASrC,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;CACnD;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKR;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,aAAa,MAAM,KAAKD,gBAAgB,UAAU;KAChD,iBAAA;KACA,QAAQ,KAAKF;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;KACA,SAAS,cAAc;KAKvB,kBAAkB;IACpB,CAAC;IACD,aAAa,KAAKI,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAKA,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKO,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KAEpD,KAAKF,OAAO,aAAa;KAEzB,KAAKC,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAKF,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAKA,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAKA,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKL,QAAQ,MAAM;CACrB;AACF"}
|
|
1
|
+
{"version":3,"file":"incremental-sync.js","names":["#lc","#taskID","#id","#changeStreamer","#worker","#mode","#statusPublisher","#notifier","#reporter","#state","#replicationEvents","#initialConnection","#handleResult"],"sources":["../../../../../../zero-cache/src/services/replicator/incremental-sync.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {AbortError} from '../../../../shared/src/abort-error.ts';\nimport type {Enum} from '../../../../shared/src/enum.ts';\nimport {mapPostgresToLiteIndex} from '../../db/pg-to-lite.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport type {Sized, Source} from '../../types/streams.ts';\nimport type {DownloadStatus} from '../change-source/protocol/current.ts';\nimport type {ChangeStreamData} from '../change-source/protocol/current/downstream.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport {\n errorTypeToReadableName,\n PROTOCOL_VERSION,\n type ChangeStreamer,\n} from '../change-streamer/change-streamer.ts';\nimport type * as ErrorType from '../change-streamer/error-type-enum.ts';\nimport type {SubscribeDownstream} from '../change-streamer/subscribe.ts';\nimport {RunningState} from '../running-state.ts';\nimport type {CommitResult} from './change-processor.ts';\nimport {Notifier} from './notifier.ts';\nimport {\n IndexingProgress,\n type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {ReplicaState, ReplicatorMode} from './replicator.ts';\nimport {ReplicationReportRecorder} from './reporter/recorder.ts';\nimport type {ReplicationReport} from './reporter/report-schema.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\ntype ErrorType = Enum<typeof ErrorType>;\n\nconst MAX_WORKER_BATCH_MESSAGES = 256;\nconst MAX_WORKER_BATCH_SIZE = 1024 * 1024;\n\n/**\n * The {@link IncrementalSyncer} manages a logical replication stream from upstream,\n * handling application lifecycle events (start, stop) and retrying the\n * connection with exponential backoff. The actual handling of the logical\n * replication messages is done by the {@link ChangeProcessor}, which runs\n * in a worker thread via the {@link WriteWorkerClient}.\n */\nexport class IncrementalSyncer {\n readonly #lc: LogContext;\n readonly #taskID: string;\n readonly #id: string;\n readonly #changeStreamer: ChangeStreamer;\n readonly #worker: WriteWorkerClient;\n readonly #mode: ReplicatorMode;\n readonly #statusPublisher: ReplicationStatusPublisher | null;\n readonly #notifier: Notifier;\n readonly #reporter: ReplicationReportRecorder;\n // Consumed (and cleared) by the first subscribe() in run()'s loop; every\n // subsequent (re)connection goes through #changeStreamer.subscribe().\n #initialConnection: ReservationFollowup | undefined;\n\n readonly #state = new RunningState('IncrementalSyncer');\n\n readonly #replicationEvents = getOrCreateCounter(\n 'replication',\n 'events',\n 'Number of replication events processed',\n );\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n mode: ReplicatorMode,\n statusPublisher: ReplicationStatusPublisher | null,\n initialConnection?: ReservationFollowup,\n ) {\n this.#lc = lc;\n this.#taskID = taskID;\n this.#id = id;\n this.#changeStreamer = changeStreamer;\n this.#worker = worker;\n this.#mode = mode;\n this.#statusPublisher = statusPublisher;\n this.#notifier = new Notifier();\n this.#reporter = new ReplicationReportRecorder(lc);\n this.#initialConnection = initialConnection;\n }\n\n async run() {\n const lc = this.#lc;\n let workerError: Error | undefined;\n this.#worker.onError(err => {\n workerError ??= err;\n this.#state.stop(lc, err);\n });\n lc.info?.(`Starting IncrementalSyncer`);\n const {watermark: initialWatermark} =\n await this.#worker.getSubscriptionState();\n\n // Notify any waiting subscribers that the replica is ready to be read.\n // This initial notification intentionally omits replicaReadyTimeMs because\n // it represents already-current state, not newly-unserved work.\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: initialWatermark,\n });\n\n while (this.#state.shouldRun()) {\n const {replicaVersion, watermark} =\n await this.#worker.getSubscriptionState();\n\n let downstream: Source<Sized<SubscribeDownstream>> | undefined;\n let unregister = () => {};\n let err: unknown | undefined;\n\n try {\n const ctx = {\n protocolVersion: PROTOCOL_VERSION,\n taskID: this.#taskID,\n id: this.#id,\n mode: this.#mode,\n watermark,\n replicaVersion,\n };\n // The first connection reuses the still-open reservation connection\n // (if any) from the backup restore, so the replication-manager that\n // reserved the change log is the one that serves the subscription.\n // Every reconnect after that opens a fresh connection.\n const initialConnection = this.#initialConnection;\n this.#initialConnection = undefined;\n downstream = initialConnection\n ? await initialConnection.subscribe(ctx)\n : await this.#changeStreamer.subscribe(ctx);\n unregister = this.#state.cancelOnStop(downstream);\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Replicating from ${watermark}`,\n );\n\n let backfill:\n | {status: DownloadStatus; table: string; columns: string[]}\n | undefined;\n let writeBatch: ChangeStreamData[] = [];\n let writeBatchSize = 0;\n let inTransaction = false;\n // Indexes created in the current run of consecutive index creations.\n let indexing: IndexingProgress | undefined;\n\n const publishBackfillStatus = (table: string) =>\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilling ${table} table`,\n 3000,\n () =>\n backfill\n ? {\n downloadStatus: [\n {\n ...backfill.status,\n table: backfill.table,\n columns: backfill.columns,\n },\n ],\n }\n : {},\n );\n\n const flushWrites = async () => {\n if (writeBatch.length === 0) {\n return;\n }\n const batch = writeBatch;\n writeBatch = [];\n writeBatchSize = 0;\n\n const result = await this.#worker.processMessages(batch);\n this.#handleResult(lc, result);\n if (result?.completedBackfill) {\n backfill = undefined;\n }\n };\n\n for await (const {data: message, size} of downstream) {\n // Only reset backoff after receiving an actual message.\n this.#state.resetBackoff();\n\n this.#replicationEvents.add(1);\n switch (message[0]) {\n case 'reserved':\n lc.warn?.(`unexpected snapshot reservation message received`, {\n message: message[0],\n });\n break;\n case 'status': {\n const {lagReport} = message[1];\n if (lagReport) {\n const report: ReplicationReport = {\n nextSendTimeMs: lagReport.nextSendTimeMs,\n };\n if (lagReport.lastTimings) {\n report.lastTimings = {\n ...lagReport.lastTimings,\n replicateTimeMs: Date.now(),\n };\n }\n this.#reporter.record(report);\n }\n break;\n }\n case 'error': {\n // Signal from the replication-manager that the view-syncer must\n // shut down and restore a new backup from litestream.\n const {type, message: msg} = message[1];\n this.stop(\n lc,\n // Note: The AbortError indicates a clean / intentional shutdown.\n new AbortError(\n `${errorTypeToReadableName(type as ErrorType)}: ${msg}`,\n ),\n );\n break;\n }\n default: {\n const msg = message[1];\n if (msg.tag === 'backfill' && msg.status) {\n const {status, relation} = msg;\n const first = !backfill;\n backfill = {\n status, // Update the current status\n table: relation.name,\n columns: [...relation.rowKey.columns, ...msg.columns],\n };\n if (first) {\n // Start publishing the status every 3 seconds.\n publishBackfillStatus(relation.name);\n }\n }\n\n if (msg.tag === 'create-index' && this.#statusPublisher) {\n // Creating an index on an existing table can take a long\n // time. Flush preceding changes so that the index creation\n // is processed (and timed) on its own.\n await flushWrites();\n const index = mapPostgresToLiteIndex(msg.spec);\n indexing ??= new IndexingProgress();\n indexing.start(index);\n this.#statusPublisher.publish(\n lc,\n 'Replicating',\n `Creating index ${index.name} on ${index.tableName}`,\n 3000,\n indexing.state,\n );\n // Exclude the time spent publishing the status.\n indexing.restartTimer();\n writeBatch.push(message as ChangeStreamData);\n writeBatchSize += size;\n await flushWrites();\n const elapsed = indexing.finish();\n lc.info?.(\n `Created index ${index.name} (${elapsed.toFixed(3)} ms)`,\n );\n this.#statusPublisher.publish(\n lc,\n 'Replicating',\n `Created index ${index.name} on ${index.tableName}`,\n 0,\n indexing.state,\n );\n if (backfill) {\n // Resume reporting the progress of the ongoing backfill.\n publishBackfillStatus(backfill.table);\n }\n break;\n }\n indexing = undefined;\n\n const type = message[0];\n const invalidTransactionSequence =\n type === 'begin' ? inTransaction : !inTransaction;\n if (type === 'begin') {\n inTransaction = true;\n } else if (type === 'commit' || type === 'rollback') {\n inTransaction = false;\n }\n\n writeBatch.push(message as ChangeStreamData);\n writeBatchSize += size;\n if (\n // Waiting here ensures that consuming the commit, which ACKs\n // it upstream, only happens after the SQLite commit finishes.\n type === 'commit' ||\n type === 'rollback' ||\n // Promptly surface malformed transaction sequences instead\n // of leaving a partial batch waiting for another message.\n invalidTransactionSequence ||\n writeBatch.length >= MAX_WORKER_BATCH_MESSAGES ||\n writeBatchSize >= MAX_WORKER_BATCH_SIZE\n ) {\n await flushWrites();\n }\n break;\n }\n }\n }\n this.#worker.abort();\n } catch (e) {\n err = e;\n this.#worker.abort();\n } finally {\n downstream?.cancel();\n unregister();\n this.#statusPublisher?.stop();\n }\n await this.#state.backoff(lc, err);\n }\n lc.info?.('IncrementalSyncer stopped');\n if (workerError) {\n throw workerError;\n }\n }\n\n #handleResult(lc: LogContext, result: CommitResult | null) {\n if (!result) {\n return;\n }\n if (result.completedBackfill) {\n // Publish the final status\n const status = result.completedBackfill;\n this.#statusPublisher?.publish(\n lc,\n 'Replicating',\n `Backfilled ${status.table} table`,\n 0,\n () => ({downloadStatus: [status]}),\n );\n } else if (result.schemaUpdated) {\n this.#statusPublisher?.publish(lc, 'Replicating', 'Schema updated');\n }\n if (result.watermark && result.changeLogUpdated) {\n void this.#notifier.notifySubscribers({\n state: 'version-ready',\n watermark: result.watermark,\n replicaReadyTimeMs: Date.now(),\n upstreamCommitTimeMs: result.upstreamCommitTimeMs,\n });\n }\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#notifier.subscribe();\n }\n\n stop(lc: LogContext, err?: unknown) {\n this.#state.stop(lc, err);\n // Abort any polling loop that the write worker may be in.\n this.#worker.abort();\n }\n}\n"],"mappings":";;;;;;;;;AA8BA,IAAM,4BAA4B;AAClC,IAAM,wBAAwB,OAAO;;;;;;;;AASrC,IAAa,oBAAb,MAA+B;CAC7B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAGA;CAEA,SAAkB,IAAI,aAAa,mBAAmB;CAEtD,qBAA8B,mBAC5B,eACA,UACA,wCACF;CAEA,YACE,IACA,QACA,IACA,gBACA,QACA,MACA,iBACA,mBACA;EACA,KAAKA,MAAM;EACX,KAAKC,UAAU;EACf,KAAKC,MAAM;EACX,KAAKC,kBAAkB;EACvB,KAAKC,UAAU;EACf,KAAKC,QAAQ;EACb,KAAKC,mBAAmB;EACxB,KAAKC,YAAY,IAAI,SAAS;EAC9B,KAAKC,YAAY,IAAI,0BAA0B,EAAE;EACjD,KAAKG,qBAAqB;CAC5B;CAEA,MAAM,MAAM;EACV,MAAM,KAAK,KAAKX;EAChB,IAAI;EACJ,KAAKI,QAAQ,SAAQ,QAAO;GAC1B,gBAAgB;GAChB,KAAKK,OAAO,KAAK,IAAI,GAAG;EAC1B,CAAC;EACD,GAAG,OAAO,4BAA4B;EACtC,MAAM,EAAC,WAAW,qBAChB,MAAM,KAAKL,QAAQ,qBAAqB;EAK1C,KAAUG,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW;EACb,CAAC;EAED,OAAO,KAAKE,OAAO,UAAU,GAAG;GAC9B,MAAM,EAAC,gBAAgB,cACrB,MAAM,KAAKL,QAAQ,qBAAqB;GAE1C,IAAI;GACJ,IAAI,mBAAmB,CAAC;GACxB,IAAI;GAEJ,IAAI;IACF,MAAM,MAAM;KACV,iBAAA;KACA,QAAQ,KAAKH;KACb,IAAI,KAAKC;KACT,MAAM,KAAKG;KACX;KACA;IACF;IAKA,MAAM,oBAAoB,KAAKM;IAC/B,KAAKA,qBAAqB,KAAA;IAC1B,aAAa,oBACT,MAAM,kBAAkB,UAAU,GAAG,IACrC,MAAM,KAAKR,gBAAgB,UAAU,GAAG;IAC5C,aAAa,KAAKM,OAAO,aAAa,UAAU;IAChD,KAAKH,kBAAkB,QACrB,IACA,eACA,oBAAoB,WACtB;IAEA,IAAI;IAGJ,IAAI,aAAiC,CAAC;IACtC,IAAI,iBAAiB;IACrB,IAAI,gBAAgB;IAEpB,IAAI;IAEJ,MAAM,yBAAyB,UAC7B,KAAKA,kBAAkB,QACrB,IACA,eACA,eAAe,MAAM,SACrB,WAEE,WACI,EACE,gBAAgB,CACd;KACE,GAAG,SAAS;KACZ,OAAO,SAAS;KAChB,SAAS,SAAS;IACpB,CACF,EACF,IACA,CAAC,CACT;IAEF,MAAM,cAAc,YAAY;KAC9B,IAAI,WAAW,WAAW,GACxB;KAEF,MAAM,QAAQ;KACd,aAAa,CAAC;KACd,iBAAiB;KAEjB,MAAM,SAAS,MAAM,KAAKF,QAAQ,gBAAgB,KAAK;KACvD,KAAKQ,cAAc,IAAI,MAAM;KAC7B,IAAI,QAAQ,mBACV,WAAW,KAAA;IAEf;IAEA,WAAW,MAAM,EAAC,MAAM,SAAS,UAAS,YAAY;KAEpD,KAAKH,OAAO,aAAa;KAEzB,KAAKC,mBAAmB,IAAI,CAAC;KAC7B,QAAQ,QAAQ,IAAhB;MACE,KAAK;OACH,GAAG,OAAO,oDAAoD,EAC5D,SAAS,QAAQ,GACnB,CAAC;OACD;MACF,KAAK,UAAU;OACb,MAAM,EAAC,cAAa,QAAQ;OAC5B,IAAI,WAAW;QACb,MAAM,SAA4B,EAChC,gBAAgB,UAAU,eAC5B;QACA,IAAI,UAAU,aACZ,OAAO,cAAc;SACnB,GAAG,UAAU;SACb,iBAAiB,KAAK,IAAI;QAC5B;QAEF,KAAKF,UAAU,OAAO,MAAM;OAC9B;OACA;MACF;MACA,KAAK,SAAS;OAGZ,MAAM,EAAC,MAAM,SAAS,QAAO,QAAQ;OACrC,KAAK,KACH,IAEA,IAAI,WACF,GAAG,wBAAwB,IAAiB,EAAE,IAAI,KACpD,CACF;OACA;MACF;MACA,SAAS;OACP,MAAM,MAAM,QAAQ;OACpB,IAAI,IAAI,QAAQ,cAAc,IAAI,QAAQ;QACxC,MAAM,EAAC,QAAQ,aAAY;QAC3B,MAAM,QAAQ,CAAC;QACf,WAAW;SACT;SACA,OAAO,SAAS;SAChB,SAAS,CAAC,GAAG,SAAS,OAAO,SAAS,GAAG,IAAI,OAAO;QACtD;QACA,IAAI,OAEF,sBAAsB,SAAS,IAAI;OAEvC;OAEA,IAAI,IAAI,QAAQ,kBAAkB,KAAKF,kBAAkB;QAIvD,MAAM,YAAY;QAClB,MAAM,QAAQ,uBAAuB,IAAI,IAAI;QAC7C,aAAa,IAAI,iBAAiB;QAClC,SAAS,MAAM,KAAK;QACpB,KAAKA,iBAAiB,QACpB,IACA,eACA,kBAAkB,MAAM,KAAK,MAAM,MAAM,aACzC,KACA,SAAS,KACX;QAEA,SAAS,aAAa;QACtB,WAAW,KAAK,OAA2B;QAC3C,kBAAkB;QAClB,MAAM,YAAY;QAClB,MAAM,UAAU,SAAS,OAAO;QAChC,GAAG,OACD,iBAAiB,MAAM,KAAK,IAAI,QAAQ,QAAQ,CAAC,EAAE,KACrD;QACA,KAAKA,iBAAiB,QACpB,IACA,eACA,iBAAiB,MAAM,KAAK,MAAM,MAAM,aACxC,GACA,SAAS,KACX;QACA,IAAI,UAEF,sBAAsB,SAAS,KAAK;QAEtC;OACF;OACA,WAAW,KAAA;OAEX,MAAM,OAAO,QAAQ;OACrB,MAAM,6BACJ,SAAS,UAAU,gBAAgB,CAAC;OACtC,IAAI,SAAS,SACX,gBAAgB;YACX,IAAI,SAAS,YAAY,SAAS,YACvC,gBAAgB;OAGlB,WAAW,KAAK,OAA2B;OAC3C,kBAAkB;OAClB,IAGE,SAAS,YACT,SAAS,cAGT,8BACA,WAAW,UAAU,6BACrB,kBAAkB,uBAElB,MAAM,YAAY;OAEpB;MACF;KACF;IACF;IACA,KAAKF,QAAQ,MAAM;GACrB,SAAS,GAAG;IACV,MAAM;IACN,KAAKA,QAAQ,MAAM;GACrB,UAAU;IACR,YAAY,OAAO;IACnB,WAAW;IACX,KAAKE,kBAAkB,KAAK;GAC9B;GACA,MAAM,KAAKG,OAAO,QAAQ,IAAI,GAAG;EACnC;EACA,GAAG,OAAO,2BAA2B;EACrC,IAAI,aACF,MAAM;CAEV;CAEA,cAAc,IAAgB,QAA6B;EACzD,IAAI,CAAC,QACH;EAEF,IAAI,OAAO,mBAAmB;GAE5B,MAAM,SAAS,OAAO;GACtB,KAAKH,kBAAkB,QACrB,IACA,eACA,cAAc,OAAO,MAAM,SAC3B,UACO,EAAC,gBAAgB,CAAC,MAAM,EAAC,EAClC;EACF,OAAO,IAAI,OAAO,eAChB,KAAKA,kBAAkB,QAAQ,IAAI,eAAe,gBAAgB;EAEpE,IAAI,OAAO,aAAa,OAAO,kBAC7B,KAAUC,UAAU,kBAAkB;GACpC,OAAO;GACP,WAAW,OAAO;GAClB,oBAAoB,KAAK,IAAI;GAC7B,sBAAsB,OAAO;EAC/B,CAAC;CAEL;CAEA,YAAkC;EAChC,OAAO,KAAKA,UAAU,UAAU;CAClC;CAEA,KAAK,IAAgB,KAAe;EAClC,KAAKE,OAAO,KAAK,IAAI,GAAG;EAExB,KAAKL,QAAQ,MAAM;CACrB;AACF"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { LogContext } from '@rocicorp/logger';
|
|
2
2
|
import type { ReadonlyJSONObject } from '../../../../shared/src/json.ts';
|
|
3
3
|
import type { Source } from '../../types/streams.ts';
|
|
4
|
+
import type { ReservationFollowup } from '../change-streamer/change-streamer-http.ts';
|
|
4
5
|
import type { ChangeStreamer } from '../change-streamer/change-streamer.ts';
|
|
5
6
|
import type { Service } from '../service.ts';
|
|
6
7
|
import { type ReplicationStatusPublisher } from './replication-status.ts';
|
|
@@ -71,7 +72,7 @@ export type ReplicatorMode = 'backup' | 'serving';
|
|
|
71
72
|
export declare class ReplicatorService implements Replicator, Service {
|
|
72
73
|
#private;
|
|
73
74
|
readonly id: string;
|
|
74
|
-
constructor(lc: LogContext, taskID: string, id: string, mode: ReplicatorMode, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, statusPublisher: ReplicationStatusPublisher | null);
|
|
75
|
+
constructor(lc: LogContext, taskID: string, id: string, mode: ReplicatorMode, changeStreamer: ChangeStreamer, worker: WriteWorkerClient, statusPublisher: ReplicationStatusPublisher | null, initialConnection?: ReservationFollowup);
|
|
75
76
|
status(): Promise<{
|
|
76
77
|
status: string;
|
|
77
78
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,oDAAoD;AACpD,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjD;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAInD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,UAAW,SAAQ,oBAAoB;IACtD;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElD,qBAAa,iBAAkB,YAAW,UAAU,EAAE,OAAO;;IAC3D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAOpB,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,eAAe,EAAE,0BAA0B,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"replicator.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AACnD,OAAO,KAAK,EAAC,mBAAmB,EAAC,MAAM,4CAA4C,CAAC;AACpF,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,eAAe,CAAC;AAE3C,OAAO,EAEL,KAAK,0BAA0B,EAChC,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,0BAA0B,CAAC;AAEhE,oDAAoD;AACpD,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;OAEG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAEjD;;;;;;;;;;;;;OAaG;IACH,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAInD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C,CAAC;AAEF,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,UAAW,SAAQ,oBAAoB;IACtD;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACvC;AAED,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,SAAS,CAAC;AAElD,qBAAa,iBAAkB,YAAW,UAAU,EAAE,OAAO;;IAC3D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAOpB,YACE,EAAE,EAAE,UAAU,EACd,MAAM,EAAE,MAAM,EACd,EAAE,EAAE,MAAM,EACV,IAAI,EAAE,cAAc,EACpB,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,iBAAiB,EACzB,eAAe,EAAE,0BAA0B,GAAG,IAAI,EAKlD,iBAAiB,CAAC,EAAE,mBAAmB,EAmBxC;IAED,MAAM;;OAEL;IAED,GAAG,kBAYF;IAED,SAAS,IAAI,MAAM,CAAC,YAAY,CAAC,CAEhC;IAEK,IAAI,kBAOT;CACF"}
|
|
@@ -8,12 +8,12 @@ var ReplicatorService = class {
|
|
|
8
8
|
#shouldPublishErrors;
|
|
9
9
|
#worker;
|
|
10
10
|
#runPromise;
|
|
11
|
-
constructor(lc, taskID, id, mode, changeStreamer, worker, statusPublisher) {
|
|
11
|
+
constructor(lc, taskID, id, mode, changeStreamer, worker, statusPublisher, initialConnection) {
|
|
12
12
|
this.id = id;
|
|
13
13
|
this.#lc = lc.withContext("component", "replicator").withContext("serviceID", this.id);
|
|
14
14
|
this.#worker = worker;
|
|
15
15
|
this.#shouldPublishErrors = statusPublisher !== null;
|
|
16
|
-
this.#incrementalSyncer = new IncrementalSyncer(this.#lc, taskID, `${taskID}/${id}`, changeStreamer, worker, mode, statusPublisher);
|
|
16
|
+
this.#incrementalSyncer = new IncrementalSyncer(this.#lc, taskID, `${taskID}/${id}`, changeStreamer, worker, mode, statusPublisher, initialConnection);
|
|
17
17
|
}
|
|
18
18
|
status() {
|
|
19
19
|
return Promise.resolve({ status: "ok" });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n publishReplicationError,\n type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n readonly state: 'version-ready';\n /**\n * The replica watermark that became ready. Omitted for the initial\n * \"current state is ready\" notification, which should not count as newly\n * unserved work for a freshly-started ViewSyncer.\n */\n readonly watermark?: string | undefined;\n /**\n * Millisecond epoch when `watermark` became ready to read from the replica.\n */\n readonly replicaReadyTimeMs?: number | undefined;\n\n /**\n * Millisecond epoch at which the transaction behind `watermark` committed\n * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n * which is stamped locally when the change lands in the replica, this is the\n * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n * the change is poked to clients is the end-to-end serving lag.\n *\n * Note that this crosses a clock domain: it comes from the upstream\n * database, whereas it is subtracted from the local clock in the ViewSyncer.\n * The existing `zero.replication.total_lag` metric spans the same boundary.\n *\n * Undefined when the ChangeSource does not report a commit time, and for the\n * initial \"current state is ready\" notification.\n */\n readonly upstreamCommitTimeMs?: number | undefined;\n\n // Used in tests to verify behavior when additional information\n // is ferried in the future. Not set in production.\n readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n /**\n * Creates a cancelable subscription of changes in the replica state.\n *\n * A `version-ready` message indicates that the replica is ready to be\n * read, and henceforth that a _new_ version is ready, i.e. whenever a\n * change is committed to the replica. The `version-ready` message itself\n * otherwise contains no other information; the subscriber queries the\n * replica for the current data.\n *\n * A `maintenance` state indicates that the replica should not be read from.\n * If a subscriber is holding any transaction locks, it should release them\n * until the next `version-ready` signal.\n *\n * Upon subscription, the current state of the replica is sent immediately\n * if known. If multiple notifications occur before the subscriber\n * can consume them, all but the last notification are discarded by the\n * Subscription object (i.e. not buffered). Thus, a subscriber only\n * ever consumes the current (i.e. known) state of the replica. This avoids\n * a buildup of \"work\" if a subscriber is too busy to consume all\n * notifications.\n */\n subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n /**\n * Returns an opaque message for human-readable consumption. This is\n * purely for ensuring that the Replicator has started at least once to\n * bootstrap a new replica.\n */\n status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #incrementalSyncer: IncrementalSyncer;\n readonly #shouldPublishErrors: boolean;\n readonly #worker: WriteWorkerClient;\n #runPromise: Promise<void> | undefined;\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n mode: ReplicatorMode,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n statusPublisher: ReplicationStatusPublisher | null,\n ) {\n this.id = id;\n this.#lc = lc\n .withContext('component', 'replicator')\n .withContext('serviceID', this.id);\n this.#worker = worker;\n this.#shouldPublishErrors = statusPublisher !== null;\n\n this.#incrementalSyncer = new IncrementalSyncer(\n this.#lc,\n taskID,\n `${taskID}/${id}`,\n changeStreamer,\n worker,\n mode,\n statusPublisher,\n );\n }\n\n status() {\n return Promise.resolve({status: 'ok'});\n }\n\n run() {\n this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n if (this.#shouldPublishErrors) {\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n 'Replication stopped because the replica writer failed',\n );\n }\n throw error;\n });\n return this.#runPromise;\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#incrementalSyncer.subscribe();\n }\n\n async stop() {\n this.#incrementalSyncer.stop(this.#lc);\n // Wait for the syncer's run loop to finish so that any in-flight\n // worker.processMessages() call completes and clears #pending\n // before we send the 'stop' message to the worker.\n await this.#runPromise;\n await this.#worker.stop();\n }\n}\n"],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"replicator.js","names":["#lc","#incrementalSyncer","#shouldPublishErrors","#worker","#runPromise"],"sources":["../../../../../../zero-cache/src/services/replicator/replicator.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {ReadonlyJSONObject} from '../../../../shared/src/json.ts';\nimport type {Source} from '../../types/streams.ts';\nimport type {ReservationFollowup} from '../change-streamer/change-streamer-http.ts';\nimport type {ChangeStreamer} from '../change-streamer/change-streamer.ts';\nimport type {Service} from '../service.ts';\nimport {IncrementalSyncer} from './incremental-sync.ts';\nimport {\n publishReplicationError,\n type ReplicationStatusPublisher,\n} from './replication-status.ts';\nimport type {WriteWorkerClient} from './write-worker-client.ts';\n\n/** See {@link ReplicaStateNotifier.subscribe()}. */\nexport type ReplicaState = {\n readonly state: 'version-ready';\n /**\n * The replica watermark that became ready. Omitted for the initial\n * \"current state is ready\" notification, which should not count as newly\n * unserved work for a freshly-started ViewSyncer.\n */\n readonly watermark?: string | undefined;\n /**\n * Millisecond epoch when `watermark` became ready to read from the replica.\n */\n readonly replicaReadyTimeMs?: number | undefined;\n\n /**\n * Millisecond epoch at which the transaction behind `watermark` committed\n * *upstream*, as reported by the ChangeSource. Unlike `replicaReadyTimeMs`,\n * which is stamped locally when the change lands in the replica, this is the\n * origin of the whole pipeline, so `now - upstreamCommitTimeMs` at the point\n * the change is poked to clients is the end-to-end serving lag.\n *\n * Note that this crosses a clock domain: it comes from the upstream\n * database, whereas it is subtracted from the local clock in the ViewSyncer.\n * The existing `zero.replication.total_lag` metric spans the same boundary.\n *\n * Undefined when the ChangeSource does not report a commit time, and for the\n * initial \"current state is ready\" notification.\n */\n readonly upstreamCommitTimeMs?: number | undefined;\n\n // Used in tests to verify behavior when additional information\n // is ferried in the future. Not set in production.\n readonly testSeqNum?: number | undefined;\n};\n\nexport interface ReplicaStateNotifier {\n /**\n * Creates a cancelable subscription of changes in the replica state.\n *\n * A `version-ready` message indicates that the replica is ready to be\n * read, and henceforth that a _new_ version is ready, i.e. whenever a\n * change is committed to the replica. The `version-ready` message itself\n * otherwise contains no other information; the subscriber queries the\n * replica for the current data.\n *\n * A `maintenance` state indicates that the replica should not be read from.\n * If a subscriber is holding any transaction locks, it should release them\n * until the next `version-ready` signal.\n *\n * Upon subscription, the current state of the replica is sent immediately\n * if known. If multiple notifications occur before the subscriber\n * can consume them, all but the last notification are discarded by the\n * Subscription object (i.e. not buffered). Thus, a subscriber only\n * ever consumes the current (i.e. known) state of the replica. This avoids\n * a buildup of \"work\" if a subscriber is too busy to consume all\n * notifications.\n */\n subscribe(): Source<ReplicaState>;\n}\n\nexport interface Replicator extends ReplicaStateNotifier {\n /**\n * Returns an opaque message for human-readable consumption. This is\n * purely for ensuring that the Replicator has started at least once to\n * bootstrap a new replica.\n */\n status(): Promise<ReadonlyJSONObject>;\n}\n\nexport type ReplicatorMode = 'backup' | 'serving';\n\nexport class ReplicatorService implements Replicator, Service {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #incrementalSyncer: IncrementalSyncer;\n readonly #shouldPublishErrors: boolean;\n readonly #worker: WriteWorkerClient;\n #runPromise: Promise<void> | undefined;\n\n constructor(\n lc: LogContext,\n taskID: string,\n id: string,\n mode: ReplicatorMode,\n changeStreamer: ChangeStreamer,\n worker: WriteWorkerClient,\n statusPublisher: ReplicationStatusPublisher | null,\n // The still-open connection from the snapshot reservation that produced\n // the replica this process just restored (serving mode only). Used for\n // IncrementalSyncer's first subscribe() so the replication-manager that\n // reserved the change log is the one that serves the subscription.\n initialConnection?: ReservationFollowup,\n ) {\n this.id = id;\n this.#lc = lc\n .withContext('component', 'replicator')\n .withContext('serviceID', this.id);\n this.#worker = worker;\n this.#shouldPublishErrors = statusPublisher !== null;\n\n this.#incrementalSyncer = new IncrementalSyncer(\n this.#lc,\n taskID,\n `${taskID}/${id}`,\n changeStreamer,\n worker,\n mode,\n statusPublisher,\n initialConnection,\n );\n }\n\n status() {\n return Promise.resolve({status: 'ok'});\n }\n\n run() {\n this.#runPromise = this.#incrementalSyncer.run().catch(async error => {\n if (this.#shouldPublishErrors) {\n await publishReplicationError(\n this.#lc,\n 'Replicating',\n 'Replication stopped because the replica writer failed',\n );\n }\n throw error;\n });\n return this.#runPromise;\n }\n\n subscribe(): Source<ReplicaState> {\n return this.#incrementalSyncer.subscribe();\n }\n\n async stop() {\n this.#incrementalSyncer.stop(this.#lc);\n // Wait for the syncer's run loop to finish so that any in-flight\n // worker.processMessages() call completes and clears #pending\n // before we send the 'stop' message to the worker.\n await this.#runPromise;\n await this.#worker.stop();\n }\n}\n"],"mappings":";;;AAoFA,IAAa,oBAAb,MAA8D;CAC5D;CACA;CACA;CACA;CACA;CACA;CAEA,YACE,IACA,QACA,IACA,MACA,gBACA,QACA,iBAKA,mBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM,GACR,YAAY,aAAa,YAAY,EACrC,YAAY,aAAa,KAAK,EAAE;EACnC,KAAKG,UAAU;EACf,KAAKD,uBAAuB,oBAAoB;EAEhD,KAAKD,qBAAqB,IAAI,kBAC5B,KAAKD,KACL,QACA,GAAG,OAAO,GAAG,MACb,gBACA,QACA,MACA,iBACA,iBACF;CACF;CAEA,SAAS;EACP,OAAO,QAAQ,QAAQ,EAAC,QAAQ,KAAI,CAAC;CACvC;CAEA,MAAM;EACJ,KAAKI,cAAc,KAAKH,mBAAmB,IAAI,EAAE,MAAM,OAAM,UAAS;GACpE,IAAI,KAAKC,sBACP,MAAM,wBACJ,KAAKF,KACL,eACA,uDACF;GAEF,MAAM;EACR,CAAC;EACD,OAAO,KAAKI;CACd;CAEA,YAAkC;EAChC,OAAO,KAAKH,mBAAmB,UAAU;CAC3C;CAEA,MAAM,OAAO;EACX,KAAKA,mBAAmB,KAAK,KAAKD,GAAG;EAIrC,MAAM,KAAKI;EACX,MAAM,KAAKD,QAAQ,KAAK;CAC1B;AACF"}
|
|
@@ -31,8 +31,25 @@ export interface PokeHandler {
|
|
|
31
31
|
cancel(): Promise<void>;
|
|
32
32
|
end(finalVersion: CVRVersion): Promise<void>;
|
|
33
33
|
}
|
|
34
|
+
/** A {@link PokeHandler} for a single client. */
|
|
35
|
+
export interface ClientPokeHandler extends PokeHandler {
|
|
36
|
+
/**
|
|
37
|
+
* Whether a patch has been sent to the client. Once one has, the poke can
|
|
38
|
+
* only be ended with a `finalVersion` that is ahead of the client's base
|
|
39
|
+
* version.
|
|
40
|
+
*/
|
|
41
|
+
readonly patchesSent: boolean;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* A {@link PokeHandler} for multiple clients, as returned by
|
|
45
|
+
* {@link startPoke}.
|
|
46
|
+
*/
|
|
47
|
+
export interface MultiPokeHandler extends PokeHandler {
|
|
48
|
+
/** Whether a patch has been sent to any of the clients. */
|
|
49
|
+
readonly patchesSent: boolean;
|
|
50
|
+
}
|
|
34
51
|
/** Wraps PokeHandlers for multiple clients in a single PokeHandler. */
|
|
35
|
-
export declare function startPoke(lc: LogContext, clients: ClientHandler[], tentativeVersion: CVRVersion):
|
|
52
|
+
export declare function startPoke(lc: LogContext, clients: ClientHandler[], tentativeVersion: CVRVersion): MultiPokeHandler;
|
|
36
53
|
export declare const POKE_PART_FLUSH_THRESHOLD_CHARS: number;
|
|
37
54
|
/**
|
|
38
55
|
* Handles a single `ViewSyncer` connection.
|
|
@@ -45,7 +62,7 @@ export declare class ClientHandler {
|
|
|
45
62
|
version(): NullableCVRVersion;
|
|
46
63
|
fail(e: unknown): void;
|
|
47
64
|
close(reason: string): void;
|
|
48
|
-
startPoke(tentativeVersion: CVRVersion):
|
|
65
|
+
startPoke(tentativeVersion: CVRVersion): ClientPokeHandler;
|
|
49
66
|
sendDeleteClients(lc: LogContext, deletedClientIDs: string[], deletedClientGroupIDs: string[]): Promise<void>;
|
|
50
67
|
sendQueryTransformApplicationErrors(errors: ErroredQuery[]): void;
|
|
51
68
|
sendQueryTransformFailedError(error: TransformFailedBody): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client-handler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/client-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAEL,KAAK,UAAU,IAAI,cAAc,EAClC,MAAM,gCAAgC,CAAC;AAIxC,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,iDAAiD,CAAC;AAIlF,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAanF,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAKL,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,KAAK,EACX,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;IACV,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;CACX,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,cAAc,CAAC;AACpD,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,aAAa,CAAC;AAExD,MAAM,MAAM,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE3C,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,GAAG,CAAC,YAAY,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C;
|
|
1
|
+
{"version":3,"file":"client-handler.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/view-syncer/client-handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAEL,KAAK,UAAU,IAAI,cAAc,EAClC,MAAM,gCAAgC,CAAC;AAIxC,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,iDAAiD,CAAC;AAIlF,OAAO,EAEL,KAAK,mBAAmB,EACzB,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,+CAA+C,CAAC;AAanF,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AACnE,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAKL,KAAK,UAAU,EACf,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,KAAK,EACX,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;IACV,QAAQ,EAAE,UAAU,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,KAAK,CAAC;IACZ,EAAE,EAAE,KAAK,CAAC;IACV,EAAE,EAAE,KAAK,CAAC;CACX,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,cAAc,CAAC;AACpD,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,aAAa,CAAC;AAExD,MAAM,MAAM,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE3C,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,GAAG,CAAC,YAAY,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C;AAED,iDAAiD;AACjD,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD,2DAA2D;IAC3D,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AASD,uEAAuE;AACvE,wBAAgB,SAAS,CACvB,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,aAAa,EAAE,EACxB,gBAAgB,EAAE,UAAU,GAC3B,gBAAgB,CAsClB;AAKD,eAAO,MAAM,+BAA+B,QAAc,CAAC;AAG3D;;GAEG;AACH,qBAAa,aAAa;;IAExB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IA6BtB,YACE,EAAE,EAAE,UAAU,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,UAAU,EAAE,YAAY,CAAC,oBAAoB,CAAC,EAW/C;IAED,OAAO,IAAI,kBAAkB,CAE5B;IAOD,IAAI,CAAC,CAAC,EAAE,OAAO,QAMd;IAED,KAAK,CAAC,MAAM,EAAE,MAAM,QAGnB;IAED,SAAS,CAAC,gBAAgB,EAAE,UAAU,GAAG,iBAAiB,CAqLzD;IAEK,iBAAiB,CACrB,EAAE,EAAE,UAAU,EACd,gBAAgB,EAAE,MAAM,EAAE,EAC1B,qBAAqB,EAAE,MAAM,EAAE,iBAWhC;IAED,mCAAmC,CAAC,MAAM,EAAE,YAAY,EAAE,QAEzD;IAED,6BAA6B,CAAC,KAAK,EAAE,mBAAmB,QAEvD;IAED,mBAAmB,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAMnE;CAuBF;AA0CD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,UAAU,GAAG,cAAc,CAkB9D"}
|
|
@@ -12,6 +12,7 @@ import { cmpVersions, cookieToVersion, versionToCookie, versionToNullableCookie
|
|
|
12
12
|
import { getLogLevel, wrapWithProtocolError } from "../../types/error-with-level.js";
|
|
13
13
|
//#region ../zero-cache/src/services/view-syncer/client-handler.ts
|
|
14
14
|
var NOOP = {
|
|
15
|
+
patchesSent: false,
|
|
15
16
|
addPatch: () => promiseVoid,
|
|
16
17
|
cancel: () => promiseVoid,
|
|
17
18
|
end: () => promiseVoid
|
|
@@ -23,6 +24,9 @@ function startPoke(lc, clients, tentativeVersion) {
|
|
|
23
24
|
for (const result of await Promise.allSettled(results)) if (result.status === "rejected") lc.error?.(`unhandled error poking client (${op})`, result.reason);
|
|
24
25
|
};
|
|
25
26
|
return {
|
|
27
|
+
get patchesSent() {
|
|
28
|
+
return pokers.some((poker) => poker.patchesSent);
|
|
29
|
+
},
|
|
26
30
|
addPatch: (patch) => settle("addPatch", pokers.map((poker) => poker.addPatch(patch))),
|
|
27
31
|
cancel: () => settle("cancel", pokers.map((poker) => poker.cancel())),
|
|
28
32
|
end: (finalVersion) => settle("end", pokers.map((poker) => poker.end(finalVersion)))
|
|
@@ -164,6 +168,9 @@ var ClientHandler = class {
|
|
|
164
168
|
if (serializedRowsLength >= 1048576 || partCount >= PART_COUNT_FLUSH_THRESHOLD) await flushBody();
|
|
165
169
|
};
|
|
166
170
|
return {
|
|
171
|
+
get patchesSent() {
|
|
172
|
+
return pokeStarted;
|
|
173
|
+
},
|
|
167
174
|
addPatch: async (patchToVersion) => {
|
|
168
175
|
try {
|
|
169
176
|
await addPatch(patchToVersion);
|
|
@@ -189,7 +196,7 @@ var ClientHandler = class {
|
|
|
189
196
|
if (!pokeStarted) {
|
|
190
197
|
if (cmpVersions(this.#baseVersion, finalVersion) === 0 && !forceInitialPoke) return;
|
|
191
198
|
await this.#push(["pokeStart", pokeStart]);
|
|
192
|
-
} else if (cmpVersions(this.#baseVersion, finalVersion) >= 0) throw new Error(`Patches were sent but finalVersion ${
|
|
199
|
+
} else if (cmpVersions(this.#baseVersion, finalVersion) >= 0) throw new Error(`Patches were sent but finalVersion ${cookie} is not greater than baseVersion ${versionToNullableCookie(this.#baseVersion)}`);
|
|
193
200
|
await flushBody();
|
|
194
201
|
await this.#push(["pokeEnd", {
|
|
195
202
|
pokeID,
|