@rocicorp/zero 1.10.0-canary.1 → 1.10.0-canary.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/shared/src/promise-race.d.ts +14 -0
- package/out/shared/src/promise-race.d.ts.map +1 -1
- package/out/shared/src/promise-race.js +32 -1
- package/out/shared/src/promise-race.js.map +1 -1
- package/out/zero/package.js +1 -1
- package/out/zero/package.js.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.d.ts.map +1 -1
- package/out/zero-cache/src/auth/write-authorizer.js +8 -1
- package/out/zero-cache/src/auth/write-authorizer.js.map +1 -1
- package/out/zero-cache/src/config/normalize.d.ts +5 -0
- package/out/zero-cache/src/config/normalize.d.ts.map +1 -1
- package/out/zero-cache/src/config/normalize.js +16 -3
- package/out/zero-cache/src/config/normalize.js.map +1 -1
- package/out/zero-cache/src/config/zero-config.d.ts +17 -5
- package/out/zero-cache/src/config/zero-config.d.ts.map +1 -1
- package/out/zero-cache/src/config/zero-config.js +34 -18
- package/out/zero-cache/src/config/zero-config.js.map +1 -1
- package/out/zero-cache/src/db/migration-lite.d.ts.map +1 -1
- package/out/zero-cache/src/db/migration-lite.js +5 -3
- package/out/zero-cache/src/db/migration-lite.js.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.d.ts +5 -15
- package/out/zero-cache/src/db/transaction-pool.d.ts.map +1 -1
- package/out/zero-cache/src/db/transaction-pool.js +39 -73
- package/out/zero-cache/src/db/transaction-pool.js.map +1 -1
- package/out/zero-cache/src/server/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/server/change-streamer.js +25 -14
- package/out/zero-cache/src/server/change-streamer.js.map +1 -1
- package/out/zero-cache/src/server/main.d.ts.map +1 -1
- package/out/zero-cache/src/server/main.js +4 -4
- package/out/zero-cache/src/server/main.js.map +1 -1
- package/out/zero-cache/src/server/replicator.d.ts.map +1 -1
- package/out/zero-cache/src/server/replicator.js +1 -1
- package/out/zero-cache/src/server/replicator.js.map +1 -1
- package/out/zero-cache/src/server/runner/run-worker.js +1 -1
- package/out/zero-cache/src/server/worker-urls.d.ts +0 -1
- package/out/zero-cache/src/server/worker-urls.d.ts.map +1 -1
- package/out/zero-cache/src/server/worker-urls.js +1 -2
- package/out/zero-cache/src/server/worker-urls.js.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/common/replica-schema.js +8 -2
- package/out/zero-cache/src/services/change-source/common/replica-schema.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js +18 -4
- package/out/zero-cache/src/services/change-source/pg/backfill-stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/change-source.js +12 -10
- package/out/zero-cache/src/services/change-source/pg/change-source.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/decommission.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/decommission.js +20 -11
- package/out/zero-cache/src/services/change-source/pg/decommission.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js +51 -74
- package/out/zero-cache/src/services/change-source/pg/initial-sync.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts +20 -0
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js +43 -3
- package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts +24 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js +35 -5
- package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts +1 -4
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js +11 -18
- package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js +8 -3
- package/out/zero-cache/src/services/change-streamer/backup-monitor.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.d.ts +34 -0
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.js +67 -0
- package/out/zero-cache/src/services/change-streamer/change-log-compare-digest.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts +2 -4
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js +8 -2
- package/out/zero-cache/src/services/change-streamer/change-streamer-http.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts +34 -13
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js +164 -35
- package/out/zero-cache/src/services/change-streamer/change-streamer-service.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts +3 -9
- package/out/zero-cache/src/services/change-streamer/change-streamer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/change-streamer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js +1 -1
- package/out/zero-cache/src/services/change-streamer/forwarder.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js +11 -5
- package/out/zero-cache/src/services/change-streamer/litestream3-prometheus-poller.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts +11 -0
- package/out/zero-cache/src/services/change-streamer/shard-sampling.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/shard-sampling.js +19 -0
- package/out/zero-cache/src/services/change-streamer/shard-sampling.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts +11 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js +38 -5
- package/out/zero-cache/src/services/change-streamer/snapshot-reservations.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts +25 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js +95 -12
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-catchup.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts +83 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js +417 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-comparator.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.d.ts +73 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js +157 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-read-router.js.map +1 -0
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js +8 -3
- package/out/zero-cache/src/services/change-streamer/sqlite-change-log-reader.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts +63 -1
- package/out/zero-cache/src/services/change-streamer/storer.d.ts.map +1 -1
- package/out/zero-cache/src/services/change-streamer/storer.js +217 -49
- package/out/zero-cache/src/services/change-streamer/storer.js.map +1 -1
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts +29 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.d.ts.map +1 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js +154 -0
- package/out/zero-cache/src/services/change-streamer/vfs-watermark-poller.js.map +1 -0
- package/out/zero-cache/src/services/http-service.d.ts +1 -0
- package/out/zero-cache/src/services/http-service.d.ts.map +1 -1
- package/out/zero-cache/src/services/http-service.js +7 -3
- package/out/zero-cache/src/services/http-service.js.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/commands.js +8 -4
- package/out/zero-cache/src/services/litestream/commands.js.map +1 -1
- package/out/zero-cache/src/services/litestream/metrics.d.ts +1 -0
- package/out/zero-cache/src/services/litestream/metrics.d.ts.map +1 -1
- package/out/zero-cache/src/services/litestream/metrics.js +4 -1
- package/out/zero-cache/src/services/litestream/metrics.js.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.d.ts.map +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js +1 -1
- package/out/zero-cache/src/services/mutagen/mutagen.js.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts +10 -0
- package/out/zero-cache/src/services/replicator/change-log-db.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/change-log-db.js +15 -1
- package/out/zero-cache/src/services/replicator/change-log-db.js.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts +3 -2
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.d.ts.map +1 -1
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js +8 -6
- package/out/zero-cache/src/services/replicator/sqlite-change-log-observability.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts +3 -2
- package/out/zero-cache/src/services/view-syncer/client-handler.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/client-handler.js +25 -17
- package/out/zero-cache/src/services/view-syncer/client-handler.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js +14 -1
- package/out/zero-cache/src/services/view-syncer/cvr-store.js.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts +3 -3
- package/out/zero-cache/src/services/view-syncer/view-syncer.d.ts.map +1 -1
- package/out/zero-cache/src/services/view-syncer/view-syncer.js +6 -2
- package/out/zero-cache/src/services/view-syncer/view-syncer.js.map +1 -1
- package/out/zero-cache/src/types/downstream.d.ts +11 -0
- package/out/zero-cache/src/types/downstream.d.ts.map +1 -0
- package/out/zero-cache/src/types/pg.d.ts.map +1 -1
- package/out/zero-cache/src/types/pg.js +14 -4
- package/out/zero-cache/src/types/pg.js.map +1 -1
- package/out/zero-cache/src/types/poke-chunk.d.ts +18 -0
- package/out/zero-cache/src/types/poke-chunk.d.ts.map +1 -0
- package/out/zero-cache/src/types/poke-chunk.js +57 -0
- package/out/zero-cache/src/types/poke-chunk.js.map +1 -0
- package/out/zero-cache/src/types/processes.js +1 -4
- package/out/zero-cache/src/types/processes.js.map +1 -1
- package/out/zero-cache/src/types/streams.d.ts +4 -9
- package/out/zero-cache/src/types/streams.d.ts.map +1 -1
- package/out/zero-cache/src/types/streams.js.map +1 -1
- package/out/zero-cache/src/types/subscription.d.ts +2 -11
- package/out/zero-cache/src/types/subscription.d.ts.map +1 -1
- package/out/zero-cache/src/types/subscription.js +4 -25
- package/out/zero-cache/src/types/subscription.js.map +1 -1
- package/out/zero-cache/src/workers/connection.d.ts +22 -4
- package/out/zero-cache/src/workers/connection.d.ts.map +1 -1
- package/out/zero-cache/src/workers/connection.js +112 -14
- package/out/zero-cache/src/workers/connection.js.map +1 -1
- package/out/zero-client/src/client/version.js +1 -1
- package/out/zero-client/src/client/zero-poke-handler.d.ts +7 -2
- package/out/zero-client/src/client/zero-poke-handler.d.ts.map +1 -1
- package/out/zero-client/src/client/zero-poke-handler.js +52 -2
- package/out/zero-client/src/client/zero-poke-handler.js.map +1 -1
- package/out/zero-client/src/client/zero.d.ts.map +1 -1
- package/out/zero-client/src/client/zero.js +38 -6
- package/out/zero-client/src/client/zero.js.map +1 -1
- package/out/zero-protocol/src/down.d.ts +1 -1
- package/out/zero-protocol/src/poke.d.ts +89 -7
- package/out/zero-protocol/src/poke.d.ts.map +1 -1
- package/out/zero-protocol/src/poke.js +16 -8
- package/out/zero-protocol/src/poke.js.map +1 -1
- package/out/zero-protocol/src/protocol-version.d.ts +12 -7
- package/out/zero-protocol/src/protocol-version.d.ts.map +1 -1
- package/out/zero-protocol/src/protocol-version.js.map +1 -1
- package/out/zql/src/ivm/memory-source.d.ts +10 -2
- package/out/zql/src/ivm/memory-source.d.ts.map +1 -1
- package/out/zql/src/ivm/memory-source.js +14 -6
- package/out/zql/src/ivm/memory-source.js.map +1 -1
- package/out/zql/src/ivm/stream.d.ts +0 -1
- package/out/zql/src/ivm/stream.d.ts.map +1 -1
- package/out/zql/src/ivm/stream.js +1 -7
- package/out/zql/src/ivm/stream.js.map +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts +1 -1
- package/out/zql/src/ivm/union-fan-in.d.ts.map +1 -1
- package/out/zql/src/ivm/union-fan-in.js +11 -2
- package/out/zql/src/ivm/union-fan-in.js.map +1 -1
- package/out/zqlite/src/db.js +2 -2
- package/out/zqlite/src/sqlite-cost-model.js +5 -5
- package/out/zqlite/src/table-source.d.ts.map +1 -1
- package/out/zqlite/src/table-source.js +4 -4
- package/out/zqlite/src/table-source.js.map +1 -1
- package/package.json +1 -1
- package/out/zero-cache/src/server/backup-watermark-poller.js +0 -31
- package/out/zero-cache/src/server/backup-watermark-poller.js.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts +0 -57
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.d.ts.map +0 -1
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js +0 -137
- package/out/zero-cache/src/services/litestream/vfs-watermark-poller.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n | 'success'\n | 'no_backup'\n | 'invalid_replica'\n | 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string | undefined;\n result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string | undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' | 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n backupURL,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) ||\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n const {backupURL} = config;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const replicaExistedBeforeRestore = existsSync(replicaFile);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.stderr.on('data', chunk => (stderr += chunk));\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n // A successful restore does not emit ERROR-level output; forward\n // litestream's own (minimal, debug-level) restore logging to the pod's\n // stdout/stderr, matching the previous `stdio: 'inherit'` behavior.\n if (stdout) {\n process.stdout.write(stdout);\n }\n if (stderr) {\n process.stderr.write(stderr);\n }\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints | undefined,\n) {\n let db: Database | undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' | 'error' | 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' | 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 ||\n created === undefined ||\n created === 'created' /* header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;AAgDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAE7C,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;EACjD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;GAIA,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;GAE7B,IAAI,QACF,QAAQ,OAAO,MAAM,MAAM;EAE/B,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"commands.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/commands.ts"],"sourcesContent":["import type {ChildProcess} from 'node:child_process';\nimport {spawn} from 'node:child_process';\nimport {existsSync, readdirSync, rmSync, statSync} from 'node:fs';\nimport {basename, dirname, join} from 'node:path';\nimport type {LogContext, LogLevel} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport {must} from '../../../../shared/src/must.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {type LitestreamConfig} from '../../config/normalize.ts';\nimport {deleteLiteDB} from '../../db/delete-lite-db.ts';\nimport {\n isSQLiteCorruption,\n logSQLiteCorruptionDiagnostics,\n} from '../../db/sqlite-corruption.ts';\nimport {StatementRunner} from '../../db/statements.ts';\nimport {deleteChangeLogDB} from '../replicator/change-log-db.ts';\nimport {getSubscriptionState} from '../replicator/schema/replication-state.ts';\nimport {\n litestreamBackupListDuration,\n litestreamBackupMetricAttrs,\n litestreamBackupProcessDuration,\n litestreamBackupProcessMetricAttrs,\n litestreamBackupProcessRuns,\n litestreamRestoreAttempts,\n litestreamRestoredDbBytes,\n litestreamRestoreMetricAttrs,\n litestreamRestoreProcessDuration,\n litestreamRestoreValidationDuration,\n type LitestreamRole,\n} from './metrics.ts';\n\nexport type ReplicaConstraints = {\n replicaVersion: string;\n minWatermark: string;\n};\n\nexport type RestoreResult =\n | 'success'\n | 'no_backup'\n | 'invalid_replica'\n | 'error';\n\ntype RestoreAttempt = {\n restored: boolean;\n backupURL: string | undefined;\n result: RestoreResult;\n};\n\nconst MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES = 100;\nconst LEGACY_RESTORE_WAL_INDEX = /^[0-9a-f]{8}$/;\n\nexport class BackupNotFoundException extends Error {\n static readonly name = 'BackupNotFoundException';\n\n constructor(backupURL: string | undefined) {\n super(`backup not found at ${backupURL}`);\n }\n}\n\nfunction getLitestream(\n mode: 'restore' | 'replicate',\n config: LitestreamConfig,\n replicaFile: string,\n logLevelOverride?: LogLevel,\n): {\n litestream: string;\n env: NodeJS.ProcessEnv;\n} {\n const {\n executable,\n executableV5,\n restoreUsingV5,\n backupUsingV5,\n backupURL,\n logLevel,\n configPath,\n configPathV5,\n endpoint,\n region,\n port,\n checkpointThresholdMB,\n minCheckpointPageCount = checkpointThresholdMB * 250, // SQLite page size is 4KB\n maxCheckpointPageCount = minCheckpointPageCount * 10,\n incrementalBackupIntervalMinutes,\n incrementalBackupIntervalSeconds,\n snapshotBackupIntervalHours,\n multipartConcurrency,\n multipartSize,\n } = config;\n\n const v5 =\n (mode === 'restore' && restoreUsingV5) ||\n (mode === 'replicate' && backupUsingV5);\n const litestream =\n (v5 ? executableV5 : executable) ??\n must(executable, `Missing --litestream-executable`);\n const litestreamConfig = v5 ? configPathV5 : configPath;\n\n return {\n litestream,\n env: {\n ...process.env,\n ['ZERO_REPLICA_FILE']: replicaFile,\n ['ZERO_LITESTREAM_BACKUP_URL']: must(backupURL),\n ['ZERO_LITESTREAM_MIN_CHECKPOINT_PAGE_COUNT']: String(\n minCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_MAX_CHECKPOINT_PAGE_COUNT']: String(\n maxCheckpointPageCount,\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_MINUTES']: String(\n incrementalBackupIntervalMinutes, // v3 only\n ),\n ['ZERO_LITESTREAM_INCREMENTAL_BACKUP_INTERVAL_SECONDS']: String(\n incrementalBackupIntervalSeconds, // v5 only\n ),\n ['ZERO_LITESTREAM_LOG_LEVEL']: logLevelOverride ?? logLevel,\n ['ZERO_LITESTREAM_SNAPSHOT_BACKUP_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours,\n ),\n ['ZERO_LITESTREAM_SNAPSHOT_RETENTION_INTERVAL_HOURS']: String(\n snapshotBackupIntervalHours + 6, // delete old snapshots after 6 hours\n ),\n ['ZERO_LITESTREAM_MULTIPART_CONCURRENCY']: String(multipartConcurrency),\n ['ZERO_LITESTREAM_MULTIPART_SIZE']: String(multipartSize),\n ['ZERO_LOG_FORMAT']: 'json',\n ['LITESTREAM_CONFIG']: litestreamConfig,\n ['LITESTREAM_PORT']: String(port),\n ...(endpoint ? {['ZERO_LITESTREAM_ENDPOINT']: endpoint} : {}),\n ...(region ? {['ZERO_LITESTREAM_REGION']: region} : {}),\n },\n };\n}\n\nexport async function tryRestore(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n replicaConstraints: ReplicaConstraints | undefined,\n role: LitestreamRole,\n): Promise<RestoreAttempt> {\n const {backupURL} = config;\n const attrs = litestreamRestoreMetricAttrs(config, role, backupURL);\n let result: RestoreResult = 'error';\n try {\n logRestoreDirectoryContents(lc, replicaFile, 'before-temp-cleanup');\n deleteRestoreTempFiles(replicaFile);\n const replicaExistedBeforeRestore = existsSync(replicaFile);\n const {litestream, env} = getLitestream(\n 'restore',\n config,\n replicaFile,\n 'debug', // Include all output from `litestream restore`, as it's minimal.\n );\n const {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n } = config;\n lc.info?.(`starting litestream restore`, {\n restoreParallelism: parallelism,\n multipartConcurrency,\n multipartSize,\n });\n // Pipe (rather than inherit) litestream's stdout/stderr so that its own\n // `\"level\":\"ERROR\"` output on a failed restore does not go straight to the\n // pod's stdout — where a log-scraper alert would page on it — before our\n // code has decided how to handle the failure. The captured output is\n // included in the thrown error so the caller can log the final outcome.\n // See INC-961.\n const proc = spawn(\n litestream,\n [\n 'restore',\n '-if-db-not-exists',\n '-if-replica-exists',\n '-parallelism',\n String(parallelism),\n replicaFile,\n ],\n {env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true},\n );\n let stdout = '';\n let stderr = '';\n proc.stdout.setEncoding('utf-8');\n proc.stderr.setEncoding('utf-8');\n // Output AND capture the stdout and stderr, the former for observability\n // and the latter for error handling.\n proc.stdout.on('data', chunk => {\n process.stdout.write(chunk);\n stdout += chunk;\n });\n proc.stderr.on('data', chunk => {\n process.stderr.write(chunk);\n stderr += chunk;\n });\n const {promise, resolve, reject} = resolver();\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(`litestream killed with ${signal}`);\n } else if (code !== 0) {\n reject(`litestream exited with code ${code}`);\n } else {\n resolve();\n }\n });\n const processStart = performance.now();\n try {\n await promise;\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'success'},\n );\n } catch (e) {\n litestreamRestoreProcessDuration().recordMs(\n performance.now() - processStart,\n {...attrs, result: 'error'},\n );\n const output = [stdout, stderr]\n .map(value => value.trim())\n .filter(Boolean)\n .join('\\n');\n throw new Error(output ? `${String(e)}\\n${output}` : String(e), {\n cause: e,\n });\n }\n if (!existsSync(replicaFile)) {\n result = 'no_backup';\n return {restored: false, backupURL, result};\n }\n const validationStart = performance.now();\n const valid = replicaIsValid(lc, replicaFile, replicaConstraints);\n litestreamRestoreValidationDuration().recordMs(\n performance.now() - validationStart,\n {...attrs, result: valid ? 'success' : 'invalid_replica'},\n );\n if (!valid) {\n result = 'invalid_replica';\n lc.info?.(`Deleting local replica and retrying restore`);\n deleteLiteDB(replicaFile);\n deleteChangeLogDB(replicaFile);\n return {restored: false, backupURL, result};\n }\n result = 'success';\n if (!replicaExistedBeforeRestore) {\n // This restore materialized the replica file, so any change log beside it\n // was written against a replica that is no longer there. Reconciliation\n // is the safety net — a restored replica keeps its replicaVersion, so the\n // rule falls through to truncate-or-reseed on head mismatch — but the log\n // is a cache and deleting it here is the explicit statement that nothing\n // in it survives the restore.\n //\n // A restore that reuses an existing replica (`-if-db-not-exists` made it\n // a no-op) deliberately keeps the log: that is the process-restart path,\n // where the log is still the one written beside this exact replica and\n // discarding it would cost every reconnecting subscriber a `too-old`.\n deleteChangeLogDB(replicaFile);\n litestreamRestoredDbBytes().add(statSync(replicaFile).size, {\n ...attrs,\n result: 'success',\n });\n }\n return {restored: true, backupURL, result};\n } finally {\n try {\n deleteRestoreTempFiles(replicaFile);\n } catch (e) {\n lc.warn?.('Unable to clean up Litestream restore temporary files', {\n replicaFile,\n error: String(e),\n });\n }\n logRestoreDirectoryContents(lc, replicaFile, 'after-temp-cleanup');\n litestreamRestoreAttempts().add(1, {...attrs, result});\n }\n}\n\nfunction deleteRestoreTempFiles(replicaFile: string) {\n const temporaryReplicaFile = `${replicaFile}.tmp`;\n deleteLiteDB(temporaryReplicaFile);\n\n // rocicorp/litestream zero@v0.0.9 downloads WAL indexes in parallel to\n // `<output>.tmp-<8 lowercase hex digits>-wal` before applying them.\n const directory = dirname(temporaryReplicaFile);\n if (!existsSync(directory)) {\n return;\n }\n const prefix = `${basename(temporaryReplicaFile)}-`;\n for (const name of readdirSync(directory)) {\n if (!name.startsWith(prefix) || !name.endsWith('-wal')) {\n continue;\n }\n const index = name.slice(prefix.length, -'-wal'.length);\n if (LEGACY_RESTORE_WAL_INDEX.test(index)) {\n rmSync(join(directory, name), {force: true});\n }\n }\n}\n\nfunction logRestoreDirectoryContents(\n lc: LogContext,\n replicaFile: string,\n phase: 'before-temp-cleanup' | 'after-temp-cleanup',\n) {\n const directory = dirname(replicaFile);\n try {\n const names = readdirSync(directory).sort();\n const entries = names\n .slice(0, MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES)\n .map(name => ({name, sizeBytes: statSync(join(directory, name)).size}));\n lc.info?.('Litestream restore directory contents', {\n phase,\n directory,\n replicaFile,\n entryCount: names.length,\n omittedEntryCount: Math.max(\n 0,\n names.length - MAX_LOGGED_RESTORE_DIRECTORY_ENTRIES,\n ),\n entries,\n });\n } catch (e) {\n lc.warn?.('Unable to inspect Litestream restore directory', {\n phase,\n directory,\n replicaFile,\n error: String(e),\n });\n }\n}\n\nfunction replicaIsValid(\n lc: LogContext,\n replica: string,\n constraints: ReplicaConstraints | undefined,\n) {\n let db: Database | undefined;\n try {\n // Note: Open the database and read the subscription state as a\n // sanity / corruption check, even if there are no constraints.\n db = new Database(lc, replica);\n const {replicaVersion, watermark} = getSubscriptionState(\n new StatementRunner(db),\n );\n if (constraints) {\n if (replicaVersion !== constraints.replicaVersion) {\n lc.warn?.(\n `Local replica version ${replicaVersion} does not match expected replicaVersion ${constraints.replicaVersion}`,\n constraints,\n );\n return false;\n }\n if (watermark < constraints.minWatermark) {\n lc.warn?.(\n `Local replica watermark ${watermark} is earlier than minWatermark ${constraints.minWatermark}`,\n );\n return false;\n }\n lc.info?.(\n `Local replica at version ${replicaVersion} and watermark ${watermark} is compatible`,\n constraints,\n );\n }\n return true;\n } catch (e) {\n if (isSQLiteCorruption(e)) {\n logSQLiteCorruptionDiagnostics(lc, 'restored replica', replica, e);\n }\n lc.error?.('Error while validating restored replica', e);\n return false;\n } finally {\n db?.close();\n }\n}\n\nexport function startReplicaBackupProcess(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): ChildProcess {\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const attrs = litestreamBackupProcessMetricAttrs(config);\n lc.info?.(`starting litestream backup to ${config.backupURL}`);\n const start = performance.now();\n const proc = spawn(litestream, ['replicate'], {\n env,\n stdio: 'inherit',\n windowsHide: true,\n });\n let recorded = false;\n const record = (result: 'success' | 'error' | 'stopped') => {\n if (recorded) {\n return;\n }\n recorded = true;\n const labels = {...attrs, result};\n litestreamBackupProcessRuns().add(1, labels);\n litestreamBackupProcessDuration().recordMs(\n performance.now() - start,\n labels,\n );\n };\n proc.on('error', e => {\n lc.warn?.(`litestream backup process error`, e);\n record('error');\n });\n proc.on('close', (code, signal) => {\n if (signal) {\n lc.info?.(`litestream backup process stopped`, {signal});\n record('stopped');\n } else if (code === 0) {\n record('success');\n } else {\n lc.warn?.(`litestream backup process exited with code ${code}`);\n record('error');\n }\n });\n return proc;\n}\n\n// Listing the backup state requires a few S3 LIST requests, which should\n// normally complete well within this timeout.\nconst LIST_BACKUP_TIMEOUT_MS = 30_000;\n\nconst wsRe = /\\s+/;\n\n/**\n * Returns the time of the most recent object (snapshot or WAL segment)\n * actually uploaded to the backup replica destination, as listed by the\n * bundled litestream CLI (`litestream snapshots` / `litestream wal`).\n *\n * This queries the replica destination (e.g. S3) directly, and thus serves\n * as a source of truth for backup durability. This is in contrast to the\n * `litestream_replica_progress` metric, which is exported when litestream\n * *believes* an upload has succeeded, and has been observed to advance even\n * when nothing is actually written to the destination.\n *\n * Rejects if the backup state cannot be determined (spawn error, non-zero\n * exit, timeout, or empty/unparseable listing).\n */\nexport async function getLastBackupTime(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n): Promise<Date> {\n const [snapshots, wal] = await Promise.all([\n listBackupCreatedTimes(lc, config, replicaFile, 'snapshots'),\n listBackupCreatedTimes(lc, config, replicaFile, 'wal'),\n ]);\n const times = [...snapshots, ...wal];\n if (times.length === 0) {\n // Note: the litestream CLI exits with code 0 and logs listing errors\n // (e.g. S3 failures) to stderr, so an empty listing cannot be\n // distinguished from a failed one. Since a valid backup always contains\n // at least one snapshot, an empty listing is treated as a failure.\n throw new Error(\n `no snapshots or WAL segments listed at ${config.backupURL}`,\n );\n }\n return new Date(Math.max(...times.map(time => time.getTime())));\n}\n\n/**\n * Runs `litestream <snapshots|wal> <replica-file>` with the same config /\n * environment used by the `litestream replicate` process (so that the\n * backupURL, endpoint, region, and credentials are identical), and parses\n * the `created` column (RFC3339) of the tab-formatted output, e.g.:\n *\n * ```\n * replica generation index size created\n * s3 1862f44967b3863f 0 4546445 2026-06-10T01:11:32Z\n * ```\n */\nasync function listBackupCreatedTimes(\n lc: LogContext,\n config: LitestreamConfig,\n replicaFile: string,\n command: 'snapshots' | 'wal',\n): Promise<Date[]> {\n const start = performance.now();\n let result: 'success' | 'empty' | 'timeout' | 'error' = 'error';\n const {litestream, env} = getLitestream('replicate', config, replicaFile);\n const proc = spawn(litestream, [command, replicaFile], {\n env,\n stdio: ['ignore', 'pipe', 'inherit'],\n windowsHide: true,\n });\n const {promise, resolve, reject} = resolver<string>();\n let stdout = '';\n proc.stdout.setEncoding('utf-8');\n proc.stdout.on('data', chunk => (stdout += chunk));\n proc.on('error', reject);\n proc.on('close', (code, signal) => {\n if (signal) {\n reject(new Error(`litestream ${command} killed with ${signal}`));\n } else if (code !== 0) {\n reject(new Error(`litestream ${command} exited with code ${code}`));\n } else {\n resolve(stdout);\n }\n });\n const timeout = setTimeout(() => {\n result = 'timeout';\n reject(new Error(`timed out listing backup state (litestream ${command})`));\n proc.kill('SIGKILL');\n }, LIST_BACKUP_TIMEOUT_MS);\n\n try {\n const output = await promise;\n const times = parseBackupCreatedTimes(lc, command, output);\n result = times.length ? 'success' : 'empty';\n return times;\n } finally {\n clearTimeout(timeout);\n litestreamBackupListDuration().recordMs(performance.now() - start, {\n ...litestreamBackupMetricAttrs(config),\n command,\n result,\n });\n }\n}\n\n/**\n * Parses the `created` column (the last, RFC3339-formatted column) from the\n * tab-formatted output of `litestream snapshots` / `litestream wal`. The\n * header row and any unparseable lines are skipped.\n *\n * Exported for testing.\n */\nexport function parseBackupCreatedTimes(\n lc: LogContext,\n command: 'snapshots' | 'wal',\n output: string,\n): Date[] {\n const times: Date[] = [];\n for (const line of output.split('\\n')) {\n const cols = line.trim().split(wsRe);\n const created = cols.at(-1);\n if (\n cols.length < 2 ||\n created === undefined ||\n created === 'created' /* header row */\n ) {\n continue;\n }\n const time = new Date(created);\n if (Number.isNaN(time.getTime())) {\n lc.warn?.(`unexpected line in litestream ${command} output: ${line}`);\n continue;\n }\n times.push(time);\n }\n return times;\n}\n"],"mappings":";;;;;;;;;;;;;;AAgDA,IAAM,uCAAuC;AAC7C,IAAM,2BAA2B;AAUjC,SAAS,cACP,MACA,QACA,aACA,kBAIA;CACA,MAAM,EACJ,YACA,cACA,gBACA,eACA,WACA,UACA,YACA,cACA,UACA,QACA,MACA,uBACA,yBAAyB,wBAAwB,KACjD,yBAAyB,yBAAyB,IAClD,kCACA,kCACA,6BACA,sBACA,kBACE;CAEJ,MAAM,KACH,SAAS,aAAa,kBACtB,SAAS,eAAe;CAC3B,MAAM,cACH,KAAK,eAAe,eACrB,KAAK,YAAY,iCAAiC;CACpD,MAAM,mBAAmB,KAAK,eAAe;CAE7C,OAAO;EACL;EACA,KAAK;GACH,GAAG,QAAQ;IACV,sBAAsB;IACtB,+BAA+B,KAAK,SAAS;IAC7C,8CAA8C,OAC7C,sBACF;IACC,8CAA8C,OAC7C,sBACF;IACC,wDAAwD,OACvD,gCACF;IACC,wDAAwD,OACvD,gCACF;IACC,8BAA8B,oBAAoB;IAClD,mDAAmD,OAClD,2BACF;IACC,sDAAsD,OACrD,8BAA8B,CAChC;IACC,0CAA0C,OAAO,oBAAoB;IACrE,mCAAmC,OAAO,aAAa;IACvD,oBAAoB;IACpB,sBAAsB;IACtB,oBAAoB,OAAO,IAAI;GAChC,GAAI,WAAW,GAAE,6BAA6B,SAAQ,IAAI,CAAC;GAC3D,GAAI,SAAS,GAAE,2BAA2B,OAAM,IAAI,CAAC;EACvD;CACF;AACF;AAEA,eAAsB,WACpB,IACA,QACA,aACA,oBACA,MACyB;CACzB,MAAM,EAAC,cAAa;CACpB,MAAM,QAAQ,6BAA6B,QAAQ,MAAM,SAAS;CAClE,IAAI,SAAwB;CAC5B,IAAI;EACF,4BAA4B,IAAI,aAAa,qBAAqB;EAClE,uBAAuB,WAAW;EAClC,MAAM,8BAA8B,WAAW,WAAW;EAC1D,MAAM,EAAC,YAAY,QAAO,cACxB,WACA,QACA,aACA,OACF;EACA,MAAM,EACJ,oBAAoB,aACpB,sBACA,kBACE;EACJ,GAAG,OAAO,+BAA+B;GACvC,oBAAoB;GACpB;GACA;EACF,CAAC;EAOD,MAAM,OAAO,MACX,YACA;GACE;GACA;GACA;GACA;GACA,OAAO,WAAW;GAClB;EACF,GACA;GAAC;GAAK,OAAO;IAAC;IAAU;IAAQ;GAAM;GAAG,aAAa;EAAI,CAC5D;EACA,IAAI,SAAS;EACb,IAAI,SAAS;EACb,KAAK,OAAO,YAAY,OAAO;EAC/B,KAAK,OAAO,YAAY,OAAO;EAG/B,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,KAAK,OAAO,GAAG,SAAQ,UAAS;GAC9B,QAAQ,OAAO,MAAM,KAAK;GAC1B,UAAU;EACZ,CAAC;EACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAS;EAC5C,KAAK,GAAG,SAAS,MAAM;EACvB,KAAK,GAAG,UAAU,MAAM,WAAW;GACjC,IAAI,QACF,OAAO,0BAA0B,QAAQ;QACpC,IAAI,SAAS,GAClB,OAAO,+BAA+B,MAAM;QAE5C,QAAQ;EAEZ,CAAC;EACD,MAAM,eAAe,YAAY,IAAI;EACrC,IAAI;GACF,MAAM;GACN,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAS,CAC9B;EACF,SAAS,GAAG;GACV,iCAAiC,EAAE,SACjC,YAAY,IAAI,IAAI,cACpB;IAAC,GAAG;IAAO,QAAQ;GAAO,CAC5B;GACA,MAAM,SAAS,CAAC,QAAQ,MAAM,EAC3B,KAAI,UAAS,MAAM,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,KAAK,IAAI;GACZ,MAAM,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC,EAAE,IAAI,WAAW,OAAO,CAAC,GAAG,EAC9D,OAAO,EACT,CAAC;EACH;EACA,IAAI,CAAC,WAAW,WAAW,GAAG;GAC5B,SAAS;GACT,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,MAAM,kBAAkB,YAAY,IAAI;EACxC,MAAM,QAAQ,eAAe,IAAI,aAAa,kBAAkB;EAChE,oCAAoC,EAAE,SACpC,YAAY,IAAI,IAAI,iBACpB;GAAC,GAAG;GAAO,QAAQ,QAAQ,YAAY;EAAiB,CAC1D;EACA,IAAI,CAAC,OAAO;GACV,SAAS;GACT,GAAG,OAAO,6CAA6C;GACvD,aAAa,WAAW;GACxB,kBAAkB,WAAW;GAC7B,OAAO;IAAC,UAAU;IAAO;IAAW;GAAM;EAC5C;EACA,SAAS;EACT,IAAI,CAAC,6BAA6B;GAYhC,kBAAkB,WAAW;GAC7B,0BAA0B,EAAE,IAAI,SAAS,WAAW,EAAE,MAAM;IAC1D,GAAG;IACH,QAAQ;GACV,CAAC;EACH;EACA,OAAO;GAAC,UAAU;GAAM;GAAW;EAAM;CAC3C,UAAU;EACR,IAAI;GACF,uBAAuB,WAAW;EACpC,SAAS,GAAG;GACV,GAAG,OAAO,yDAAyD;IACjE;IACA,OAAO,OAAO,CAAC;GACjB,CAAC;EACH;EACA,4BAA4B,IAAI,aAAa,oBAAoB;EACjE,0BAA0B,EAAE,IAAI,GAAG;GAAC,GAAG;GAAO;EAAM,CAAC;CACvD;AACF;AAEA,SAAS,uBAAuB,aAAqB;CACnD,MAAM,uBAAuB,GAAG,YAAY;CAC5C,aAAa,oBAAoB;CAIjC,MAAM,YAAY,QAAQ,oBAAoB;CAC9C,IAAI,CAAC,WAAW,SAAS,GACvB;CAEF,MAAM,SAAS,GAAG,SAAS,oBAAoB,EAAE;CACjD,KAAK,MAAM,QAAQ,YAAY,SAAS,GAAG;EACzC,IAAI,CAAC,KAAK,WAAW,MAAM,KAAK,CAAC,KAAK,SAAS,MAAM,GACnD;EAEF,MAAM,QAAQ,KAAK,MAAM,OAAO,QAAQ,EAAc;EACtD,IAAI,yBAAyB,KAAK,KAAK,GACrC,OAAO,KAAK,WAAW,IAAI,GAAG,EAAC,OAAO,KAAI,CAAC;CAE/C;AACF;AAEA,SAAS,4BACP,IACA,aACA,OACA;CACA,MAAM,YAAY,QAAQ,WAAW;CACrC,IAAI;EACF,MAAM,QAAQ,YAAY,SAAS,EAAE,KAAK;EAC1C,MAAM,UAAU,MACb,MAAM,GAAG,oCAAoC,EAC7C,KAAI,UAAS;GAAC;GAAM,WAAW,SAAS,KAAK,WAAW,IAAI,CAAC,EAAE;EAAI,EAAE;EACxE,GAAG,OAAO,yCAAyC;GACjD;GACA;GACA;GACA,YAAY,MAAM;GAClB,mBAAmB,KAAK,IACtB,GACA,MAAM,SAAS,oCACjB;GACA;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,kDAAkD;GAC1D;GACA;GACA;GACA,OAAO,OAAO,CAAC;EACjB,CAAC;CACH;AACF;AAEA,SAAS,eACP,IACA,SACA,aACA;CACA,IAAI;CACJ,IAAI;EAGF,KAAK,IAAI,SAAS,IAAI,OAAO;EAC7B,MAAM,EAAC,gBAAgB,cAAa,qBAClC,IAAI,gBAAgB,EAAE,CACxB;EACA,IAAI,aAAa;GACf,IAAI,mBAAmB,YAAY,gBAAgB;IACjD,GAAG,OACD,yBAAyB,eAAe,0CAA0C,YAAY,kBAC9F,WACF;IACA,OAAO;GACT;GACA,IAAI,YAAY,YAAY,cAAc;IACxC,GAAG,OACD,2BAA2B,UAAU,gCAAgC,YAAY,cACnF;IACA,OAAO;GACT;GACA,GAAG,OACD,4BAA4B,eAAe,iBAAiB,UAAU,iBACtE,WACF;EACF;EACA,OAAO;CACT,SAAS,GAAG;EACV,IAAI,mBAAmB,CAAC,GACtB,+BAA+B,IAAI,oBAAoB,SAAS,CAAC;EAEnE,GAAG,QAAQ,2CAA2C,CAAC;EACvD,OAAO;CACT,UAAU;EACR,IAAI,MAAM;CACZ;AACF;AAEA,SAAgB,0BACd,IACA,QACA,aACc;CACd,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,QAAQ,mCAAmC,MAAM;CACvD,GAAG,OAAO,iCAAiC,OAAO,WAAW;CAC7D,MAAM,QAAQ,YAAY,IAAI;CAC9B,MAAM,OAAO,MAAM,YAAY,CAAC,WAAW,GAAG;EAC5C;EACA,OAAO;EACP,aAAa;CACf,CAAC;CACD,IAAI,WAAW;CACf,MAAM,UAAU,WAA4C;EAC1D,IAAI,UACF;EAEF,WAAW;EACX,MAAM,SAAS;GAAC,GAAG;GAAO;EAAM;EAChC,4BAA4B,EAAE,IAAI,GAAG,MAAM;EAC3C,gCAAgC,EAAE,SAChC,YAAY,IAAI,IAAI,OACpB,MACF;CACF;CACA,KAAK,GAAG,UAAS,MAAK;EACpB,GAAG,OAAO,mCAAmC,CAAC;EAC9C,OAAO,OAAO;CAChB,CAAC;CACD,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QAAQ;GACV,GAAG,OAAO,qCAAqC,EAAC,OAAM,CAAC;GACvD,OAAO,SAAS;EAClB,OAAO,IAAI,SAAS,GAClB,OAAO,SAAS;OACX;GACL,GAAG,OAAO,8CAA8C,MAAM;GAC9D,OAAO,OAAO;EAChB;CACF,CAAC;CACD,OAAO;AACT;AAIA,IAAM,yBAAyB;AAE/B,IAAM,OAAO;;;;;;;;;;;;;;;AAgBb,eAAsB,kBACpB,IACA,QACA,aACe;CACf,MAAM,CAAC,WAAW,OAAO,MAAM,QAAQ,IAAI,CACzC,uBAAuB,IAAI,QAAQ,aAAa,WAAW,GAC3D,uBAAuB,IAAI,QAAQ,aAAa,KAAK,CACvD,CAAC;CACD,MAAM,QAAQ,CAAC,GAAG,WAAW,GAAG,GAAG;CACnC,IAAI,MAAM,WAAW,GAKnB,MAAM,IAAI,MACR,0CAA0C,OAAO,WACnD;CAEF,OAAO,IAAI,KAAK,KAAK,IAAI,GAAG,MAAM,KAAI,SAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC;AAChE;;;;;;;;;;;;AAaA,eAAe,uBACb,IACA,QACA,aACA,SACiB;CACjB,MAAM,QAAQ,YAAY,IAAI;CAC9B,IAAI,SAAoD;CACxD,MAAM,EAAC,YAAY,QAAO,cAAc,aAAa,QAAQ,WAAW;CACxE,MAAM,OAAO,MAAM,YAAY,CAAC,SAAS,WAAW,GAAG;EACrD;EACA,OAAO;GAAC;GAAU;GAAQ;EAAS;EACnC,aAAa;CACf,CAAC;CACD,MAAM,EAAC,SAAS,SAAS,WAAU,SAAiB;CACpD,IAAI,SAAS;CACb,KAAK,OAAO,YAAY,OAAO;CAC/B,KAAK,OAAO,GAAG,SAAQ,UAAU,UAAU,KAAM;CACjD,KAAK,GAAG,SAAS,MAAM;CACvB,KAAK,GAAG,UAAU,MAAM,WAAW;EACjC,IAAI,QACF,uBAAO,IAAI,MAAM,cAAc,QAAQ,eAAe,QAAQ,CAAC;OAC1D,IAAI,SAAS,GAClB,uBAAO,IAAI,MAAM,cAAc,QAAQ,oBAAoB,MAAM,CAAC;OAElE,QAAQ,MAAM;CAElB,CAAC;CACD,MAAM,UAAU,iBAAiB;EAC/B,SAAS;EACT,uBAAO,IAAI,MAAM,8CAA8C,QAAQ,EAAE,CAAC;EAC1E,KAAK,KAAK,SAAS;CACrB,GAAG,sBAAsB;CAEzB,IAAI;EAEF,MAAM,QAAQ,wBAAwB,IAAI,SAAS,MAD9B,OACoC;EACzD,SAAS,MAAM,SAAS,YAAY;EACpC,OAAO;CACT,UAAU;EACR,aAAa,OAAO;EACpB,6BAA6B,EAAE,SAAS,YAAY,IAAI,IAAI,OAAO;GACjE,GAAG,4BAA4B,MAAM;GACrC;GACA;EACF,CAAC;CACH;AACF;;;;;;;;AASA,SAAgB,wBACd,IACA,SACA,QACQ;CACR,MAAM,QAAgB,CAAC;CACvB,KAAK,MAAM,QAAQ,OAAO,MAAM,IAAI,GAAG;EACrC,MAAM,OAAO,KAAK,KAAK,EAAE,MAAM,IAAI;EACnC,MAAM,UAAU,KAAK,GAAG,EAAE;EAC1B,IACE,KAAK,SAAS,KACd,YAAY,KAAA,KACZ,YAAY,WAEZ;EAEF,MAAM,OAAO,IAAI,KAAK,OAAO;EAC7B,IAAI,OAAO,MAAM,KAAK,QAAQ,CAAC,GAAG;GAChC,GAAG,OAAO,iCAAiC,QAAQ,WAAW,MAAM;GACpE;EACF;EACA,MAAM,KAAK,IAAI;CACjB;CACA,OAAO;AACT"}
|
|
@@ -26,5 +26,6 @@ export declare function litestreamBackupProcessDuration(): import("../../observa
|
|
|
26
26
|
export declare function litestreamBackupListDuration(): import("../../observability/metrics.ts").LatencyHistogram;
|
|
27
27
|
export declare function litestreamBackupVerificationDuration(): import("../../observability/metrics.ts").LatencyHistogram;
|
|
28
28
|
export declare function litestreamSnapshotReservationDuration(): import("../../observability/metrics.ts").LatencyHistogram;
|
|
29
|
+
export declare function litestreamSnapshotReservationConfirmDuration(): import("../../observability/metrics.ts").LatencyHistogram;
|
|
29
30
|
export {};
|
|
30
31
|
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAOhE,MAAM,MAAM,cAAc,GAAG,qBAAqB,GAAG,aAAa,CAAC;AACnE,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,IAAI,CAAC;AAEhD,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CAAC;AAEF,KAAK,8BAA8B,GAAG;IACpC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,cAAc,EACpB,SAAS,qBAAmB,GAC3B,qBAAqB,GAAG,8BAA8B,CAaxD;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,gBAAgB,GACvB,qBAAqB,CAUvB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,gBAAgB,GACvB,qBAAqB,GAAG,8BAA8B,CAKxD;AAED,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,iBAAiB,EAC7B,IAAI,EAAE,cAAc,GACnB,qBAAqB,CAMvB;AAuBD,wBAAgB,qBAAqB,kFAMpC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,2BAA2B,kFAM1C;AAED,wBAAgB,yBAAyB,8DAKxC;AAED,wBAAgB,6BAA6B,8DAK5C;AAED,wBAAgB,gCAAgC,8DAK/C;AAED,wBAAgB,mCAAmC,8DAKlD;AAED,wBAAgB,+BAA+B,8DAK9C;AAED,wBAAgB,4BAA4B,8DAK3C;AAED,wBAAgB,oCAAoC,8DAKnD;AAED,wBAAgB,qCAAqC,8DAKpD"}
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AAOhE,MAAM,MAAM,cAAc,GAAG,qBAAqB,GAAG,aAAa,CAAC;AACnE,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,IAAI,CAAC;AAEhD,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,cAAc,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,iBAAiB,CAAC;CAC/B,CAAC;AAEF,KAAK,8BAA8B,GAAG;IACpC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,cAAc,EACpB,SAAS,qBAAmB,GAC3B,qBAAqB,GAAG,8BAA8B,CAaxD;AAED,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,gBAAgB,GACvB,qBAAqB,CAUvB;AAED,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,gBAAgB,GACvB,qBAAqB,GAAG,8BAA8B,CAKxD;AAED,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,iBAAiB,EAC7B,IAAI,EAAE,cAAc,GACnB,qBAAqB,CAMvB;AAuBD,wBAAgB,qBAAqB,kFAMpC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,yBAAyB,kFAMxC;AAED,wBAAgB,2BAA2B,kFAM1C;AAED,wBAAgB,yBAAyB,8DAKxC;AAED,wBAAgB,6BAA6B,8DAK5C;AAED,wBAAgB,gCAAgC,8DAK/C;AAED,wBAAgB,mCAAmC,8DAKlD;AAED,wBAAgB,+BAA+B,8DAK9C;AAED,wBAAgB,4BAA4B,8DAK3C;AAED,wBAAgB,oCAAoC,8DAKnD;AAED,wBAAgB,qCAAqC,8DAKpD;AAED,wBAAgB,4CAA4C,8DAM3D"}
|
|
@@ -81,6 +81,9 @@ function litestreamBackupVerificationDuration() {
|
|
|
81
81
|
function litestreamSnapshotReservationDuration() {
|
|
82
82
|
return litestreamDurationHistogram("litestream.snapshot.reservation_duration", "Duration of a snapshot reservation while a view-syncer restores and subscribes.");
|
|
83
83
|
}
|
|
84
|
+
function litestreamSnapshotReservationConfirmDuration() {
|
|
85
|
+
return litestreamDurationHistogram("litestream.snapshot.reservation_confirm_duration", "Time from opening a snapshot reservation to confirming its change-log bounds, by the source those bounds came from.");
|
|
86
|
+
}
|
|
84
87
|
function litestreamDurationHistogram(name, description) {
|
|
85
88
|
return getOrCreateHistogram("replica", name, {
|
|
86
89
|
description,
|
|
@@ -89,6 +92,6 @@ function litestreamDurationHistogram(name, description) {
|
|
|
89
92
|
});
|
|
90
93
|
}
|
|
91
94
|
//#endregion
|
|
92
|
-
export { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoredDbBytes, litestreamSnapshotReservationDuration };
|
|
95
|
+
export { litestreamBackupListDuration, litestreamBackupMetricAttrs, litestreamBackupProcessDuration, litestreamBackupProcessMetricAttrs, litestreamBackupProcessRuns, litestreamBackupVerificationDuration, litestreamMonitorMetricAttrs, litestreamRestoreAttempts, litestreamRestoreDuration, litestreamRestoreMetricAttrs, litestreamRestoreProcessDuration, litestreamRestoreRuns, litestreamRestoreValidationDuration, litestreamRestoredDbBytes, litestreamSnapshotReservationConfirmDuration, litestreamSnapshotReservationDuration };
|
|
93
96
|
|
|
94
97
|
//# sourceMappingURL=metrics.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"sourcesContent":["import type {LitestreamConfig} from '../../config/normalize.ts';\nimport {\n getOrCreateCounter,\n getOrCreateHistogram,\n LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../../observability/metrics.ts';\n\nexport type LitestreamRole = 'replication_manager' | 'view_syncer';\nexport type LitestreamVersion = 'legacy' | 'v5';\n\nexport type LitestreamMetricAttrs = {\n role: LitestreamRole;\n backup_scheme: string;\n litestream: LitestreamVersion;\n};\n\ntype LitestreamMultipartMetricAttrs = {\n multipart_concurrency: number;\n multipart_size_mib: number;\n};\n\nexport function litestreamRestoreMetricAttrs(\n config: LitestreamConfig,\n role: LitestreamRole,\n backupURL = config.backupURL,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n const {executable, executableV5, restoreUsingV5} = config;\n const selectedExecutable =\n (restoreUsingV5 ? executableV5 : executable) ?? executable;\n return {\n role,\n backup_scheme: litestreamBackupScheme(backupURL),\n litestream:\n executableV5 !== undefined && selectedExecutable === executableV5\n ? 'v5'\n : 'legacy',\n ...litestreamMultipartMetricAttrs(config),\n };\n}\n\nexport function litestreamBackupMetricAttrs(\n config: LitestreamConfig,\n): LitestreamMetricAttrs {\n return {\n role: 'replication_manager',\n backup_scheme: litestreamBackupScheme(config.backupURL),\n litestream:\n config.executableV5 !== undefined &&\n config.executable === config.executableV5\n ? 'v5'\n : 'legacy',\n };\n}\n\nexport function litestreamBackupProcessMetricAttrs(\n config: LitestreamConfig,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n return {\n ...litestreamBackupMetricAttrs(config),\n ...litestreamMultipartMetricAttrs(config),\n };\n}\n\nexport function litestreamMonitorMetricAttrs(\n backupURL: string,\n litestream: LitestreamVersion,\n role: LitestreamRole,\n): LitestreamMetricAttrs {\n return {\n role,\n backup_scheme: litestreamBackupScheme(backupURL),\n litestream,\n };\n}\n\nfunction litestreamBackupScheme(backupURL: string | undefined): string {\n if (!backupURL) {\n return 'unknown';\n }\n try {\n const protocol = new URL(backupURL).protocol;\n return protocol.endsWith(':') ? protocol.slice(0, -1) : protocol;\n } catch {\n return 'unknown';\n }\n}\n\nfunction litestreamMultipartMetricAttrs(\n config: LitestreamConfig,\n): LitestreamMultipartMetricAttrs {\n return {\n multipart_concurrency: config.multipartConcurrency,\n multipart_size_mib: Math.round(config.multipartSize / 1024 / 1024),\n };\n}\n\nexport function litestreamRestoreRuns() {\n return getOrCreateCounter(\n 'replica',\n 'litestream.restore.runs',\n 'Litestream restore runs, labeled by result.',\n );\n}\n\nexport function litestreamRestoreAttempts() {\n return getOrCreateCounter(\n 'replica',\n 'litestream.restore.attempts',\n 'Litestream restore subprocess attempts, labeled by result.',\n );\n}\n\nexport function litestreamRestoredDbBytes() {\n return getOrCreateCounter('replica', 'litestream.restore.db_bytes', {\n description:\n 'SQLite database bytes restored by successful litestream restores.',\n unit: 'bytes',\n });\n}\n\nexport function litestreamBackupProcessRuns() {\n return getOrCreateCounter(\n 'replica',\n 'litestream.backup.process_runs',\n 'Litestream backup process exits, labeled by result.',\n );\n}\n\nexport function litestreamRestoreDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.duration',\n 'Wall-clock duration of a litestream restore run, labeled by result.',\n );\n}\n\nexport function litestreamRestoreWaitDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.wait_duration',\n 'Time spent waiting for the replication-manager snapshot status before restoring.',\n );\n}\n\nexport function litestreamRestoreProcessDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.process_duration',\n 'Wall-clock duration of the litestream restore subprocess.',\n );\n}\n\nexport function litestreamRestoreValidationDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.validation_duration',\n 'Time spent validating a restored replica database.',\n );\n}\n\nexport function litestreamBackupProcessDuration() {\n return litestreamDurationHistogram(\n 'litestream.backup.process_duration',\n 'Runtime duration of the litestream backup subprocess before it exits.',\n );\n}\n\nexport function litestreamBackupListDuration() {\n return litestreamDurationHistogram(\n 'litestream.backup.list_duration',\n 'Duration of litestream backup destination listing commands.',\n );\n}\n\nexport function litestreamBackupVerificationDuration() {\n return litestreamDurationHistogram(\n 'litestream.backup.verification_duration',\n 'Duration of verifying the actual backup state in the backup destination.',\n );\n}\n\nexport function litestreamSnapshotReservationDuration() {\n return litestreamDurationHistogram(\n 'litestream.snapshot.reservation_duration',\n 'Duration of a snapshot reservation while a view-syncer restores and subscribes.',\n );\n}\n\nfunction litestreamDurationHistogram(name: string, description: string) {\n return getOrCreateHistogram('replica', name, {\n description,\n unit: 's',\n bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n });\n}\n"],"mappings":";;AAqBA,SAAgB,6BACd,QACA,MACA,YAAY,OAAO,WACqC;CACxD,MAAM,EAAC,YAAY,cAAc,mBAAkB;CACnD,MAAM,sBACH,iBAAiB,eAAe,eAAe;CAClD,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C,YACE,iBAAiB,KAAA,KAAa,uBAAuB,eACjD,OACA;EACN,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,4BACd,QACuB;CACvB,OAAO;EACL,MAAM;EACN,eAAe,uBAAuB,OAAO,SAAS;EACtD,YACE,OAAO,iBAAiB,KAAA,KACxB,OAAO,eAAe,OAAO,eACzB,OACA;CACR;AACF;AAEA,SAAgB,mCACd,QACwD;CACxD,OAAO;EACL,GAAG,4BAA4B,MAAM;EACrC,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,6BACd,WACA,YACA,MACuB;CACvB,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C;CACF;AACF;AAEA,SAAS,uBAAuB,WAAuC;CACrE,IAAI,CAAC,WACH,OAAO;CAET,IAAI;EACF,MAAM,WAAW,IAAI,IAAI,SAAS,EAAE;EACpC,OAAO,SAAS,SAAS,GAAG,IAAI,SAAS,MAAM,GAAG,EAAE,IAAI;CAC1D,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,+BACP,QACgC;CAChC,OAAO;EACL,uBAAuB,OAAO;EAC9B,oBAAoB,KAAK,MAAM,OAAO,gBAAgB,OAAO,IAAI;CACnE;AACF;AAEA,SAAgB,wBAAwB;CACtC,OAAO,mBACL,WACA,2BACA,6CACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBACL,WACA,+BACA,4DACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBAAmB,WAAW,+BAA+B;EAClE,aACE;EACF,MAAM;CACR,CAAC;AACH;AAEA,SAAgB,8BAA8B;CAC5C,OAAO,mBACL,WACA,kCACA,qDACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,4BACL,+BACA,qEACF;AACF;AASA,SAAgB,mCAAmC;CACjD,OAAO,4BACL,uCACA,2DACF;AACF;AAEA,SAAgB,sCAAsC;CACpD,OAAO,4BACL,0CACA,oDACF;AACF;AAEA,SAAgB,kCAAkC;CAChD,OAAO,4BACL,sCACA,uEACF;AACF;AAEA,SAAgB,+BAA+B;CAC7C,OAAO,4BACL,mCACA,6DACF;AACF;AAEA,SAAgB,uCAAuC;CACrD,OAAO,4BACL,2CACA,0EACF;AACF;AAEA,SAAgB,wCAAwC;CACtD,OAAO,4BACL,4CACA,iFACF;AACF;AAEA,SAAS,4BAA4B,MAAc,aAAqB;CACtE,OAAO,qBAAqB,WAAW,MAAM;EAC3C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH"}
|
|
1
|
+
{"version":3,"file":"metrics.js","names":[],"sources":["../../../../../../zero-cache/src/services/litestream/metrics.ts"],"sourcesContent":["import type {LitestreamConfig} from '../../config/normalize.ts';\nimport {\n getOrCreateCounter,\n getOrCreateHistogram,\n LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n} from '../../observability/metrics.ts';\n\nexport type LitestreamRole = 'replication_manager' | 'view_syncer';\nexport type LitestreamVersion = 'legacy' | 'v5';\n\nexport type LitestreamMetricAttrs = {\n role: LitestreamRole;\n backup_scheme: string;\n litestream: LitestreamVersion;\n};\n\ntype LitestreamMultipartMetricAttrs = {\n multipart_concurrency: number;\n multipart_size_mib: number;\n};\n\nexport function litestreamRestoreMetricAttrs(\n config: LitestreamConfig,\n role: LitestreamRole,\n backupURL = config.backupURL,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n const {executable, executableV5, restoreUsingV5} = config;\n const selectedExecutable =\n (restoreUsingV5 ? executableV5 : executable) ?? executable;\n return {\n role,\n backup_scheme: litestreamBackupScheme(backupURL),\n litestream:\n executableV5 !== undefined && selectedExecutable === executableV5\n ? 'v5'\n : 'legacy',\n ...litestreamMultipartMetricAttrs(config),\n };\n}\n\nexport function litestreamBackupMetricAttrs(\n config: LitestreamConfig,\n): LitestreamMetricAttrs {\n return {\n role: 'replication_manager',\n backup_scheme: litestreamBackupScheme(config.backupURL),\n litestream:\n config.executableV5 !== undefined &&\n config.executable === config.executableV5\n ? 'v5'\n : 'legacy',\n };\n}\n\nexport function litestreamBackupProcessMetricAttrs(\n config: LitestreamConfig,\n): LitestreamMetricAttrs & LitestreamMultipartMetricAttrs {\n return {\n ...litestreamBackupMetricAttrs(config),\n ...litestreamMultipartMetricAttrs(config),\n };\n}\n\nexport function litestreamMonitorMetricAttrs(\n backupURL: string,\n litestream: LitestreamVersion,\n role: LitestreamRole,\n): LitestreamMetricAttrs {\n return {\n role,\n backup_scheme: litestreamBackupScheme(backupURL),\n litestream,\n };\n}\n\nfunction litestreamBackupScheme(backupURL: string | undefined): string {\n if (!backupURL) {\n return 'unknown';\n }\n try {\n const protocol = new URL(backupURL).protocol;\n return protocol.endsWith(':') ? protocol.slice(0, -1) : protocol;\n } catch {\n return 'unknown';\n }\n}\n\nfunction litestreamMultipartMetricAttrs(\n config: LitestreamConfig,\n): LitestreamMultipartMetricAttrs {\n return {\n multipart_concurrency: config.multipartConcurrency,\n multipart_size_mib: Math.round(config.multipartSize / 1024 / 1024),\n };\n}\n\nexport function litestreamRestoreRuns() {\n return getOrCreateCounter(\n 'replica',\n 'litestream.restore.runs',\n 'Litestream restore runs, labeled by result.',\n );\n}\n\nexport function litestreamRestoreAttempts() {\n return getOrCreateCounter(\n 'replica',\n 'litestream.restore.attempts',\n 'Litestream restore subprocess attempts, labeled by result.',\n );\n}\n\nexport function litestreamRestoredDbBytes() {\n return getOrCreateCounter('replica', 'litestream.restore.db_bytes', {\n description:\n 'SQLite database bytes restored by successful litestream restores.',\n unit: 'bytes',\n });\n}\n\nexport function litestreamBackupProcessRuns() {\n return getOrCreateCounter(\n 'replica',\n 'litestream.backup.process_runs',\n 'Litestream backup process exits, labeled by result.',\n );\n}\n\nexport function litestreamRestoreDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.duration',\n 'Wall-clock duration of a litestream restore run, labeled by result.',\n );\n}\n\nexport function litestreamRestoreWaitDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.wait_duration',\n 'Time spent waiting for the replication-manager snapshot status before restoring.',\n );\n}\n\nexport function litestreamRestoreProcessDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.process_duration',\n 'Wall-clock duration of the litestream restore subprocess.',\n );\n}\n\nexport function litestreamRestoreValidationDuration() {\n return litestreamDurationHistogram(\n 'litestream.restore.validation_duration',\n 'Time spent validating a restored replica database.',\n );\n}\n\nexport function litestreamBackupProcessDuration() {\n return litestreamDurationHistogram(\n 'litestream.backup.process_duration',\n 'Runtime duration of the litestream backup subprocess before it exits.',\n );\n}\n\nexport function litestreamBackupListDuration() {\n return litestreamDurationHistogram(\n 'litestream.backup.list_duration',\n 'Duration of litestream backup destination listing commands.',\n );\n}\n\nexport function litestreamBackupVerificationDuration() {\n return litestreamDurationHistogram(\n 'litestream.backup.verification_duration',\n 'Duration of verifying the actual backup state in the backup destination.',\n );\n}\n\nexport function litestreamSnapshotReservationDuration() {\n return litestreamDurationHistogram(\n 'litestream.snapshot.reservation_duration',\n 'Duration of a snapshot reservation while a view-syncer restores and subscribes.',\n );\n}\n\nexport function litestreamSnapshotReservationConfirmDuration() {\n return litestreamDurationHistogram(\n 'litestream.snapshot.reservation_confirm_duration',\n 'Time from opening a snapshot reservation to confirming its change-log ' +\n 'bounds, by the source those bounds came from.',\n );\n}\n\nfunction litestreamDurationHistogram(name: string, description: string) {\n return getOrCreateHistogram('replica', name, {\n description,\n unit: 's',\n bucketBoundaries: LONG_DURATION_HISTOGRAM_BOUNDARIES_S,\n });\n}\n"],"mappings":";;AAqBA,SAAgB,6BACd,QACA,MACA,YAAY,OAAO,WACqC;CACxD,MAAM,EAAC,YAAY,cAAc,mBAAkB;CACnD,MAAM,sBACH,iBAAiB,eAAe,eAAe;CAClD,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C,YACE,iBAAiB,KAAA,KAAa,uBAAuB,eACjD,OACA;EACN,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,4BACd,QACuB;CACvB,OAAO;EACL,MAAM;EACN,eAAe,uBAAuB,OAAO,SAAS;EACtD,YACE,OAAO,iBAAiB,KAAA,KACxB,OAAO,eAAe,OAAO,eACzB,OACA;CACR;AACF;AAEA,SAAgB,mCACd,QACwD;CACxD,OAAO;EACL,GAAG,4BAA4B,MAAM;EACrC,GAAG,+BAA+B,MAAM;CAC1C;AACF;AAEA,SAAgB,6BACd,WACA,YACA,MACuB;CACvB,OAAO;EACL;EACA,eAAe,uBAAuB,SAAS;EAC/C;CACF;AACF;AAEA,SAAS,uBAAuB,WAAuC;CACrE,IAAI,CAAC,WACH,OAAO;CAET,IAAI;EACF,MAAM,WAAW,IAAI,IAAI,SAAS,EAAE;EACpC,OAAO,SAAS,SAAS,GAAG,IAAI,SAAS,MAAM,GAAG,EAAE,IAAI;CAC1D,QAAQ;EACN,OAAO;CACT;AACF;AAEA,SAAS,+BACP,QACgC;CAChC,OAAO;EACL,uBAAuB,OAAO;EAC9B,oBAAoB,KAAK,MAAM,OAAO,gBAAgB,OAAO,IAAI;CACnE;AACF;AAEA,SAAgB,wBAAwB;CACtC,OAAO,mBACL,WACA,2BACA,6CACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBACL,WACA,+BACA,4DACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,mBAAmB,WAAW,+BAA+B;EAClE,aACE;EACF,MAAM;CACR,CAAC;AACH;AAEA,SAAgB,8BAA8B;CAC5C,OAAO,mBACL,WACA,kCACA,qDACF;AACF;AAEA,SAAgB,4BAA4B;CAC1C,OAAO,4BACL,+BACA,qEACF;AACF;AASA,SAAgB,mCAAmC;CACjD,OAAO,4BACL,uCACA,2DACF;AACF;AAEA,SAAgB,sCAAsC;CACpD,OAAO,4BACL,0CACA,oDACF;AACF;AAEA,SAAgB,kCAAkC;CAChD,OAAO,4BACL,sCACA,uEACF;AACF;AAEA,SAAgB,+BAA+B;CAC7C,OAAO,4BACL,mCACA,6DACF;AACF;AAEA,SAAgB,uCAAuC;CACrD,OAAO,4BACL,2CACA,0EACF;AACF;AAEA,SAAgB,wCAAwC;CACtD,OAAO,4BACL,4CACA,iFACF;AACF;AAEA,SAAgB,+CAA+C;CAC7D,OAAO,4BACL,oDACA,qHAEF;AACF;AAEA,SAAS,4BAA4B,MAAc,aAAqB;CACtE,OAAO,qBAAqB,WAAW,MAAM;EAC3C;EACA,MAAM;EACN,kBAAkB;CACpB,CAAC;AACH"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"mutagen.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,MAAM,CAAC;AACrC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,EAAC,SAAS,EAAC,MAAM,6CAA6C,CAAC;AAOtE,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,QAAQ,EACb,KAAK,QAAQ,EAEb,KAAK,QAAQ,EACd,MAAM,2CAA2C,CAAC;AAKnD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,4CAA4C,CAAC;AAEhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,KAAK,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAK5D,OAAO,KAAK,EAAC,UAAU,EAAE,mBAAmB,EAAC,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAEnE,OAAO,KAAK,EAAC,iBAAiB,EAAE,OAAO,EAAC,MAAM,eAAe,CAAC;AAK9D,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,SAAS,CAAC,cAAc,GAAG,SAAS,CAAC,mBAAmB;IAC9D,IAAI,EAAE,MAAM;CACb,CAAC;AAEF,MAAM,WAAW,OAAQ,SAAQ,iBAAiB;IAChD,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,EAAE,OAAO,GAC7B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC;CACvC;AAED,qBAAa,cAAe,YAAW,OAAO,EAAE,OAAO;;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;gBAkBlB,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,UAAU,EACpB,MAAM,EAAE,UAAU,EAClB,iBAAiB,EAAE,eAAe;IA0BpC,GAAG;IAKH,KAAK;IAQL,OAAO,IAAI,OAAO;IAIlB,eAAe,CACb,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;IAuBrC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAIpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAStB;AAID,wBAAsB,eAAe,CACnC,EAAE,EAAE,UAAU,EACd,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,EAAE,EAAE,UAAU,EACd,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,eAAe,EAChC,SAAS,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc;AACtD,qBAAqB,UAAQ,GAC5B,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAqIpC;AAED,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,mBAAmB,EACvB,QAAQ,EAAE,UAAU,GAAG,SAAS,EAChC,KAAK,EAAE,OAAO,EACd,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,OAAO,EAClB,UAAU,EAAE,eAAe,iBAgE5B;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,MAAM,EAAE,QAAQ,GACf,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAKvC;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,cAAc,EAC3B,GAAG,EAAE,QAAQ,GACZ,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAOvC"}
|
|
@@ -127,7 +127,7 @@ async function processMutationWithTx(lc, tx, authData, shard, clientGroupID, mut
|
|
|
127
127
|
return await stmt.execute();
|
|
128
128
|
} finally {
|
|
129
129
|
const q = stmt;
|
|
130
|
-
lc.debug?.(`${q.string}
|
|
130
|
+
lc.debug?.(`${q.string} (${q.parameters.length} params)`);
|
|
131
131
|
}
|
|
132
132
|
}
|
|
133
133
|
authorizer.reloadPermissions();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n isProtocolError,\n ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n type CRUDMutation,\n type DeleteOp,\n type InsertOp,\n type Mutation,\n type UpdateOp,\n type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n primaryKeyValueSchema,\n type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n WriteAuthorizerImpl,\n type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n processMutation(\n mutation: Mutation,\n authData: JWTPayload | undefined,\n customMutatorsEnabled: boolean,\n ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #upstream: PostgresDB;\n readonly #shard: ShardID;\n readonly #stopped = resolver();\n readonly #replica: Database;\n readonly #writeAuthorizer: WriteAuthorizerImpl;\n readonly #limiter: SlidingWindowLimiter | undefined;\n #refCount = 0;\n #isStopped = false;\n\n readonly #crudMutations = getOrCreateCounter(\n 'mutation',\n 'crud',\n 'Number of CRUD mutations processed',\n );\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n clientGroupID: string,\n upstream: PostgresDB,\n config: ZeroConfig,\n writeAuthzStorage: DatabaseStorage,\n ) {\n this.id = clientGroupID;\n this.#lc = lc;\n this.#upstream = upstream;\n this.#shard = shard;\n this.#replica = new Database(this.#lc, config.replica.file, {\n fileMustExist: true,\n });\n this.#writeAuthorizer = new WriteAuthorizerImpl(\n this.#lc,\n config,\n this.#replica,\n shard.appID,\n clientGroupID,\n writeAuthzStorage,\n );\n\n if (config.perUserMutationLimit.max !== undefined) {\n this.#limiter = new SlidingWindowLimiter(\n config.perUserMutationLimit.windowMs,\n config.perUserMutationLimit.max,\n );\n }\n }\n\n ref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n ++this.#refCount;\n }\n\n unref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n --this.#refCount;\n if (this.#refCount <= 0) {\n void this.stop();\n }\n }\n\n hasRefs(): boolean {\n return this.#refCount > 0;\n }\n\n processMutation(\n mutation: Mutation,\n authData: JWTPayload | undefined,\n customMutatorsEnabled = false,\n ): Promise<MutationError | undefined> {\n if (this.#limiter?.canDo() === false) {\n return Promise.resolve([\n ErrorKind.MutationRateLimited,\n 'Rate limit exceeded',\n ]);\n }\n this.#crudMutations.add(1, {\n clientGroupID: this.id,\n });\n return processMutation(\n this.#lc,\n authData,\n this.#upstream,\n this.#shard,\n this.id,\n mutation,\n this.#writeAuthorizer,\n undefined,\n customMutatorsEnabled,\n );\n }\n\n run(): Promise<void> {\n return this.#stopped.promise;\n }\n\n stop(): Promise<void> {\n if (this.#isStopped) {\n return this.#stopped.promise;\n }\n this.#writeAuthorizer.destroy();\n this.#isStopped = true;\n this.#stopped.resolve();\n return this.#stopped.promise;\n }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n lc: LogContext,\n authData: JWTPayload | undefined,\n db: PostgresDB,\n shard: ShardID,\n clientGroupID: string,\n mutation: Mutation,\n writeAuthorizer: WriteAuthorizer,\n onTxStart?: () => void | Promise<void>, // for testing\n customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n assert(\n mutation.type === MutationType.CRUD,\n 'Only CRUD mutations are supported',\n );\n lc = lc.withContext('mutationID', mutation.id);\n lc = lc.withContext('processMutation');\n lc.debug?.('Process mutation start', mutation);\n\n // Record mutation processing attempt for telemetry (regardless of success/failure)\n recordMutation('crud');\n\n let result: MutationError | undefined;\n\n const start = Date.now();\n try {\n // Mutations can fail for a variety of reasons:\n //\n // - application error\n // - network/db error\n // - zero bug\n //\n // For application errors what we want is to re-run the mutation in\n // \"error mode\", which skips the actual mutation and just updates the\n // lastMutationID. Then return the error to the app.\n //\n // However, it's hard to tell the difference between application errors\n // and the other types.\n //\n // A reasonable policy ends up being to just retry every mutation once\n // in error mode. If the error mode mutation succeeds then we assume it\n // was an application error and return the error to the app. Otherwise,\n // we know it was something internal and we log it.\n //\n // This is not 100% correct - there are theoretical cases where we\n // return an internal error to the app that shouldn't have been. But it\n // would have to be a crazy coincidence: we'd have to have a network\n // error on the first attempt that resolves by the second attempt.\n //\n // One might ask why not try/catch just the calls to the mutators and\n // consider those application errors. That is actually what we do in\n // Replicache:\n //\n // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n //\n // We don't do it here because:\n //\n // 1. It's still not perfect. It's hard to isolate SQL errors in\n // mutators due to app developer mistakes from SQL errors due to\n // Zero mistakes.\n // 2. It's not possible to do this with the pg library we're using in\n // Zero anyway: https://github.com/porsager/postgres/issues/455.\n //\n // Personally I think this simple retry policy is nice.\n let errorMode = false;\n for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n try {\n await runTx(\n db,\n async tx => {\n // Simulates a concurrent request for testing. In production this is a noop.\n const done = onTxStart?.();\n try {\n return await processMutationWithTx(\n lc,\n tx,\n authData,\n shard,\n clientGroupID,\n mutation,\n errorMode,\n writeAuthorizer,\n );\n } finally {\n await done;\n }\n },\n {mode: Mode.SERIALIZABLE},\n );\n if (errorMode) {\n lc.debug?.('Ran mutation successfully in error mode');\n }\n break;\n } catch (e) {\n if (e instanceof MutationAlreadyProcessedError) {\n lc.debug?.(e.message);\n // Don't double-count already processed mutations, but they were counted above\n return undefined;\n }\n if (\n isProtocolError(e) &&\n !errorMode &&\n e.kind === ErrorKind.InvalidPush &&\n customMutatorsEnabled &&\n i < 2\n ) {\n // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n // This can create a lot of OOO mutation errors since we do not know when the API server\n // has applied a custom mutation before moving on to process CRUD mutations.\n // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n // They should migrate completely to custom mutators.\n lc.info?.(\n 'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n e,\n );\n await new Promise(resolve => setTimeout(resolve, 100));\n continue;\n }\n if (isProtocolError(e) || errorMode) {\n lc.error?.('Process mutation error', e);\n throw e;\n }\n if (\n e instanceof postgres.PostgresError &&\n e.code === PG_SERIALIZATION_FAILURE\n ) {\n lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n }\n result = [ErrorKind.MutationFailed, String(e)];\n if (errorMode) {\n break;\n }\n lc.error?.('Got error running mutation, re-running in error mode', e);\n errorMode = true;\n i--;\n }\n }\n } finally {\n lc.debug?.('Process mutation complete in', Date.now() - start);\n }\n return result;\n}\n\nexport async function processMutationWithTx(\n lc: LogContext,\n tx: PostgresTransaction,\n authData: JWTPayload | undefined,\n shard: ShardID,\n clientGroupID: string,\n mutation: CRUDMutation,\n errorMode: boolean,\n authorizer: WriteAuthorizer,\n) {\n const tasks: (() => Promise<unknown>)[] = [];\n\n async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n try {\n return await stmt.execute();\n } finally {\n const q = stmt as unknown as Query;\n lc.debug?.(`${q.string}: ${JSON.stringify(q.parameters)}`);\n }\n }\n\n authorizer.reloadPermissions();\n\n if (!errorMode) {\n const {ops} = mutation.args[0];\n\n // Validate all table names before processing\n authorizer.validateTableNames(ops);\n\n const normalizedOps = authorizer.normalizeOps(ops);\n const [canPre, canPost] = await Promise.all([\n authorizer.canPreMutation(authData, normalizedOps),\n authorizer.canPostMutation(authData, normalizedOps),\n ]);\n if (canPre && canPost) {\n for (const op of ops) {\n switch (op.op) {\n case 'insert':\n tasks.push(() => execute(getInsertSQL(tx, op)));\n break;\n case 'upsert':\n tasks.push(() => execute(getUpsertSQL(tx, op)));\n break;\n case 'update':\n tasks.push(() => execute(getUpdateSQL(tx, op)));\n break;\n case 'delete':\n tasks.push(() => execute(getDeleteSQL(tx, op)));\n break;\n default:\n unreachable(op);\n }\n }\n }\n }\n\n // Confirm the mutation even though it may have been blocked by the authorizer.\n // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n tasks.unshift(() =>\n checkSchemaVersionAndIncrementLastMutationID(\n tx,\n shard,\n clientGroupID,\n mutation.clientID,\n mutation.id,\n ),\n );\n\n // Note: An error thrown from any Promise aborts the entire transaction.\n await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n tx: postgres.TransactionSql,\n create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n return tx`\n INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n `;\n}\n\nexport function getUpsertSQL(\n tx: postgres.TransactionSql,\n set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = set;\n return tx`\n INSERT INTO ${tx(tableName)} ${tx(value)}\n ON CONFLICT (${tx(primaryKey)})\n DO UPDATE SET ${tx(value)}\n `;\n}\n\nfunction getUpdateSQL(\n tx: postgres.TransactionSql,\n update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const table = update.tableName;\n const {primaryKey, value} = update;\n const id: Record<string, PrimaryKeyValue> = {};\n for (const key of primaryKey) {\n id[key] = v.parse(value[key], primaryKeyValueSchema);\n }\n return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n id,\n ).flatMap(([key, value], i) =>\n i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n )}`;\n}\n\nfunction getDeleteSQL(\n tx: postgres.TransactionSql,\n deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = deleteOp;\n\n const conditions = [];\n for (const key of primaryKey) {\n if (conditions.length > 0) {\n conditions.push(tx`AND`);\n }\n conditions.push(tx`${tx(key)} = ${value[key]}`);\n }\n\n return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n tx: PostgresTransaction,\n shard: ShardID,\n clientGroupID: string,\n clientID: string,\n receivedMutationID: number,\n) {\n const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n INSERT INTO ${tx(upstreamSchema(shard))}.clients \n as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n VALUES (${clientGroupID}, ${clientID}, ${1})\n ON CONFLICT (\"clientGroupID\", \"clientID\")\n DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n RETURNING \"lastMutationID\"\n `;\n\n // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n if (receivedMutationID < lastMutationID) {\n throw new MutationAlreadyProcessedError(\n clientID,\n receivedMutationID,\n lastMutationID,\n );\n } else if (receivedMutationID > lastMutationID) {\n throw new ProtocolError({\n kind: ErrorKind.InvalidPush,\n message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n origin: ErrorOrigin.ZeroCache,\n });\n }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM;EACX,KAAKC,YAAY;EACjB,KAAKC,SAAS;EACd,KAAKE,WAAW,IAAI,SAAS,KAAKJ,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAKK,mBAAmB,IAAI,oBAC1B,KAAKL,KACL,QACA,KAAKI,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAKE,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAKE,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAKD,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;EACP,IAAI,KAAKA,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAKA,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAKH,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrB,qBACA,qBACF,CAAC;EAEH,KAAKC,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAKP,KACL,UACA,KAAKC,WACL,KAAKC,QACL,KAAK,IACL,UACA,KAAKG,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAKF,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAKK,YACP,OAAO,KAAKL,SAAS;EAEvB,KAAKE,iBAAiB,QAAQ;EAC9B,KAAKG,aAAa;EAClB,KAAKL,SAAS,QAAQ;EACtB,OAAO,KAAKA,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAAS,MAClB,mCACF;CACA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CACrC,GAAG,QAAQ,0BAA0B,QAAQ;CAG7C,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAM,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAAC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GACV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,KAAK,UAAU,EAAE,UAAU,GAAG;EAC3D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;EAG5B,WAAW,mBAAmB,GAAG;EAEjC,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAO,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,EAAE,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQ;CACV,CAAC;AAEL"}
|
|
1
|
+
{"version":3,"file":"mutagen.js","names":["#lc","#upstream","#shard","#stopped","#replica","#writeAuthorizer","#limiter","#crudMutations","#isStopped","#refCount"],"sources":["../../../../../../zero-cache/src/services/mutagen/mutagen.ts"],"sourcesContent":["import {PG_SERIALIZATION_FAILURE} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {resolver} from '@rocicorp/resolver';\nimport type {JWTPayload} from 'jose';\nimport postgres from 'postgres';\nimport {assert, unreachable} from '../../../../shared/src/asserts.ts';\nimport * as v from '../../../../shared/src/valita.ts';\nimport {ErrorKind} from '../../../../zero-protocol/src/error-kind.ts';\nimport {ErrorOrigin} from '../../../../zero-protocol/src/error-origin.ts';\nimport {\n isProtocolError,\n ProtocolError,\n} from '../../../../zero-protocol/src/error.ts';\nimport * as MutationType from '../../../../zero-protocol/src/mutation-type-enum.ts';\nimport {\n type CRUDMutation,\n type DeleteOp,\n type InsertOp,\n type Mutation,\n type UpdateOp,\n type UpsertOp,\n} from '../../../../zero-protocol/src/mutation.ts';\nimport {\n primaryKeyValueSchema,\n type PrimaryKeyValue,\n} from '../../../../zero-protocol/src/primary-key.ts';\nimport type {DatabaseStorage} from '../../../../zqlite/src/database-storage.ts';\nimport {Database} from '../../../../zqlite/src/db.ts';\nimport {\n WriteAuthorizerImpl,\n type WriteAuthorizer,\n} from '../../auth/write-authorizer.ts';\nimport {type ZeroConfig} from '../../config/zero-config.ts';\nimport * as Mode from '../../db/mode-enum.ts';\nimport {runTx} from '../../db/run-transaction.ts';\nimport {getOrCreateCounter} from '../../observability/metrics.ts';\nimport {recordMutation} from '../../server/anonymous-otel-start.ts';\nimport type {PostgresDB, PostgresTransaction} from '../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../types/shards.ts';\nimport {SlidingWindowLimiter} from '../limiter/sliding-window-limiter.ts';\nimport type {RefCountedService, Service} from '../service.ts';\nimport {MutationAlreadyProcessedError} from './error.ts';\n\n// An error encountered processing a mutation.\n// Returned back to application for display to user.\nexport type MutationError = [\n kind: ErrorKind.MutationFailed | ErrorKind.MutationRateLimited,\n desc: string,\n];\n\nexport interface Mutagen extends RefCountedService {\n processMutation(\n mutation: Mutation,\n authData: JWTPayload | undefined,\n customMutatorsEnabled: boolean,\n ): Promise<MutationError | undefined>;\n}\n\nexport class MutagenService implements Mutagen, Service {\n readonly id: string;\n readonly #lc: LogContext;\n readonly #upstream: PostgresDB;\n readonly #shard: ShardID;\n readonly #stopped = resolver();\n readonly #replica: Database;\n readonly #writeAuthorizer: WriteAuthorizerImpl;\n readonly #limiter: SlidingWindowLimiter | undefined;\n #refCount = 0;\n #isStopped = false;\n\n readonly #crudMutations = getOrCreateCounter(\n 'mutation',\n 'crud',\n 'Number of CRUD mutations processed',\n );\n\n constructor(\n lc: LogContext,\n shard: ShardID,\n clientGroupID: string,\n upstream: PostgresDB,\n config: ZeroConfig,\n writeAuthzStorage: DatabaseStorage,\n ) {\n this.id = clientGroupID;\n this.#lc = lc;\n this.#upstream = upstream;\n this.#shard = shard;\n this.#replica = new Database(this.#lc, config.replica.file, {\n fileMustExist: true,\n });\n this.#writeAuthorizer = new WriteAuthorizerImpl(\n this.#lc,\n config,\n this.#replica,\n shard.appID,\n clientGroupID,\n writeAuthzStorage,\n );\n\n if (config.perUserMutationLimit.max !== undefined) {\n this.#limiter = new SlidingWindowLimiter(\n config.perUserMutationLimit.windowMs,\n config.perUserMutationLimit.max,\n );\n }\n }\n\n ref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n ++this.#refCount;\n }\n\n unref() {\n assert(!this.#isStopped, 'MutagenService is already stopped');\n --this.#refCount;\n if (this.#refCount <= 0) {\n void this.stop();\n }\n }\n\n hasRefs(): boolean {\n return this.#refCount > 0;\n }\n\n processMutation(\n mutation: Mutation,\n authData: JWTPayload | undefined,\n customMutatorsEnabled = false,\n ): Promise<MutationError | undefined> {\n if (this.#limiter?.canDo() === false) {\n return Promise.resolve([\n ErrorKind.MutationRateLimited,\n 'Rate limit exceeded',\n ]);\n }\n this.#crudMutations.add(1, {\n clientGroupID: this.id,\n });\n return processMutation(\n this.#lc,\n authData,\n this.#upstream,\n this.#shard,\n this.id,\n mutation,\n this.#writeAuthorizer,\n undefined,\n customMutatorsEnabled,\n );\n }\n\n run(): Promise<void> {\n return this.#stopped.promise;\n }\n\n stop(): Promise<void> {\n if (this.#isStopped) {\n return this.#stopped.promise;\n }\n this.#writeAuthorizer.destroy();\n this.#isStopped = true;\n this.#stopped.resolve();\n return this.#stopped.promise;\n }\n}\n\nconst MAX_SERIALIZATION_ATTEMPTS = 10;\n\nexport async function processMutation(\n lc: LogContext,\n authData: JWTPayload | undefined,\n db: PostgresDB,\n shard: ShardID,\n clientGroupID: string,\n mutation: Mutation,\n writeAuthorizer: WriteAuthorizer,\n onTxStart?: () => void | Promise<void>, // for testing\n customMutatorsEnabled = false,\n): Promise<MutationError | undefined> {\n assert(\n mutation.type === MutationType.CRUD,\n 'Only CRUD mutations are supported',\n );\n lc = lc.withContext('mutationID', mutation.id);\n lc = lc.withContext('processMutation');\n lc.debug?.('Process mutation start', mutation);\n\n // Record mutation processing attempt for telemetry (regardless of success/failure)\n recordMutation('crud');\n\n let result: MutationError | undefined;\n\n const start = Date.now();\n try {\n // Mutations can fail for a variety of reasons:\n //\n // - application error\n // - network/db error\n // - zero bug\n //\n // For application errors what we want is to re-run the mutation in\n // \"error mode\", which skips the actual mutation and just updates the\n // lastMutationID. Then return the error to the app.\n //\n // However, it's hard to tell the difference between application errors\n // and the other types.\n //\n // A reasonable policy ends up being to just retry every mutation once\n // in error mode. If the error mode mutation succeeds then we assume it\n // was an application error and return the error to the app. Otherwise,\n // we know it was something internal and we log it.\n //\n // This is not 100% correct - there are theoretical cases where we\n // return an internal error to the app that shouldn't have been. But it\n // would have to be a crazy coincidence: we'd have to have a network\n // error on the first attempt that resolves by the second attempt.\n //\n // One might ask why not try/catch just the calls to the mutators and\n // consider those application errors. That is actually what we do in\n // Replicache:\n //\n // https://github.com/rocicorp/todo-row-versioning/blob/9a0a79dc2d2de32c4fac61b5d1634bd9a9e66b7c/server/src/push.ts#L131\n //\n // We don't do it here because:\n //\n // 1. It's still not perfect. It's hard to isolate SQL errors in\n // mutators due to app developer mistakes from SQL errors due to\n // Zero mistakes.\n // 2. It's not possible to do this with the pg library we're using in\n // Zero anyway: https://github.com/porsager/postgres/issues/455.\n //\n // Personally I think this simple retry policy is nice.\n let errorMode = false;\n for (let i = 0; i < MAX_SERIALIZATION_ATTEMPTS; i++) {\n try {\n await runTx(\n db,\n async tx => {\n // Simulates a concurrent request for testing. In production this is a noop.\n const done = onTxStart?.();\n try {\n return await processMutationWithTx(\n lc,\n tx,\n authData,\n shard,\n clientGroupID,\n mutation,\n errorMode,\n writeAuthorizer,\n );\n } finally {\n await done;\n }\n },\n {mode: Mode.SERIALIZABLE},\n );\n if (errorMode) {\n lc.debug?.('Ran mutation successfully in error mode');\n }\n break;\n } catch (e) {\n if (e instanceof MutationAlreadyProcessedError) {\n lc.debug?.(e.message);\n // Don't double-count already processed mutations, but they were counted above\n return undefined;\n }\n if (\n isProtocolError(e) &&\n !errorMode &&\n e.kind === ErrorKind.InvalidPush &&\n customMutatorsEnabled &&\n i < 2\n ) {\n // We're temporarily supporting custom mutators AND CRUD mutators at the same time.\n // This can create a lot of OOO mutation errors since we do not know when the API server\n // has applied a custom mutation before moving on to process CRUD mutations.\n // The temporary workaround (since CRUD is being deprecated) is to retry the mutation\n // after a small delay. Users are not expected to be running both CRUD and Custom mutators.\n // They should migrate completely to custom mutators.\n lc.info?.(\n 'Both CRUD and Custom mutators are being used at once. This is supported for now but IS NOT RECOMMENDED. Migrate completely to custom mutators.',\n e,\n );\n await new Promise(resolve => setTimeout(resolve, 100));\n continue;\n }\n if (isProtocolError(e) || errorMode) {\n lc.error?.('Process mutation error', e);\n throw e;\n }\n if (\n e instanceof postgres.PostgresError &&\n e.code === PG_SERIALIZATION_FAILURE\n ) {\n lc.info?.(`attempt ${i + 1}: ${String(e)}`, e);\n continue; // Retry up to MAX_SERIALIZATION_ATTEMPTS.\n }\n result = [ErrorKind.MutationFailed, String(e)];\n if (errorMode) {\n break;\n }\n lc.error?.('Got error running mutation, re-running in error mode', e);\n errorMode = true;\n i--;\n }\n }\n } finally {\n lc.debug?.('Process mutation complete in', Date.now() - start);\n }\n return result;\n}\n\nexport async function processMutationWithTx(\n lc: LogContext,\n tx: PostgresTransaction,\n authData: JWTPayload | undefined,\n shard: ShardID,\n clientGroupID: string,\n mutation: CRUDMutation,\n errorMode: boolean,\n authorizer: WriteAuthorizer,\n) {\n const tasks: (() => Promise<unknown>)[] = [];\n\n async function execute(stmt: postgres.PendingQuery<postgres.Row[]>) {\n try {\n return await stmt.execute();\n } finally {\n const q = stmt as unknown as Query;\n // `q.parameters` are the bound row values -- log the parameterized\n // statement and the parameter count, never the values themselves.\n lc.debug?.(`${q.string} (${q.parameters.length} params)`);\n }\n }\n\n authorizer.reloadPermissions();\n\n if (!errorMode) {\n const {ops} = mutation.args[0];\n\n // Validate all table names before processing\n authorizer.validateTableNames(ops);\n\n const normalizedOps = authorizer.normalizeOps(ops);\n const [canPre, canPost] = await Promise.all([\n authorizer.canPreMutation(authData, normalizedOps),\n authorizer.canPostMutation(authData, normalizedOps),\n ]);\n if (canPre && canPost) {\n for (const op of ops) {\n switch (op.op) {\n case 'insert':\n tasks.push(() => execute(getInsertSQL(tx, op)));\n break;\n case 'upsert':\n tasks.push(() => execute(getUpsertSQL(tx, op)));\n break;\n case 'update':\n tasks.push(() => execute(getUpdateSQL(tx, op)));\n break;\n case 'delete':\n tasks.push(() => execute(getDeleteSQL(tx, op)));\n break;\n default:\n unreachable(op);\n }\n }\n }\n }\n\n // Confirm the mutation even though it may have been blocked by the authorizer.\n // Authorizer blocking a mutation is not an error but the correct result of the mutation.\n tasks.unshift(() =>\n checkSchemaVersionAndIncrementLastMutationID(\n tx,\n shard,\n clientGroupID,\n mutation.clientID,\n mutation.id,\n ),\n );\n\n // Note: An error thrown from any Promise aborts the entire transaction.\n await Promise.all(tasks.map(task => task()));\n}\n\nexport function getInsertSQL(\n tx: postgres.TransactionSql,\n create: InsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n return tx`\n INSERT INTO ${tx(create.tableName)} ${tx(create.value)}\n ON CONFLICT (${tx(create.primaryKey)}) DO NOTHING\n `;\n}\n\nexport function getUpsertSQL(\n tx: postgres.TransactionSql,\n set: UpsertOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = set;\n return tx`\n INSERT INTO ${tx(tableName)} ${tx(value)}\n ON CONFLICT (${tx(primaryKey)})\n DO UPDATE SET ${tx(value)}\n `;\n}\n\nfunction getUpdateSQL(\n tx: postgres.TransactionSql,\n update: UpdateOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const table = update.tableName;\n const {primaryKey, value} = update;\n const id: Record<string, PrimaryKeyValue> = {};\n for (const key of primaryKey) {\n id[key] = v.parse(value[key], primaryKeyValueSchema);\n }\n return tx`UPDATE ${tx(table)} SET ${tx(value)} WHERE ${Object.entries(\n id,\n ).flatMap(([key, value], i) =>\n i ? [tx`AND`, tx`${tx(key)} = ${value}`] : tx`${tx(key)} = ${value}`,\n )}`;\n}\n\nfunction getDeleteSQL(\n tx: postgres.TransactionSql,\n deleteOp: DeleteOp,\n): postgres.PendingQuery<postgres.Row[]> {\n const {tableName, primaryKey, value} = deleteOp;\n\n const conditions = [];\n for (const key of primaryKey) {\n if (conditions.length > 0) {\n conditions.push(tx`AND`);\n }\n conditions.push(tx`${tx(key)} = ${value[key]}`);\n }\n\n return tx`DELETE FROM ${tx(tableName)} WHERE ${conditions}`;\n}\n\nasync function checkSchemaVersionAndIncrementLastMutationID(\n tx: PostgresTransaction,\n shard: ShardID,\n clientGroupID: string,\n clientID: string,\n receivedMutationID: number,\n) {\n const [{lastMutationID}] = await tx<{lastMutationID: bigint}[]>`\n INSERT INTO ${tx(upstreamSchema(shard))}.clients \n as current (\"clientGroupID\", \"clientID\", \"lastMutationID\")\n VALUES (${clientGroupID}, ${clientID}, ${1})\n ON CONFLICT (\"clientGroupID\", \"clientID\")\n DO UPDATE SET \"lastMutationID\" = current.\"lastMutationID\" + 1\n RETURNING \"lastMutationID\"\n `;\n\n // ABORT if the resulting lastMutationID is not equal to the receivedMutationID.\n if (receivedMutationID < lastMutationID) {\n throw new MutationAlreadyProcessedError(\n clientID,\n receivedMutationID,\n lastMutationID,\n );\n } else if (receivedMutationID > lastMutationID) {\n throw new ProtocolError({\n kind: ErrorKind.InvalidPush,\n message: `Push contains unexpected mutation id ${receivedMutationID} for client ${clientID}. Expected mutation id ${lastMutationID.toString()}.`,\n origin: ErrorOrigin.ZeroCache,\n });\n }\n}\n\n// The slice of information from the Query object in Postgres.js that gets logged for debugging.\n// https://github.com/porsager/postgres/blob/f58cd4f3affd3e8ce8f53e42799672d86cd2c70b/src/connection.js#L219\ntype Query = {string: string; parameters: object[]};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA0DA,IAAa,iBAAb,MAAwD;CACtD;CACA;CACA;CACA;CACA,WAAoB,SAAS;CAC7B;CACA;CACA;CACA,YAAY;CACZ,aAAa;CAEb,iBAA0B,mBACxB,YACA,QACA,oCACF;CAEA,YACE,IACA,OACA,eACA,UACA,QACA,mBACA;EACA,KAAK,KAAK;EACV,KAAKA,MAAM;EACX,KAAKC,YAAY;EACjB,KAAKC,SAAS;EACd,KAAKE,WAAW,IAAI,SAAS,KAAKJ,KAAK,OAAO,QAAQ,MAAM,EAC1D,eAAe,KACjB,CAAC;EACD,KAAKK,mBAAmB,IAAI,oBAC1B,KAAKL,KACL,QACA,KAAKI,UACL,MAAM,OACN,eACA,iBACF;EAEA,IAAI,OAAO,qBAAqB,QAAQ,KAAA,GACtC,KAAKE,WAAW,IAAI,qBAClB,OAAO,qBAAqB,UAC5B,OAAO,qBAAqB,GAC9B;CAEJ;CAEA,MAAM;EACJ,OAAO,CAAC,KAAKE,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;CACT;CAEA,QAAQ;EACN,OAAO,CAAC,KAAKD,YAAY,mCAAmC;EAC5D,EAAE,KAAKC;EACP,IAAI,KAAKA,aAAa,GACpB,KAAU,KAAK;CAEnB;CAEA,UAAmB;EACjB,OAAO,KAAKA,YAAY;CAC1B;CAEA,gBACE,UACA,UACA,wBAAwB,OACY;EACpC,IAAI,KAAKH,UAAU,MAAM,MAAM,OAC7B,OAAO,QAAQ,QAAQ,CACrB,qBACA,qBACF,CAAC;EAEH,KAAKC,eAAe,IAAI,GAAG,EACzB,eAAe,KAAK,GACtB,CAAC;EACD,OAAO,gBACL,KAAKP,KACL,UACA,KAAKC,WACL,KAAKC,QACL,KAAK,IACL,UACA,KAAKG,kBACL,KAAA,GACA,qBACF;CACF;CAEA,MAAqB;EACnB,OAAO,KAAKF,SAAS;CACvB;CAEA,OAAsB;EACpB,IAAI,KAAKK,YACP,OAAO,KAAKL,SAAS;EAEvB,KAAKE,iBAAiB,QAAQ;EAC9B,KAAKG,aAAa;EAClB,KAAKL,SAAS,QAAQ;EACtB,OAAO,KAAKA,SAAS;CACvB;AACF;AAEA,IAAM,6BAA6B;AAEnC,eAAsB,gBACpB,IACA,UACA,IACA,OACA,eACA,UACA,iBACA,WACA,wBAAwB,OACY;CACpC,OACE,SAAS,SAAS,MAClB,mCACF;CACA,KAAK,GAAG,YAAY,cAAc,SAAS,EAAE;CAC7C,KAAK,GAAG,YAAY,iBAAiB;CACrC,GAAG,QAAQ,0BAA0B,QAAQ;CAG7C,eAAe,MAAM;CAErB,IAAI;CAEJ,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI;EAuCF,IAAI,YAAY;EAChB,KAAK,IAAI,IAAI,GAAG,IAAI,4BAA4B,KAC9C,IAAI;GACF,MAAM,MACJ,IACA,OAAM,OAAM;IAEV,MAAM,OAAO,YAAY;IACzB,IAAI;KACF,OAAO,MAAM,sBACX,IACA,IACA,UACA,OACA,eACA,UACA,WACA,eACF;IACF,UAAU;KACR,MAAM;IACR;GACF,GACA,EAAC,MAAM,aAAiB,CAC1B;GACA,IAAI,WACF,GAAG,QAAQ,yCAAyC;GAEtD;EACF,SAAS,GAAG;GACV,IAAI,aAAa,+BAA+B;IAC9C,GAAG,QAAQ,EAAE,OAAO;IAEpB;GACF;GACA,IACE,gBAAgB,CAAC,KACjB,CAAC,aACD,EAAE,SAAS,iBACX,yBACA,IAAI,GACJ;IAOA,GAAG,OACD,kJACA,CACF;IACA,MAAM,IAAI,SAAQ,YAAW,WAAW,SAAS,GAAG,CAAC;IACrD;GACF;GACA,IAAI,gBAAgB,CAAC,KAAK,WAAW;IACnC,GAAG,QAAQ,0BAA0B,CAAC;IACtC,MAAM;GACR;GACA,IACE,aAAa,SAAS,iBACtB,EAAE,SAAS,0BACX;IACA,GAAG,OAAO,WAAW,IAAI,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC;IAC7C;GACF;GACA,SAAS,CAAC,gBAA0B,OAAO,CAAC,CAAC;GAC7C,IAAI,WACF;GAEF,GAAG,QAAQ,wDAAwD,CAAC;GACpE,YAAY;GACZ;EACF;CAEJ,UAAU;EACR,GAAG,QAAQ,gCAAgC,KAAK,IAAI,IAAI,KAAK;CAC/D;CACA,OAAO;AACT;AAEA,eAAsB,sBACpB,IACA,IACA,UACA,OACA,eACA,UACA,WACA,YACA;CACA,MAAM,QAAoC,CAAC;CAE3C,eAAe,QAAQ,MAA6C;EAClE,IAAI;GACF,OAAO,MAAM,KAAK,QAAQ;EAC5B,UAAU;GACR,MAAM,IAAI;GAGV,GAAG,QAAQ,GAAG,EAAE,OAAO,IAAI,EAAE,WAAW,OAAO,SAAS;EAC1D;CACF;CAEA,WAAW,kBAAkB;CAE7B,IAAI,CAAC,WAAW;EACd,MAAM,EAAC,QAAO,SAAS,KAAK;EAG5B,WAAW,mBAAmB,GAAG;EAEjC,MAAM,gBAAgB,WAAW,aAAa,GAAG;EACjD,MAAM,CAAC,QAAQ,WAAW,MAAM,QAAQ,IAAI,CAC1C,WAAW,eAAe,UAAU,aAAa,GACjD,WAAW,gBAAgB,UAAU,aAAa,CACpD,CAAC;EACD,IAAI,UAAU,SACZ,KAAK,MAAM,MAAM,KACf,QAAQ,GAAG,IAAX;GACE,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,KAAK;IACH,MAAM,WAAW,QAAQ,aAAa,IAAI,EAAE,CAAC,CAAC;IAC9C;GACF,SACE,YAAY,EAAE;EAClB;CAGN;CAIA,MAAM,cACJ,6CACE,IACA,OACA,eACA,SAAS,UACT,SAAS,EACX,CACF;CAGA,MAAM,QAAQ,IAAI,MAAM,KAAI,SAAQ,KAAK,CAAC,CAAC;AAC7C;AAEA,SAAgB,aACd,IACA,QACuC;CACvC,OAAO,EAAE;kBACO,GAAG,OAAO,SAAS,EAAE,GAAG,GAAG,OAAO,KAAK,EAAE;mBACxC,GAAG,OAAO,UAAU,EAAE;;AAEzC;AAEA,SAAgB,aACd,IACA,KACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CACvC,OAAO,EAAE;kBACO,GAAG,SAAS,EAAE,GAAG,GAAG,KAAK,EAAE;mBAC1B,GAAG,UAAU,EAAE;oBACd,GAAG,KAAK,EAAE;;AAE9B;AAEA,SAAS,aACP,IACA,QACuC;CACvC,MAAM,QAAQ,OAAO;CACrB,MAAM,EAAC,YAAY,UAAS;CAC5B,MAAM,KAAsC,CAAC;CAC7C,KAAK,MAAM,OAAO,YAChB,GAAG,OAAO,MAAQ,MAAM,MAAM,qBAAqB;CAErD,OAAO,EAAE,UAAU,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,SAAS,OAAO,QAC5D,EACF,EAAE,SAAS,CAAC,KAAK,QAAQ,MACvB,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAAO,IAAI,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,OAC/D;AACF;AAEA,SAAS,aACP,IACA,UACuC;CACvC,MAAM,EAAC,WAAW,YAAY,UAAS;CAEvC,MAAM,aAAa,CAAC;CACpB,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI,WAAW,SAAS,GACtB,WAAW,KAAK,EAAE,KAAK;EAEzB,WAAW,KAAK,EAAE,GAAG,GAAG,GAAG,EAAE,KAAK,MAAM,MAAM;CAChD;CAEA,OAAO,EAAE,eAAe,GAAG,SAAS,EAAE,SAAS;AACjD;AAEA,eAAe,6CACb,IACA,OACA,eACA,UACA,oBACA;CACA,MAAM,CAAC,EAAC,oBAAmB,MAAM,EAA8B;kBAC/C,GAAG,eAAe,KAAK,CAAC,EAAE;;oBAExB,cAAc,IAAI,SAAS,IAAI,EAAE;;;;;CAOnD,IAAI,qBAAqB,gBACvB,MAAM,IAAI,8BACR,UACA,oBACA,cACF;MACK,IAAI,qBAAqB,gBAC9B,MAAM,IAAI,cAAc;EACtB,MAAM;EACN,SAAS,wCAAwC,mBAAmB,cAAc,SAAS,yBAAyB,eAAe,SAAS,EAAE;EAC9I,QAAQ;CACV,CAAC;AAEL"}
|
|
@@ -354,5 +354,15 @@ export declare function readChangeLogMeta(db: Database): ChangeLogMeta;
|
|
|
354
354
|
* the log is empty.
|
|
355
355
|
*/
|
|
356
356
|
export declare function readChangeLogHead(db: Database): string | null;
|
|
357
|
+
/**
|
|
358
|
+
* The covered watermark range. Both scalar subqueries seek the watermark
|
|
359
|
+
* index, so unlike a `count()` this does not scan the log. Both bounds are
|
|
360
|
+
* null when it is empty.
|
|
361
|
+
*/
|
|
362
|
+
export declare function readChangeLogBounds(db: Database): ChangeLogBounds;
|
|
363
|
+
export type ChangeLogBounds = {
|
|
364
|
+
readonly minWatermark: string | null;
|
|
365
|
+
readonly headWatermark: string | null;
|
|
366
|
+
};
|
|
357
367
|
export {};
|
|
358
368
|
//# sourceMappingURL=change-log-db.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"change-log-db.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-log-db.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAGH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAMtD,OAAO,EAML,KAAK,SAAS,EACf,MAAM,yBAAyB,CAAC;AAEjC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAK9C,eAAO,MAAM,uBAAuB,0BAA0B,CAAC;AAC/D,eAAO,MAAM,kCAAkC,sCACV,CAAC;AAItC;;;;GAIG;AACH,wBAAgB,+BAA+B,CAC7C,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,EAClB,YAAY,UAAQ,GACnB,MAAM,CASR;AAUD,eAAO,MAAM,+BAA+B,khBAe3C,CAAC;AAmBF,eAAO,MAAM,qBAAqB,wBAAwB,CAAC;AA2B3D;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAE3D;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;;;;;;OAQG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;CAC7B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,oBAAoB,GAAG;IACnD,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,kEAAkE;AAClE,MAAM,MAAM,aAAa,GAAG,iBAAiB,GAAG;IAC9C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAC,GACxB,QAAQ,CAMV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI,CAMxD;AAOD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,EAAE,EAAE,QAAQ,KAAK,eAAe,CAAC;AAE/D,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,cAAc,GACrB;IAAC,EAAE,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,eAAe,CAAC;IAAC,YAAY,EAAE,OAAO,CAAA;CAAC,CAUhE;AAuGD;;;;GAIG;AACH,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,wBAAwB,GACjC;IAAC,EAAE,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,eAAe,CAAC;IAAC,YAAY,EAAE,IAAI,CAAA;CAAC,CAQ7D;AAsCD,MAAM,MAAM,YAAY,GACpB,SAAS,GACT,iBAAiB,GACjB,mBAAmB,GACnB,KAAK,GACL,oBAAoB,CAAC;AAEzB;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QAAS,CAAC;AAClD,eAAO,MAAM,4BAA4B,QAAmB,CAAC;AAE7D,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,SAAkB,IAAI,8BAA8B;IACpD,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;gBAG3C,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,EACzB,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS;CAOtC;AAED,KAAK,gBAAgB,GAAG;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,6BAA6B,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7D,kEAAkE;IAClE,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,eAAe,GACvB;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,KAAK,CAAA;CAAC,GACnD;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,IAAI,CAAA;CAAC,GACrE;IACE,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,YAAY,EAAE,IAAI,CAAC;CACpB,CAAC;AAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,eAAe,EACvB,IAAI,GAAE,gBAAqB,GAC1B,eAAe,CAoBjB;AAgGD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,QAAQ,EACZ,QAAQ,EAAE,iBAAiB,GAC1B,YAAY,GAAG,SAAS,CAwC1B;AA2DD;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,eAAe,GACtB,IAAI,CAqBN;AAED,2EAA2E;AAC3E,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,QAAQ,GAAG,aAAa,CAY7D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,CAO7D"}
|
|
1
|
+
{"version":3,"file":"change-log-db.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/replicator/change-log-db.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAGH,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAC,QAAQ,EAAC,MAAM,8BAA8B,CAAC;AAMtD,OAAO,EAML,KAAK,SAAS,EACf,MAAM,yBAAyB,CAAC;AAEjC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAK9C,eAAO,MAAM,uBAAuB,0BAA0B,CAAC;AAC/D,eAAO,MAAM,kCAAkC,sCACV,CAAC;AAItC;;;;GAIG;AACH,wBAAgB,+BAA+B,CAC7C,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,EAClB,YAAY,UAAQ,GACnB,MAAM,CASR;AAUD,eAAO,MAAM,+BAA+B,khBAe3C,CAAC;AAmBF,eAAO,MAAM,qBAAqB,wBAAwB,CAAC;AA2B3D;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAE3D;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC;;;;;;;;OAQG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC;CAC7B,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,oBAAoB,GAAG;IACnD,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,kEAAkE;AAClE,MAAM,MAAM,aAAa,GAAG,iBAAiB,GAAG;IAC9C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAC,GACxB,QAAQ,CAMV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI,CAMxD;AAOD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,EAAE,EAAE,QAAQ,KAAK,eAAe,CAAC;AAE/D,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,cAAc,GACrB;IAAC,EAAE,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,eAAe,CAAC;IAAC,YAAY,EAAE,OAAO,CAAA;CAAC,CAUhE;AAuGD;;;;GAIG;AACH,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,wBAAwB,GACjC;IAAC,EAAE,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,eAAe,CAAC;IAAC,YAAY,EAAE,IAAI,CAAA;CAAC,CAQ7D;AAsCD,MAAM,MAAM,YAAY,GACpB,SAAS,GACT,iBAAiB,GACjB,mBAAmB,GACnB,KAAK,GACL,oBAAoB,CAAC;AAEzB;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QAAS,CAAC;AAClD,eAAO,MAAM,4BAA4B,QAAmB,CAAC;AAE7D,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,SAAkB,IAAI,8BAA8B;IACpD,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;gBAG3C,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,EACzB,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS;CAOtC;AAED,KAAK,gBAAgB,GAAG;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,6BAA6B,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7D,kEAAkE;IAClE,QAAQ,CAAC,YAAY,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;CAClD,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,MAAM,eAAe,GACvB;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,KAAK,CAAA;CAAC,GACnD;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,IAAI,CAAA;CAAC,GACrE;IACE,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,CAAC;IACrB,YAAY,EAAE,IAAI,CAAC;CACpB,CAAC;AAEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,UAAU,EACd,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,eAAe,EACvB,IAAI,GAAE,gBAAqB,GAC1B,eAAe,CAoBjB;AAgGD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,QAAQ,EACZ,QAAQ,EAAE,iBAAiB,GAC1B,YAAY,GAAG,SAAS,CAwC1B;AA2DD;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,QAAQ,EACZ,MAAM,EAAE,eAAe,GACtB,IAAI,CAqBN;AAED,2EAA2E;AAC3E,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,QAAQ,GAAG,aAAa,CAY7D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,CAO7D;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,QAAQ,GAAG,eAAe,CAUjE;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC,CAAC"}
|
|
@@ -457,6 +457,20 @@ function readChangeLogHead(db) {
|
|
|
457
457
|
return head;
|
|
458
458
|
}
|
|
459
459
|
/**
|
|
460
|
+
* The covered watermark range. Both scalar subqueries seek the watermark
|
|
461
|
+
* index, so unlike a `count()` this does not scan the log. Both bounds are
|
|
462
|
+
* null when it is empty.
|
|
463
|
+
*/
|
|
464
|
+
function readChangeLogBounds(db) {
|
|
465
|
+
return db.prepare(`
|
|
466
|
+
SELECT
|
|
467
|
+
(SELECT min("watermark") FROM "${CHANGE_LOG_STREAM_TABLE}")
|
|
468
|
+
AS "minWatermark",
|
|
469
|
+
(SELECT max("watermark") FROM "${CHANGE_LOG_STREAM_TABLE}")
|
|
470
|
+
AS "headWatermark"
|
|
471
|
+
`).get();
|
|
472
|
+
}
|
|
473
|
+
/**
|
|
460
474
|
* What was installed, for the reconcile log line. Normally `{0, 0}`: a nonzero
|
|
461
475
|
* `backfilling` means the resumed stream is being handed backfills to re-request.
|
|
462
476
|
*/
|
|
@@ -472,6 +486,6 @@ function tableExists(db, table) {
|
|
|
472
486
|
`).get(table) !== void 0;
|
|
473
487
|
}
|
|
474
488
|
//#endregion
|
|
475
|
-
export { CHANGE_LOG_STREAM_TABLE, ChangeLogRebuildRequired, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, estimateChangeLogStreamRowBytes, openChangeLogDBForWriting, readChangeLogHead, readChangeLogMeta, rebuildChangeLogDBForWriting, reconcileChangeLog };
|
|
489
|
+
export { CHANGE_LOG_STREAM_TABLE, ChangeLogRebuildRequired, changeLogFileName, changeLogWipeReason, deleteChangeLogDB, estimateChangeLogStreamRowBytes, openChangeLogDBForWriting, readChangeLogBounds, readChangeLogHead, readChangeLogMeta, rebuildChangeLogDBForWriting, reconcileChangeLog };
|
|
476
490
|
|
|
477
491
|
//# sourceMappingURL=change-log-db.js.map
|