npm - @rocicorp/zero - Versions diffs - 1.12.0-canary.4 → 1.12.0-canary.6 - Mend

@rocicorp/zero 1.12.0-canary.4 → 1.12.0-canary.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (164) hide show

package/out/zero-cache/src/services/change-source/pg/logical-replication/stream.js CHANGED Viewed

@@ -1,10 +1,10 @@
 import { sleep } from "../../../../../../shared/src/sleep.js";
 import { childWorker } from "../../../../types/processes.js";
-import { getOrCreateHistogram } from "../../../../observability/metrics.js";
-import { id, lit } from "../../../../types/sql.js";
 import "../../../../types/pg.js";
+import { getOrCreateHistogram } from "../../../../observability/metrics.js";
 import { SLOT_KEEPER_URL } from "../../../../server/worker-urls.js";
 import { fromBigInt, toBigInt } from "../lsn.js";
+import { id, lit } from "../../../../types/sql.js";
 import { Subscription } from "../../../../types/subscription.js";
 import { pipe } from "../../../../types/streams.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAM1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,QAA2C,MAAM,UAAU,CAAC;AAMnE,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;~~AAUtE~~,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAOL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,QAAQ,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAIjC,aAAa,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CACxC,CAAC;AAUF,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAOzD,0EAA0E;AAC1E,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,8BAA8B;CAC5C;AAKD,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,EACpC,aAAa,GACd,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwD1B;AAKD;;;;;;;;;;;GAWG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,iBAmCjB;AA0BD,MAAM,MAAM,2BAA2B,GAAG;IAExC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC,CAAC;AAsBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,KAAK,EAAE,YAAY,EACnB,EACE,WAAW,EACX,YAAmD,EACnD,YAAmD,GACpD,GAAE,2BAAgC,GAClC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,~~CAsKnC~~;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,YAAY,EAAE,MAAM,EACpB,EAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,MAAM,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CA0D9B;AAED,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EAAE,iBA+BhB;AAmFD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}
1	+ {"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAM1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,QAA2C,MAAM,UAAU,CAAC;AAMnE,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAWtE,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAOL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,QAAQ,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAIjC,aAAa,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;CACxC,CAAC;AAUF,eAAO,MAAM,kCAAkC,QAAS,CAAC;AAOzD,0EAA0E;AAC1E,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,QAAQ,CAAC,IAAI,8BAA8B;CAC5C;AAKD,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,EACpC,aAAa,GACd,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwD1B;AAKD;;;;;;;;;;;GAWG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,iBAmCjB;AA0BD,MAAM,MAAM,2BAA2B,GAAG;IAExC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACnC,CAAC;AAsBF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,KAAK,EAAE,YAAY,EACnB,EACE,WAAW,EACX,YAAmD,EACnD,YAAmD,GACpD,GAAE,2BAAgC,GAClC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAiLnC;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,YAAY,EAAE,MAAM,EACpB,EAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAC,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,GAAG,MAAM,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CA0D9B;AAED,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EAAE,iBA+BhB;AAmFD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js CHANGED Viewed

@@ -10,6 +10,7 @@ import { createReplica, getReplicaState, getRestoreCandidates, metadataPublicati
 import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
 import "../../../types/pg-versions.js";
 import { createReplicationSessionFor, endSession, extractConnectionConfig, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
+import { emitForkRequest } from "./fork-request.js";
 import { defu } from "defu";
 import postgres from "postgres";
 import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE, PG_LOCK_NOT_AVAILABLE } from "@drdgvhbh/postgres-error-codes";
@@ -219,6 +220,11 @@ async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaI
 				dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),
 				lsn: String(toBigInt(slot.consistent_point))
 			});
+			if (stage === 2) await emitForkRequest(lc, sql, shard, {
+				replicaID,
+				slot: slot.slot_name,
+				consistentPoint: slot.consistent_point
+			});
 			const replica = await runTx(sql, async (tx) => {
 				await createReplica(tx, shard, replicaID, slot.slot_name, epoch, stage === 0 ? toStateVersionString(slot.consistent_point) : "", backupOptions, stage);
 				return must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} was not created`);

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_INSUFFICIENT_PRIVILEGE,\n PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n createReplicationSessionFor,\n endSession,\n extractConnectionConfig,\n keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n InitialSync,\n Restore,\n type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n createReplica,\n getReplicaState,\n getRestoreCandidates,\n metadataPublicationName,\n replicationSlotExpression,\n replicationSlotPrefix,\n type BackupOptions,\n type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n slot_name: string;\n consistent_point: string;\n snapshot_name: string;\n output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n slot: ReplicationSlot;\n capturedSnapshot: T;\n initialSession: Readable;\n replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n slot: string;\n reservation: Readable;\n replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n slotName: string;\n\n // Note: ignored if pgVersion < PG_17.\n failover?: boolean;\n\n // Create a temporary slot (i.e. not persisted, and automatically\n // cleaned up when the replication session ends).\n temporary?: boolean;\n\n // For overriding in tests.\n lockTimeout?: number \| undefined;\n\n // A (non-replication) connection to the same upstream instance, used to\n // log what slot creation was waiting on if it times out.\n diagnosticsDB?: PostgresDB \| undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. /\nexport class SlotCreationTimeoutError extends Error {\n readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n// so all commands must be sent using sql.unsafe(). This is technically safe\n// because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n lc: LogContext,\n session: postgres.Sql,\n {\n slotName,\n failover,\n temporary,\n lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n diagnosticsDB,\n }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n // does NOT apply to replication commands, but lock_timeout does (it governs\n // the heavyweight lock wait inside LockAcquire). Setting it here causes\n // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n // walsender, rather than relying solely on the client-side orTimeout\n // which can leave an orphaned backend.\n //\n // An orphaned walsender is actively harmful: by this point the replication\n // slot has already been created and is pinning WAL retention and catalog_xmin.\n // Worse, the slot is marked `active` (the walsender PID is still alive), so\n // the existing cleanup code (which drops inactive slots on retry) can't\n // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n // fires (~2h default) or the blocking transaction finishes.\n await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n const {pgVersion, pid: walsenderPID} = (\n await session.unsafe<{pgVersion: number; pid: number}[]>(`\n SELECT current_setting('server_version_num') as \"pgVersion\",\n pg_backend_pid() as \"pid\";\n `)\n )[0];\n\n const maybeTemporary = temporary ? 'TEMPORARY' : '';\n const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n const createSlot = session.unsafe<ReplicationSlot[]>(/sql/ `\n CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n try {\n const raced = await orTimeout(\n createSlot,\n CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n );\n if (raced === 'timed-out') {\n throw new SlotCreationTimeoutError(\n `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n );\n }\n const [slot] = raced;\n lc.info?.(`Created replication slot ${slotName}`, slot);\n return slot;\n } catch (e) {\n if (\n diagnosticsDB &&\n (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) \|\|\n e instanceof SlotCreationTimeoutError)\n ) {\n // After a lock_timeout, the walsender is no longer waiting, but the\n // transactions it was waiting on are likely still running. After the\n // client-side timeout, it may still be waiting on them.\n await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n }\n throw e;\n }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/\n Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n \n Query text is deliberately omitted since it can contain row values.\n \n Exported for testing.\n /\nexport async function logSlotCreationBlockers(\n lc: LogContext,\n sql: PostgresDB,\n walsenderPID: number,\n slotName: string,\n) {\n try {\n const result = await orTimeout(\n Promise.all([\n sql<JSONObject[]> /sql/ `\n SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n pg_blocking_pids(pid) as \"blockingPIDs\"\n FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n sql<JSONObject[]> /sql/ `\n SELECT pid, datname, usename, application_name as \"applicationName\",\n backend_type as \"backendType\", state,\n wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n backend_xid::text as xid,\n xact_start::text as \"xactStart\",\n extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n FROM pg_stat_activity\n WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n ORDER BY xact_start NULLS LAST\n LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n ]),\n SLOT_DIAGNOSTICS_TIMEOUT_MS,\n );\n if (result === 'timed-out') {\n lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n return;\n }\n const [[walsender], transactions] = result;\n lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n walsender: walsender ?? null,\n transactions,\n });\n } catch (e) {\n lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n }\n}\n\n/\n Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n /\nfunction isSlotCreationBlocked(e: unknown): boolean {\n return (\n isPostgresError(e, PG_LOCK_NOT_AVAILABLE) \|\|\n e instanceof SlotCreationTimeoutError \|\|\n (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n // For overriding in tests.\n lockTimeout?: number \| undefined;\n maxBlockedMs?: number \| undefined;\n retryDelayMs?: number \| undefined;\n};\n\n/*\n Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n /\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n return postgres(\n defu(\n {\n max: 1,\n ['idle_timeout']: null,\n ['max_lifetime']: null as unknown as number,\n connection: {['application_name']: applicationName},\n },\n // See createReplicationSessionFor() for why this cast is necessary.\n db.options as unknown as Options<Record<string, PostgresType>>,\n ),\n );\n}\n\n/\n Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n \n * A dedicated session acquires a (session-level) advisory lock for\n * replica slot management. This is the same lock that cleanup logic\n * acquires before cleaning up replication slots. The lock is held\n * outside of a transaction so that the session is not \"idle in\n * transaction\" while slot creation waits on upstream transactions,\n * which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n * replication session. The API of CREATE_REPLICATION_SLOT is such\n * that it cannot be done in a transaction, and cannot be followed by\n * any writes, or else its snapshot (which is needed for initial sync)\n * would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n * in the `replicas` table before releasing the lock.\n \n Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n \n This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n * will not use the same name for the replication slot (which is selected\n * from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n * rank) will not delete the new slot during their cleanup logic, since\n * the slot will belong to a replica of a higher rank.\n \n When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n \n Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n \n The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n \n For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n /\nexport async function createReplicaAndSlot<T>(\n lc: LogContext,\n sql: PostgresDB,\n sessionName: string,\n shard: ShardID,\n epoch: number,\n replicaID: string,\n failover: boolean,\n backupOptions: BackupOptions,\n captureSnapshot: (snapshot: string) => Promise<T>,\n stage: ReplicaStage,\n {\n lockTimeout,\n maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n const lockName = replicationSlotManagementLock(shard);\n const slotPoolPrefix = replicationSlotPrefix(shard);\n const start = Date.now();\n let addedReplicationRole = false;\n let cleanedUpForSlotLimit = false;\n\n for (let attempt = 1; ; attempt++) {\n await dropUnclaimedSlots(lc, sql, shard);\n\n // Note: The replicationSession is used to create the replication slot\n // and closed immediately afterwards, or on error. A new session is used\n // for each attempt, since a failed attempt closes it.\n const replicationSession = createReplicationSessionFor(sql, sessionName);\n const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n // Ending the lock session releases the replication slot management lock.\n // The sessions are ended with a timeout in case a hung slot creation\n // (e.g. after a network partition) is still pending.\n const endSessions = () =>\n Promise.allSettled([\n replicationSession.end({timeout: 5}),\n lockSession.end({timeout: 5}),\n ]);\n\n let slotName: string \| undefined;\n let initialSession: Readable \| undefined;\n let creatingSlot = false;\n try {\n await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n if (stage === InitialSync) {\n // With the lock acquired, ensure that only one initial sync is\n // active at a time. getRestoreCandidates() orders its results\n // with stage = InitialSync (and active = true) first.\n const others = await getRestoreCandidates(sql, shard, epoch);\n lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n if (others.length) {\n const [{id, stage, active}] = others;\n if (stage === InitialSync && active) {\n throw new AutoResetSignal(\n `another replica (${id}) is performing initial sync`,\n );\n }\n }\n }\n\n // Pick an available slotName from the slotPoolPrefix pool.\n const names = await sql<{name: string}[]> /sql/ `\n SELECT slot_name as name FROM pg_replication_slots\n WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n `.values();\n const inUse = new Set(names.flat());\n for (let next = 0; ; next++) {\n const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n if (!inUse.has(candidateName)) {\n slotName = candidateName;\n break;\n }\n }\n\n creatingSlot = true;\n const slot = await createReplicationSlot(lc, replicationSession, {\n slotName,\n failover,\n lockTimeout,\n diagnosticsDB: sql,\n });\n creatingSlot = false;\n const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n await replicationSession.end({timeout: 5});\n initialSession = await keepSlotActiveUntilTakenOver(lc, {\n db: extractConnectionConfig(sql),\n slot: slot.slot_name,\n dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n lsn: String(toBigInt(slot.consistent_point)),\n });\n\n const replica = await runTx(sql, async tx => {\n await createReplica(\n tx,\n shard,\n replicaID,\n slot.slot_name,\n epoch,\n stage === InitialSync\n ? toStateVersionString(slot.consistent_point)\n : '', // initialized with initRestoreReplica\n backupOptions,\n stage,\n );\n return must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} was not created`,\n );\n });\n\n return {slot, capturedSnapshot, initialSession, replica};\n } catch (e) {\n // Release the lock (and end the attempt's sessions) before handling the\n // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n // session.\n await endSessions();\n if (\n !addedReplicationRole &&\n isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n ) {\n // Some Postgres variants (e.g. Google Cloud SQL) require that\n // the user have the REPLICATION role in order to create a slot.\n // Note that this must be done by the upstreamDB connection, and\n // does not work in the replicationSession itself.\n await sql`ALTER ROLE current_user WITH REPLICATION`;\n lc.info?.(`Added the REPLICATION role to database user`);\n addedReplicationRole = true;\n continue;\n }\n // Note: This is currently manually tested since max_replication_slots\n // is a PG startup parameter that other tests depend on.\n // TODO: Figure out a way to unit test this (with the full PG setup).\n if (\n !cleanedUpForSlotLimit &&\n isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n ) {\n lc.warn?.(\n `Reached max replication slots. Attempting to clean up unused slots`,\n e,\n );\n // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n await sql`\n DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n cleanedUpForSlotLimit = true;\n continue; // then let dropUnclaimedSlots() perform its cleanup\n }\n if (initialSession) {\n try {\n await endSession(initialSession);\n } catch {}\n }\n // The slot may not exist (e.g. if its creation failed), and its name\n // may be reused once the lock is released, so any created slot is\n // left to dropUnclaimedSlots(), which only drops slots without a\n // replica. This runs at the start of the next attempt.\n const blockedMs = Date.now() - start;\n if (\n creatingSlot &&\n isSlotCreationBlocked(e) &&\n blockedMs < maxBlockedMs\n ) {\n lc.warn?.(\n `Replication slot ${slotName} creation blocked by upstream transactions ` +\n `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n e,\n );\n await sleep(retryDelayMs (1 + 4 * Math.random()));\n continue;\n }\n if (slotName) {\n lc.warn?.(`deleting slot ${slotName} due to error`, e);\n await dropUnclaimedSlots(lc, sql, shard);\n }\n throw e;\n } finally {\n await endSessions();\n }\n }\n}\n\n/*\n Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n \n The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n /\nexport async function claimSlotForResumption(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n _sessionName: string,\n {id: replicaID, slot}: Pick<ReplicaState, 'id' \| 'slot'>,\n): Promise<ReservedSlot \| null> {\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n let reservation: Readable \| undefined;\n try {\n const reserved = await runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const replicas = await tx<{active: boolean; lsn: string}[]> /sql/ `\n SELECT active, confirmed_flush_lsn as lsn\n FROM pg_replication_slots\n JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n `;\n if (replicas.length === 0) {\n lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n return null;\n }\n const [{active, lsn}] = replicas;\n if (active) {\n lc.warn?.(\n `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n );\n return null;\n }\n // Mark the replica as Restoring.\n await tx`\n UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n `;\n const replica = must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} disappeared`,\n );\n\n reservation = await keepSlotActiveUntilTakenOver(lc, {\n db: extractConnectionConfig(sql),\n slot,\n dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n lsn: String(toBigInt(lsn)),\n });\n return {\n slot,\n replica,\n reservation,\n };\n });\n lc.info?.(`successfully claimed slot ${slot}`);\n return reserved;\n } catch (e) {\n if (reservation) {\n try {\n await endSession(reservation);\n } catch {}\n }\n lc.error?.(`unable to claim replication slot ${slot}`, e);\n return null;\n }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n slots: string[],\n) {\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx<{slot: string; replicaID: string \| null}[]>\n /sql/ `\n SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name IN ${tx(slots)} AND NOT active;\n `;\n if (dropped.length) {\n lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n dropped,\n });\n }\n\n const replicas = dropped\n .map(({replicaID}) => replicaID)\n .filter(id => id !== null);\n if (replicas.length) {\n // Delete replicas associated with the inactive (and now dropped) slots.\n await tx\n /sql/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n }\n });\n}\n\nfunction dropUnclaimedSlots(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n // The slot / replica cleanup happens within a transaction while holding\n // the replication slot management lock for this shard, to ensure that no\n // slot that belongs to a newer replica is dropped.\n const lockName = replicationSlotManagementLock(shard);\n const slotExpression = replicationSlotExpression(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx /sql/ `\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression} \n AND NOT active\n AND replica.id IS NULL;\n `;\n if (dropped.length) {\n lc.info?.(`dropped inactive replication slots`, {dropped});\n }\n\n // Conversely, delete replicas whose slots no longer exist, e.g. because\n // they were dropped outside of zero-cache's control (such as in an\n // upstream failover). Slot names are reused, so such rows would otherwise\n // be associated with a new slot of the same name. This is done (and\n // committed) before the slot is created, both so that the rows are never\n // associated with it, and because CREATE_REPLICATION_SLOT waits for\n // transactions that have written (i.e. been assigned an xid).\n const orphaned = await tx<{id: string}[]> /sql/ `\n DELETE FROM ${tx(replicasTable)} replica\n WHERE NOT EXISTS (\n SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n )\n RETURNING id;\n `;\n if (orphaned.length) {\n lc.warn?.(`deleted replicas whose slots no longer exist`, {\n replicas: orphaned.map(({id}) => id),\n });\n }\n\n const remaining = await tx<\n {slot: string; pid: number \| null; id: string \| null}[]\n > /sql*/ `\n SELECT slot_name as slot, active_pid as pid, replica.id as id\n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression};\n `;\n if (remaining.length) {\n lc.info?.(`remaining replication slots`, {remaining});\n }\n\n let active = 0;\n let draining = 0;\n for (const {id} of remaining) {\n if (id === null) {\n draining++;\n } else {\n active++;\n }\n }\n\n return {\n dropped: dropped.length,\n active,\n draining,\n };\n });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n n++; // Adjust for 0-based indexing\n\n let suffix = '';\n while (n > 0) {\n n--;\n suffix = ALPHABET[n % 26] + suffix;\n n = Math.floor(n / 26);\n }\n return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAwFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB;;AAG/B,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,EAAA,CACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC;AAI7C,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,CAAC,CAC/B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
1	+ {"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n PG_CONFIGURATION_LIMIT_EXCEEDED,\n PG_INSUFFICIENT_PRIVILEGE,\n PG_LOCK_NOT_AVAILABLE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport {defu} from 'defu';\nimport postgres, {type Options, type PostgresType} from 'postgres';\nimport type {JSONObject} from '../../../../../shared/src/bigint-json.ts';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {sleep} from '../../../../../shared/src/sleep.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {emitForkRequest} from './fork-request.ts';\nimport {\n createReplicationSessionFor,\n endSession,\n extractConnectionConfig,\n keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n InitialSync,\n Restore,\n type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n createReplica,\n getReplicaState,\n getRestoreCandidates,\n metadataPublicationName,\n replicationSlotExpression,\n replicationSlotPrefix,\n type BackupOptions,\n type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n slot_name: string;\n consistent_point: string;\n snapshot_name: string;\n output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n slot: ReplicationSlot;\n capturedSnapshot: T;\n initialSession: Readable;\n replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n slot: string;\n reservation: Readable;\n replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n slotName: string;\n\n // Note: ignored if pgVersion < PG_17.\n failover?: boolean;\n\n // Create a temporary slot (i.e. not persisted, and automatically\n // cleaned up when the replication session ends).\n temporary?: boolean;\n\n // For overriding in tests.\n lockTimeout?: number \| undefined;\n\n // A (non-replication) connection to the same upstream instance, used to\n // log what slot creation was waiting on if it times out.\n diagnosticsDB?: PostgresDB \| undefined;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nexport const CREATE_REPLICATION_SLOT_TIMEOUT_MS = 60_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n/** Thrown when the client-side timeout for creating a slot is reached. /\nexport class SlotCreationTimeoutError extends Error {\n readonly name = 'SlotCreationTimeoutError';\n}\n\n// Note: The replication connection does not support the extended query protocol,\n// so all commands must be sent using sql.unsafe(). This is technically safe\n// because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n lc: LogContext,\n session: postgres.Sql,\n {\n slotName,\n failover,\n temporary,\n lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n diagnosticsDB,\n }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n // does NOT apply to replication commands, but lock_timeout does (it governs\n // the heavyweight lock wait inside LockAcquire). Setting it here causes\n // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n // walsender, rather than relying solely on the client-side orTimeout\n // which can leave an orphaned backend.\n //\n // An orphaned walsender is actively harmful: by this point the replication\n // slot has already been created and is pinning WAL retention and catalog_xmin.\n // Worse, the slot is marked `active` (the walsender PID is still alive), so\n // the existing cleanup code (which drops inactive slots on retry) can't\n // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n // fires (~2h default) or the blocking transaction finishes.\n await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n const {pgVersion, pid: walsenderPID} = (\n await session.unsafe<{pgVersion: number; pid: number}[]>(`\n SELECT current_setting('server_version_num') as \"pgVersion\",\n pg_backend_pid() as \"pid\";\n `)\n )[0];\n\n const maybeTemporary = temporary ? 'TEMPORARY' : '';\n const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n const createSlot = session.unsafe<ReplicationSlot[]>(/sql/ `\n CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n try {\n const raced = await orTimeout(\n createSlot,\n CREATE_REPLICATION_SLOT_TIMEOUT_MS,\n );\n if (raced === 'timed-out') {\n throw new SlotCreationTimeoutError(\n `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}.`,\n );\n }\n const [slot] = raced;\n lc.info?.(`Created replication slot ${slotName}`, slot);\n return slot;\n } catch (e) {\n if (\n diagnosticsDB &&\n (isPostgresError(e, PG_LOCK_NOT_AVAILABLE) \|\|\n e instanceof SlotCreationTimeoutError)\n ) {\n // After a lock_timeout, the walsender is no longer waiting, but the\n // transactions it was waiting on are likely still running. After the\n // client-side timeout, it may still be waiting on them.\n await logSlotCreationBlockers(lc, diagnosticsDB, walsenderPID, slotName);\n }\n throw e;\n }\n}\n\nconst SLOT_DIAGNOSTICS_TIMEOUT_MS = 5_000;\nconst SLOT_DIAGNOSTICS_MAX_TRANSACTIONS = 20;\n\n/\n Best-effort logging of what a timed out CREATE_REPLICATION_SLOT was waiting\n * on: the walsender's wait event and blocking pids, and the oldest\n * transactions holding an xid. Slot creation waits for every such transaction\n * on the instance (in all databases), each with its own lock_timeout, so a\n * chain of overlapping transactions can exceed the client-side timeout\n * without the lock_timeout ever firing.\n \n Query text is deliberately omitted since it can contain row values.\n \n Exported for testing.\n /\nexport async function logSlotCreationBlockers(\n lc: LogContext,\n sql: PostgresDB,\n walsenderPID: number,\n slotName: string,\n) {\n try {\n const result = await orTimeout(\n Promise.all([\n sql<JSONObject[]> /sql/ `\n SELECT pid, wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n pg_blocking_pids(pid) as \"blockingPIDs\"\n FROM pg_stat_activity WHERE pid = ${walsenderPID}`,\n sql<JSONObject[]> /sql/ `\n SELECT pid, datname, usename, application_name as \"applicationName\",\n backend_type as \"backendType\", state,\n wait_event_type as \"waitEventType\", wait_event as \"waitEvent\",\n backend_xid::text as xid,\n xact_start::text as \"xactStart\",\n extract(epoch from now() - xact_start)::float8 as \"xactAgeSec\"\n FROM pg_stat_activity\n WHERE backend_xid IS NOT NULL AND pid <> ${walsenderPID}\n ORDER BY xact_start NULLS LAST\n LIMIT ${SLOT_DIAGNOSTICS_MAX_TRANSACTIONS}`,\n ]),\n SLOT_DIAGNOSTICS_TIMEOUT_MS,\n );\n if (result === 'timed-out') {\n lc.warn?.(`Timed out collecting diagnostics for slot ${slotName}`);\n return;\n }\n const [[walsender], transactions] = result;\n lc.warn?.(`Replication slot ${slotName} creation blocked`, {\n walsender: walsender ?? null,\n transactions,\n });\n } catch (e) {\n lc.warn?.(`Unable to collect diagnostics for slot ${slotName}`, e);\n }\n}\n\n/\n Whether slot creation failed because it was waiting on transactions on the\n * upstream instance: the server's lock_timeout fired, the client-side timeout\n * was reached, or the replication session was closed while it was waiting.\n * Such failures are expected on busy upstreams, and leave no durable state,\n * so the attempt can simply be retried.\n /\nfunction isSlotCreationBlocked(e: unknown): boolean {\n return (\n isPostgresError(e, PG_LOCK_NOT_AVAILABLE) \|\|\n e instanceof SlotCreationTimeoutError \|\|\n (e instanceof Error && (e as {code?: unknown}).code === 'CONNECTION_CLOSED')\n );\n}\n\n// Slot creation is retried while it is blocked by upstream transactions, but\n// a slot that cannot be created for this long is surfaced as an error (i.e.\n// by exiting the process).\nconst DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS = 30 60 * 1000;\n\n// Base delay between retries of blocked slot creation. The actual delay is\n// jittered to between 1x and 5x of this value.\nconst DEFAULT_SLOT_CREATION_RETRY_DELAY_MS = 1_000;\n\nexport type CreateReplicaAndSlotOptions = {\n // For overriding in tests.\n lockTimeout?: number \| undefined;\n maxBlockedMs?: number \| undefined;\n retryDelayMs?: number \| undefined;\n};\n\n/*\n Creates a dedicated (non-pooled) session for holding the session-level\n * replication slot management lock. Ending the session releases the lock,\n * which guarantees that the lock never outlives the attempt.\n /\nfunction createLockSessionFor(db: PostgresDB, applicationName: string) {\n return postgres(\n defu(\n {\n max: 1,\n ['idle_timeout']: null,\n ['max_lifetime']: null as unknown as number,\n connection: {['application_name']: applicationName},\n },\n // See createReplicationSessionFor() for why this cast is necessary.\n db.options as unknown as Options<Record<string, PostgresType>>,\n ),\n );\n}\n\n/\n Replica and slot creation involves several sessions for proper\n * coordination with other replica management logic:\n \n * A dedicated session acquires a (session-level) advisory lock for\n * replica slot management. This is the same lock that cleanup logic\n * acquires before cleaning up replication slots. The lock is held\n * outside of a transaction so that the session is not \"idle in\n * transaction\" while slot creation waits on upstream transactions,\n * which some upstreams terminate (e.g. idle_in_transaction_session_timeout).\n * * With the lock held, a new replication slot is created in a\n * replication session. The API of CREATE_REPLICATION_SLOT is such\n * that it cannot be done in a transaction, and cannot be followed by\n * any writes, or else its snapshot (which is needed for initial sync)\n * would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n * in the `replicas` table before releasing the lock.\n \n Slot creation waits for all transactions on the upstream instance that\n * hold an xid, so it can be blocked by long-running transactions. A blocked\n * attempt is abandoned (see {@link CREATE_REPLICATION_SLOT_TIMEOUT_MS}) and\n * retried in-process, as it leaves no durable state: the `replicas` row is\n * only inserted after the slot has been created. Retries continue until the\n * slot is created or until `maxBlockedMs` has elapsed, after which the error\n * is thrown.\n \n This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n * will not use the same name for the replication slot (which is selected\n * from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n * rank) will not delete the new slot during their cleanup logic, since\n * the slot will belong to a replica of a higher rank.\n \n When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n \n Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n \n The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n \n For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n /\nexport async function createReplicaAndSlot<T>(\n lc: LogContext,\n sql: PostgresDB,\n sessionName: string,\n shard: ShardID,\n epoch: number,\n replicaID: string,\n failover: boolean,\n backupOptions: BackupOptions,\n captureSnapshot: (snapshot: string) => Promise<T>,\n stage: ReplicaStage,\n {\n lockTimeout,\n maxBlockedMs = DEFAULT_MAX_SLOT_CREATION_BLOCKED_MS,\n retryDelayMs = DEFAULT_SLOT_CREATION_RETRY_DELAY_MS,\n }: CreateReplicaAndSlotOptions = {},\n): Promise<ReplicationSlotResult<T>> {\n const lockName = replicationSlotManagementLock(shard);\n const slotPoolPrefix = replicationSlotPrefix(shard);\n const start = Date.now();\n let addedReplicationRole = false;\n let cleanedUpForSlotLimit = false;\n\n for (let attempt = 1; ; attempt++) {\n await dropUnclaimedSlots(lc, sql, shard);\n\n // Note: The replicationSession is used to create the replication slot\n // and closed immediately afterwards, or on error. A new session is used\n // for each attempt, since a failed attempt closes it.\n const replicationSession = createReplicationSessionFor(sql, sessionName);\n const lockSession = createLockSessionFor(sql, `${sessionName}-lock`);\n // Ending the lock session releases the replication slot management lock.\n // The sessions are ended with a timeout in case a hung slot creation\n // (e.g. after a network partition) is still pending.\n const endSessions = () =>\n Promise.allSettled([\n replicationSession.end({timeout: 5}),\n lockSession.end({timeout: 5}),\n ]);\n\n let slotName: string \| undefined;\n let initialSession: Readable \| undefined;\n let creatingSlot = false;\n try {\n await lockSession`SELECT pg_advisory_lock(hashtext(${lockName}))`;\n\n if (stage === InitialSync) {\n // With the lock acquired, ensure that only one initial sync is\n // active at a time. getRestoreCandidates() orders its results\n // with stage = InitialSync (and active = true) first.\n const others = await getRestoreCandidates(sql, shard, epoch);\n lc.info?.(`current replicas at epoch ${epoch}`, {replicas: others});\n if (others.length) {\n const [{id, stage, active}] = others;\n if (stage === InitialSync && active) {\n throw new AutoResetSignal(\n `another replica (${id}) is performing initial sync`,\n );\n }\n }\n }\n\n // Pick an available slotName from the slotPoolPrefix pool.\n const names = await sql<{name: string}[]> /sql/ `\n SELECT slot_name as name FROM pg_replication_slots\n WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n `.values();\n const inUse = new Set(names.flat());\n for (let next = 0; ; next++) {\n const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n if (!inUse.has(candidateName)) {\n slotName = candidateName;\n break;\n }\n }\n\n creatingSlot = true;\n const slot = await createReplicationSlot(lc, replicationSession, {\n slotName,\n failover,\n lockTimeout,\n diagnosticsDB: sql,\n });\n creatingSlot = false;\n const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n await replicationSession.end({timeout: 5});\n initialSession = await keepSlotActiveUntilTakenOver(lc, {\n db: extractConnectionConfig(sql),\n slot: slot.slot_name,\n dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n lsn: String(toBigInt(slot.consistent_point)),\n });\n\n if (stage === Restore) {\n // Expedites the fork, which waits on the source replica's backup of\n // the new slot's position (see fork-request.ts). Emitted before the\n // replica's row is inserted, which flushes it, and outside of that\n // transaction so that a failure to emit it does not fail the fork.\n await emitForkRequest(lc, sql, shard, {\n replicaID,\n slot: slot.slot_name,\n consistentPoint: slot.consistent_point,\n });\n }\n const replica = await runTx(sql, async tx => {\n await createReplica(\n tx,\n shard,\n replicaID,\n slot.slot_name,\n epoch,\n stage === InitialSync\n ? toStateVersionString(slot.consistent_point)\n : '', // initialized with initRestoreReplica\n backupOptions,\n stage,\n );\n return must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} was not created`,\n );\n });\n\n return {slot, capturedSnapshot, initialSession, replica};\n } catch (e) {\n // Release the lock (and end the attempt's sessions) before handling the\n // error, e.g. as dropUnclaimedSlots() acquires the lock from a different\n // session.\n await endSessions();\n if (\n !addedReplicationRole &&\n isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)\n ) {\n // Some Postgres variants (e.g. Google Cloud SQL) require that\n // the user have the REPLICATION role in order to create a slot.\n // Note that this must be done by the upstreamDB connection, and\n // does not work in the replicationSession itself.\n await sql`ALTER ROLE current_user WITH REPLICATION`;\n lc.info?.(`Added the REPLICATION role to database user`);\n addedReplicationRole = true;\n continue;\n }\n // Note: This is currently manually tested since max_replication_slots\n // is a PG startup parameter that other tests depend on.\n // TODO: Figure out a way to unit test this (with the full PG setup).\n if (\n !cleanedUpForSlotLimit &&\n isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)\n ) {\n lc.warn?.(\n `Reached max replication slots. Attempting to clean up unused slots`,\n e,\n );\n // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n await sql`\n DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n cleanedUpForSlotLimit = true;\n continue; // then let dropUnclaimedSlots() perform its cleanup\n }\n if (initialSession) {\n try {\n await endSession(initialSession);\n } catch {}\n }\n // The slot may not exist (e.g. if its creation failed), and its name\n // may be reused once the lock is released, so any created slot is\n // left to dropUnclaimedSlots(), which only drops slots without a\n // replica. This runs at the start of the next attempt.\n const blockedMs = Date.now() - start;\n if (\n creatingSlot &&\n isSlotCreationBlocked(e) &&\n blockedMs < maxBlockedMs\n ) {\n lc.warn?.(\n `Replication slot ${slotName} creation blocked by upstream transactions ` +\n `(attempt ${attempt}, ${blockedMs} ms). Retrying.`,\n e,\n );\n await sleep(retryDelayMs (1 + 4 * Math.random()));\n continue;\n }\n if (slotName) {\n lc.warn?.(`deleting slot ${slotName} due to error`, e);\n await dropUnclaimedSlots(lc, sql, shard);\n }\n throw e;\n } finally {\n await endSessions();\n }\n }\n}\n\n/*\n Claims the inactive slot of the given replica for use by this task, opening\n * a replication session to mark it as active. The process should follow up by\n * taking over the slot via the PostgresChangeSource.\n \n The replica is identified by its ID (and not just its slot) because rows of\n * replicas whose slots were dropped outside of zero-cache's control may share\n * the slot name.\n /\nexport async function claimSlotForResumption(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n _sessionName: string,\n {id: replicaID, slot}: Pick<ReplicaState, 'id' \| 'slot'>,\n): Promise<ReservedSlot \| null> {\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n let reservation: Readable \| undefined;\n try {\n const reserved = await runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const replicas = await tx<{active: boolean; lsn: string}[]> /sql/ `\n SELECT active, confirmed_flush_lsn as lsn\n FROM pg_replication_slots\n JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name = ${slot} AND replica.id = ${replicaID};\n `;\n if (replicas.length === 0) {\n lc.warn?.(`no replica ${replicaID} found for slot ${slot}`);\n return null;\n }\n const [{active, lsn}] = replicas;\n if (active) {\n lc.warn?.(\n `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n );\n return null;\n }\n // Mark the replica as Restoring.\n await tx`\n UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n `;\n const replica = must(\n await getReplicaState(tx, shard, replicaID),\n `replica ${replicaID} disappeared`,\n );\n\n reservation = await keepSlotActiveUntilTakenOver(lc, {\n db: extractConnectionConfig(sql),\n slot,\n dummyPublication: metadataPublicationName(shard.appID, shard.shardNum),\n lsn: String(toBigInt(lsn)),\n });\n return {\n slot,\n replica,\n reservation,\n };\n });\n lc.info?.(`successfully claimed slot ${slot}`);\n return reserved;\n } catch (e) {\n if (reservation) {\n try {\n await endSession(reservation);\n } catch {}\n }\n lc.error?.(`unable to claim replication slot ${slot}`, e);\n return null;\n }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n slots: string[],\n) {\n const lockName = replicationSlotManagementLock(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx<{slot: string; replicaID: string \| null}[]>\n /sql/ `\n SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name IN ${tx(slots)} AND NOT active;\n `;\n if (dropped.length) {\n lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n dropped,\n });\n }\n\n const replicas = dropped\n .map(({replicaID}) => replicaID)\n .filter(id => id !== null);\n if (replicas.length) {\n // Delete replicas associated with the inactive (and now dropped) slots.\n await tx\n /sql/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n }\n });\n}\n\nfunction dropUnclaimedSlots(\n lc: LogContext,\n sql: PostgresDB,\n shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n // The slot / replica cleanup happens within a transaction while holding\n // the replication slot management lock for this shard, to ensure that no\n // slot that belongs to a newer replica is dropped.\n const lockName = replicationSlotManagementLock(shard);\n const slotExpression = replicationSlotExpression(shard);\n const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n return runTx(sql, async tx => {\n await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n const dropped = await tx /sql/ `\n SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression} \n AND NOT active\n AND replica.id IS NULL;\n `;\n if (dropped.length) {\n lc.info?.(`dropped inactive replication slots`, {dropped});\n }\n\n // Conversely, delete replicas whose slots no longer exist, e.g. because\n // they were dropped outside of zero-cache's control (such as in an\n // upstream failover). Slot names are reused, so such rows would otherwise\n // be associated with a new slot of the same name. This is done (and\n // committed) before the slot is created, both so that the rows are never\n // associated with it, and because CREATE_REPLICATION_SLOT waits for\n // transactions that have written (i.e. been assigned an xid).\n const orphaned = await tx<{id: string}[]> /sql/ `\n DELETE FROM ${tx(replicasTable)} replica\n WHERE NOT EXISTS (\n SELECT 1 FROM pg_replication_slots WHERE slot_name = replica.slot\n )\n RETURNING id;\n `;\n if (orphaned.length) {\n lc.warn?.(`deleted replicas whose slots no longer exist`, {\n replicas: orphaned.map(({id}) => id),\n });\n }\n\n const remaining = await tx<\n {slot: string; pid: number \| null; id: string \| null}[]\n > /sql*/ `\n SELECT slot_name as slot, active_pid as pid, replica.id as id\n FROM pg_replication_slots\n LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n WHERE slot_name LIKE ${slotExpression};\n `;\n if (remaining.length) {\n lc.info?.(`remaining replication slots`, {remaining});\n }\n\n let active = 0;\n let draining = 0;\n for (const {id} of remaining) {\n if (id === null) {\n draining++;\n } else {\n active++;\n }\n }\n\n return {\n dropped: dropped.length,\n active,\n draining,\n };\n });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n n++; // Adjust for 0-based indexing\n\n let suffix = '';\n while (n > 0) {\n n--;\n suffix = ALPHABET[n % 26] + suffix;\n n = Math.floor(n / 26);\n }\n return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAyFA,IAAa,qCAAqC;AAKlD,IAAM,yBAAyB;;AAG/B,IAAa,2BAAb,cAA8C,MAAM;CAClD,OAAgB;AAClB;AAKA,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,wBACd,iBAEwB;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,WAAW,KAAK,kBACrB,MAAM,QAAQ,OAA2C;;;GAG1D,EAAA,CACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAChE,MAAM,aAAa,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAAS;CAEtF,IAAI;EACF,MAAM,QAAQ,MAAM,UAClB,YACA,kCACF;EACA,IAAI,UAAU,aACZ,MAAM,IAAI,yBACR,mBAAmB,mCAAmC,gCAAgC,SAAS,EACjG;EAEF,MAAM,CAAC,QAAQ;EACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;EACtD,OAAO;CACT,SAAS,GAAG;EACV,IACE,kBACC,gBAAgB,GAAG,qBAAqB,KACvC,aAAa,2BAKf,MAAM,wBAAwB,IAAI,eAAe,cAAc,QAAQ;EAEzE,MAAM;CACR;AACF;AAEA,IAAM,8BAA8B;AACpC,IAAM,oCAAoC;;;;;;;;;;;;;AAc1C,eAAsB,wBACpB,IACA,KACA,cACA,UACA;CACA,IAAI;EACF,MAAM,SAAS,MAAM,UACnB,QAAQ,IAAI,CACV,GAA0B;;;gDAGc,gBACxC,GAA0B;;;;;;;;uDAQqB,aAAa;;oBAEhD,mCACd,CAAC,GACD,2BACF;EACA,IAAI,WAAW,aAAa;GAC1B,GAAG,OAAO,6CAA6C,UAAU;GACjE;EACF;EACA,MAAM,CAAC,CAAC,YAAY,gBAAgB;EACpC,GAAG,OAAO,oBAAoB,SAAS,oBAAoB;GACzD,WAAW,aAAa;GACxB;EACF,CAAC;CACH,SAAS,GAAG;EACV,GAAG,OAAO,0CAA0C,YAAY,CAAC;CACnE;AACF;;;;;;;;AASA,SAAS,sBAAsB,GAAqB;CAClD,OACE,gBAAgB,GAAG,qBAAqB,KACxC,aAAa,4BACZ,aAAa,SAAU,EAAuB,SAAS;AAE5D;AAKA,IAAM,uCAAuC;AAI7C,IAAM,uCAAuC;;;;;;AAc7C,SAAS,qBAAqB,IAAgB,iBAAyB;CACrE,OAAO,SACL,KACE;EACE,KAAK;GACJ,iBAAiB;GACjB,iBAAiB;EAClB,YAAY,GAAE,qBAAqB,gBAAe;CACpD,GAEA,GAAG,OACL,CACF;AACF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACA,EACE,aACA,eAAe,sCACf,eAAe,yCACgB,CAAC,GACC;CACnC,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAClD,MAAM,QAAQ,KAAK,IAAI;CACvB,IAAI,uBAAuB;CAC3B,IAAI,wBAAwB;CAE5B,KAAK,IAAI,UAAU,IAAK,WAAW;EACjC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAKvC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;EACvE,MAAM,cAAc,qBAAqB,KAAK,GAAG,YAAY,MAAM;EAInE,MAAM,oBACJ,QAAQ,WAAW,CACjB,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC,GACnC,YAAY,IAAI,EAAC,SAAS,EAAC,CAAC,CAC9B,CAAC;EAEH,IAAI;EACJ,IAAI;EACJ,IAAI,eAAe;EACnB,IAAI;GACF,MAAM,WAAW,oCAAoC,SAAS;GAE9D,IAAI,UAAA,GAAuB;IAIzB,MAAM,SAAS,MAAM,qBAAqB,KAAK,OAAO,KAAK;IAC3D,GAAG,OAAO,6BAA6B,SAAS,EAAC,UAAU,OAAM,CAAC;IAClE,IAAI,OAAO,QAAQ;KACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;KAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;IAEJ;GACF;GAGA,MAAM,QAAQ,MAAM,GAA8B;;iCAEvB,iBAAiB,IAAI;QAC9C,OAAO;GACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;GAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;IAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;IAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;KAC7B,WAAW;KACX;IACF;GACF;GAEA,eAAe;GACf,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;IAC/D;IACA;IACA;IACA,eAAe;GACjB,CAAC;GACD,eAAe;GACf,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;GACjE,MAAM,mBAAmB,IAAI,EAAC,SAAS,EAAC,CAAC;GACzC,iBAAiB,MAAM,6BAA6B,IAAI;IACtD,IAAI,wBAAwB,GAAG;IAC/B,MAAM,KAAK;IACX,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,KAAK,gBAAgB,CAAC;GAC7C,CAAC;GAED,IAAI,UAAA,GAKF,MAAM,gBAAgB,IAAI,KAAK,OAAO;IACpC;IACA,MAAM,KAAK;IACX,iBAAiB,KAAK;GACxB,CAAC;GAEH,MAAM,UAAU,MAAM,MAAM,KAAK,OAAM,OAAM;IAC3C,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IACA,OAAO,KACL,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBACvB;GACF,CAAC;GAED,OAAO;IAAC;IAAM;IAAkB;IAAgB;GAAO;EACzD,SAAS,GAAG;GAIV,MAAM,YAAY;GAClB,IACE,CAAC,wBACD,gBAAgB,GAAG,yBAAyB,GAC5C;IAKA,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD,uBAAuB;IACvB;GACF;GAIA,IACE,CAAC,yBACD,gBAAgB,GAAG,+BAA+B,GAClD;IACA,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC,wBAAwB;IACxB;GACF;GACA,IAAI,gBACF,IAAI;IACF,MAAM,WAAW,cAAc;GACjC,QAAQ,CAAC;GAMX,MAAM,YAAY,KAAK,IAAI,IAAI;GAC/B,IACE,gBACA,sBAAsB,CAAC,KACvB,YAAY,cACZ;IACA,GAAG,OACD,oBAAoB,SAAS,sDACf,QAAQ,IAAI,UAAU,kBACpC,CACF;IACA,MAAM,MAAM,gBAAgB,IAAI,IAAI,KAAK,OAAO,EAAE;IAClD;GACF;GACA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI,KAAK,KAAK;GACzC;GACA,MAAM;EACR,UAAU;GACR,MAAM,YAAY;EACpB;CACF;AACF;;;;;;;;;;AAWA,eAAsB,uBACpB,IACA,KACA,OACA,cACA,EAAC,IAAI,WAAW,QACc;CAC9B,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;CACJ,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAA6C;;;eAG3D,GAAG,aAAa,EAAE;4BACL,KAAK,oBAAoB,UAAU;;GAEzD,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,cAAc,UAAU,kBAAkB,MAAM;IAC1D,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,SAAQ;GACxB,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAE9E,MAAM,UAAU,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aACvB;GAEA,cAAc,MAAM,6BAA6B,IAAI;IACnD,IAAI,wBAAwB,GAAG;IAC/B;IACA,kBAAkB,wBAAwB,MAAM,OAAO,MAAM,QAAQ;IACrE,KAAK,OAAO,SAAS,GAAG,CAAC;GAC3B,CAAC;GACD,OAAO;IACL;IACA;IACA;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,IAAI,aACF,IAAI;GACF,MAAM,WAAW,WAAW;EAC9B,QAAQ,CAAC;EAEX,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,CAAC,CAC/B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAU3D,MAAM,WAAW,MAAM,EAA2B;oBAClC,GAAG,aAAa,EAAE;;;;;;EAMlC,IAAI,SAAS,QACX,GAAG,OAAO,gDAAgD,EACxD,UAAU,SAAS,KAAK,EAAC,SAAQ,EAAE,EACrC,CAAC;EAGH,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/change-source/pg/schema/ddl.js CHANGED Viewed

@@ -1,8 +1,8 @@
 import { assert } from "../../../../../../shared/src/asserts.js";
 import { literalUnion, valita_exports } from "../../../../../../shared/src/valita.js";
 import { upstreamSchema } from "../../../../types/shards.js";
-import { id } from "../../../../types/sql.js";
 import { publishedSchema, publishedSchemaQuery } from "./published.js";
+import { id } from "../../../../types/sql.js";
 import { literal } from "pg-format";
 var versionSchema = literalUnion(1, 2);
 var ddlEventSchema = valita_exports.object({ context: valita_exports.object({ query: valita_exports.string() }).rest(valita_exports.string()) }).extend({

package/out/zero-cache/src/services/change-source/pg/schema/init.js CHANGED Viewed

@@ -1,8 +1,8 @@
 import { assert } from "../../../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { upstreamSchema } from "../../../../types/shards.js";
-import { id } from "../../../../types/sql.js";
 import { publishedSchema } from "./published.js";
+import { id } from "../../../../types/sql.js";
 import "./replica-stage-enum.js";
 import { getMutationsTableDefinition, legacyReplicationSlot, metadataPublicationName, setupTablesAndReplication, setupTriggers } from "./shard.js";
 import { AutoResetSignal } from "../../../change-streamer/schema/tables.js";

package/out/zero-cache/src/services/change-source/pg/schema/published.js CHANGED Viewed

@@ -3,10 +3,10 @@ import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { equals } from "../../../../../../shared/src/set-utils.js";
 import { BigIntJSON } from "../../../../../../shared/src/bigint-json.js";
 import { publishedColumnSpec, publishedIndexSpec, publishedTableSpec } from "../../../../db/specs.js";
+import { getOrCreateCounter } from "../../../../observability/metrics.js";
 import { liteTableName } from "../../../../types/names.js";
 import { mapPostgresToLite, mapPostgresToLiteIndex } from "../../../../db/pg-to-lite.js";
 import { computeZqlSpecsFromLiteSpecs } from "../../../../db/lite-tables.js";
-import { getOrCreateCounter } from "../../../../observability/metrics.js";
 import { translatePostgresIndexPredicate } from "./index-predicate.js";
 import { literal } from "pg-format";
 //#region ../zero-cache/src/services/change-source/pg/schema/published.ts

package/out/zero-cache/src/services/change-source/pg/schema/shard.js CHANGED Viewed

@@ -2,8 +2,8 @@ import { assert } from "../../../../../../shared/src/asserts.js";
 import { parse, valita_exports } from "../../../../../../shared/src/valita.js";
 import { jsonObjectSchema, stringify } from "../../../../../../shared/src/bigint-json.js";
 import { appSchema, check, upstreamSchema } from "../../../../types/shards.js";
-import { id } from "../../../../types/sql.js";
 import { getPublicationInfo, publishedSchema } from "./published.js";
+import { id } from "../../../../types/sql.js";
 import { createEventFunctionStatements, createEventTriggerStatements } from "./ddl.js";
 import { replicaStageSchema } from "./replica-stage-enum.js";
 import { validate } from "./validation.js";

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts CHANGED Viewed

@@ -768,6 +768,7 @@ export declare const changeStreamMessageSchema: v.UnionType<[v.UnionType<[v.Tupl
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>]>;

package/out/zero-cache/src/services/change-source/protocol/current/downstream.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
1	+ {"version":3,"file":"downstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/downstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAW5D,QAAA,MAAM,KAAK;;;;;;eAIT,CAAC;AACH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBAGR,CAAC;AACH,QAAA,MAAM,MAAM;;;;;eAIV,CAAC;AACH,QAAA,MAAM,QAAQ;;eAAmD,CAAC;AAElE,MAAM,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,CAAC,CAAC;AAC1C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AACxC,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,CAAC;AAEhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAAyC,CAAC;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,yBAAyB;;;;eAGpC,CAAC;AACH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAIrC,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts CHANGED Viewed

@@ -14,6 +14,7 @@ export declare const downstreamStatusSchema: v.ObjectType<{
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>;
 export type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;
 export declare const downstreamStatusMessageSchema: v.TupleType<[v.Type<"status">, v.ObjectType<{
@@ -26,6 +27,7 @@ export declare const downstreamStatusMessageSchema: v.TupleType<[v.Type<"status"
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     watermark: v.Type<string>;
 }, undefined>]>;
@@ -45,6 +47,7 @@ export declare const upstreamStatusMessageSchema: v.TupleType<[v.Type<"status">,
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
     commitTimeMs: v.Optional<number>;

package/out/zero-cache/src/services/change-source/protocol/current/status.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB~~;;;;;;;;;;aAQjC~~,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B~~;;;;;;;;;;;;~~eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B~~;;;;;;;;;;;;;;;~~eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}
1	+ {"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAI5D;;;;GAIG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;aAajC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;eAIxC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;eAItC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAC3C,OAAO,6BAA6B,CACrC,CAAC;AACF,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/status.js CHANGED Viewed

@@ -9,7 +9,8 @@ import { changeSourceReportSchema } from "../../../replicator/reporter/report-sc
 */
 var downstreamStatusSchema = valita_exports.object({
 	ack: valita_exports.boolean().optional(() => true),
-	lagReport: changeSourceReportSchema.optional()
+	lagReport: changeSourceReportSchema.optional(),
+	syncBackupThrough: valita_exports.string().optional()
 });
 var downstreamStatusMessageSchema = valita_exports.tuple([
 	valita_exports.literal("status"),

package/out/zero-cache/src/services/change-source/protocol/current/status.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"status.js","names":["v"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {changeSourceReportSchema} from '../../../replicator/reporter/report-schema.ts';\nimport {commitSchema} from './data.ts';\n\n/*\n The downstream status messages contain metadata about the status\n * of the change-source. indicates whether it should be echoed\n * back in an upstream status message.\n /\nexport const downstreamStatusSchema = v.object({\n // Indicates whether the status message should be echoed back\n // in an upstream status message once the consumer has successfully\n // processed/persisted all preceding changes.\n ack: v.boolean().optional(() => true),\n\n // Contains a lag report for recording end to end latency metrics.\n lagReport: changeSourceReportSchema.optional(),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n v.literal('status'),\n downstreamStatusSchema,\n v.object({watermark: v.string()}),\n]);\n\n/\n The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n /\nexport const upstreamStatusMessageSchema = v.tuple([\n v.literal('status'),\n v.union(downstreamStatusSchema, commitSchema),\n v.object({watermark: v.string()}),\n]);\n\n/\n Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n \n A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n \n The `zero-cache` sends StatusMessages to the ChangeSource:\n \n * when it has processed a `Commit` received from the ChangeSource,\n * unless the `Begin` message specified `skipAck`.\n \n * when it receives a `StatusMessage` and all preceding `Commit` messages\n * have been processed\n \n This allows the ChangeSource to clean up change log entries appropriately.\n \n Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"mappings":";;;;;;;;;AASA,IAAa,yBAAyBA,eAAE,OAAO;CAI7C,KAAKA,eAAE,QAAQ,CAAC,CAAC,eAAe,IAAI;CAGpC,WAAW,yBAAyB,SAAS;~~AAC/C~~,CAAC;AAID,IAAa,gCAAgCA,eAAE,MAAM;CACnDA,eAAE,QAAQ,QAAQ;CAClB;CACAA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;;;;;;;AAQD,IAAa,8BAA8BA,eAAE,MAAM;CACjDA,eAAE,QAAQ,QAAQ;CAClBA,eAAE,MAAM,wBAAwB,YAAY;CAC5CA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC"}
1	+ {"version":3,"file":"status.js","names":["v"],"sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/status.ts"],"sourcesContent":["import * as v from '../../../../../../shared/src/valita.ts';\nimport {changeSourceReportSchema} from '../../../replicator/reporter/report-schema.ts';\nimport {commitSchema} from './data.ts';\n\n/*\n The downstream status messages contain metadata about the status\n * of the change-source. indicates whether it should be echoed\n * back in an upstream status message.\n /\nexport const downstreamStatusSchema = v.object({\n // Indicates whether the status message should be echoed back\n // in an upstream status message once the consumer has successfully\n // processed/persisted all preceding changes.\n ack: v.boolean().optional(() => true),\n\n // Contains a lag report for recording end to end latency metrics.\n lagReport: changeSourceReportSchema.optional(),\n\n // Requests that the consumer back up its changes through this watermark\n // promptly, rather than at its regular backup cadence, as the change source\n // is waiting on the resulting ack (e.g. to fork a new replica from it).\n syncBackupThrough: v.string().optional(),\n});\n\nexport type DownstreamStatus = v.Infer<typeof downstreamStatusSchema>;\n\nexport const downstreamStatusMessageSchema = v.tuple([\n v.literal('status'),\n downstreamStatusSchema,\n v.object({watermark: v.string()}),\n]);\n\n/\n The `zero-cache` will send the Commit payload to acknowledge a completed\n * transaction (unless the `skipAck` field was specified in the Begin message\n * of the transaction), and will echo back the downstream `status` message if\n * `ack` is true.\n /\nexport const upstreamStatusMessageSchema = v.tuple([\n v.literal('status'),\n v.union(downstreamStatusSchema, commitSchema),\n v.object({watermark: v.string()}),\n]);\n\n/\n Status messages convey positional information from both the ChangeSource\n * and the `zero-cache`.\n \n A StatusMessage from the ChangeSource indicates a position in its change\n * log. Generally, the watermarks sent in `Commit` messages already convey\n * this information, but a StatusMessage may also be sent to indicate that the\n * log has progressed without any corresponding changes relevant to the\n * subscriber. The watermarks of commit messages and status messages must be\n * monotonic in the stream of messages from the ChangeSource.\n \n The `zero-cache` sends StatusMessages to the ChangeSource:\n \n * when it has processed a `Commit` received from the ChangeSource,\n * unless the `Begin` message specified `skipAck`.\n \n * when it receives a `StatusMessage` and all preceding `Commit` messages\n * have been processed\n \n This allows the ChangeSource to clean up change log entries appropriately.\n \n Note that StatusMessages from the ChangeSource are optional. If a\n * ChangeSource implementation can track subscriber progress and clean up\n * its change log purely from Commit-driven StatusMessages there is no need\n * for the ChangeSource to send StatusMessages.\n */\nexport type DownstreamStatusMessage = v.Infer<\n typeof downstreamStatusMessageSchema\n>;\nexport type UpstreamStatusMessage = v.Infer<typeof upstreamStatusMessageSchema>;\n"],"mappings":";;;;;;;;;AASA,IAAa,yBAAyBA,eAAE,OAAO;CAI7C,KAAKA,eAAE,QAAQ,CAAC,CAAC,eAAe,IAAI;CAGpC,WAAW,yBAAyB,SAAS;CAK7C,mBAAmBA,eAAE,OAAO,CAAC,CAAC,SAAS;AACzC,CAAC;AAID,IAAa,gCAAgCA,eAAE,MAAM;CACnDA,eAAE,QAAQ,QAAQ;CAClB;CACAA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC;;;;;;;AAQD,IAAa,8BAA8BA,eAAE,MAAM;CACjDA,eAAE,QAAQ,QAAQ;CAClBA,eAAE,MAAM,wBAAwB,YAAY;CAC5CA,eAAE,OAAO,EAAC,WAAWA,eAAE,OAAO,EAAC,CAAC;AAClC,CAAC"}

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts CHANGED Viewed

@@ -10,6 +10,7 @@ export declare const changeSourceUpstreamSchema: v.TupleType<[v.Type<"status">,
         };
         nextSendTimeMs: number;
     }>;
+    syncBackupThrough: v.Optional<string>;
 }, undefined>, v.ObjectType<{
     tag: v.Type<"commit">;
     commitTimeMs: v.Optional<number>;

package/out/zero-cache/src/services/change-source/protocol/current/upstream.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B~~;;;;;;;;;;;;;;;~~eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
1	+ {"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/protocol/current/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,wCAAwC,CAAC;AAS5D,sEAAsE;AACtE,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;eAA8B,CAAC;AACtE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;aAYhC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts CHANGED Viewed

@@ -1,14 +1,12 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { NormalizedZeroConfig } from '../../config/normalize.ts';
 import { BackupMonitor } from './backup-monitor.ts';
-import type { ChangeStreamerService } from './change-streamer.ts';
 import { type BackupStateVerifier } from './litestream3-prometheus-poller.ts';
 export type BackupCleanupMonitorFactoryOptions = {
     lc: LogContext;
     config: NormalizedZeroConfig;
     replicaFile: string;
-    changeStreamer: ChangeStreamerService;
     verifyBackupState?: BackupStateVerifier | undefined;
 };
-export declare function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, verifyBackupState, }: BackupCleanupMonitorFactoryOptions): BackupMonitor;
+export declare function createBackupCleanupMonitor({ lc, config, replicaFile, verifyBackupState, }: BackupCleanupMonitorFactoryOptions): BackupMonitor;
 //# sourceMappingURL=backup-cleanup-monitor-factory.d.ts.map

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;~~AAMpE~~,OAAO,~~EAAyB~~,aAAa,~~EAAC~~,MAAM,qBAAqB,CAAC;~~AAC1E~~,OAAO,~~KAAK,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAEhE,OAAO,~~EACL,KAAK,mBAAmB,EAEzB,MAAM,oCAAoC,CAAC;AAG5C,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,~~cAAc,EAAE,qBAAqB,CAAC;IACtC,~~iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,~~cAAc,EACd,~~iBAAiB,GAClB,EAAE,kCAAkC,GAAG,aAAa,CAyBpD"}
1	+ {"version":3,"file":"backup-cleanup-monitor-factory.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAKpE,OAAO,EAAC,aAAa,EAAoB,MAAM,qBAAqB,CAAC;AAErE,OAAO,EACL,KAAK,mBAAmB,EAEzB,MAAM,oCAAoC,CAAC;AAG5C,MAAM,MAAM,kCAAkC,GAAG;IAC/C,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC;CACrD,CAAC;AAEF,wBAAgB,0BAA0B,CAAC,EACzC,EAAE,EACF,MAAM,EACN,WAAW,EACX,iBAAiB,GAClB,EAAE,kCAAkC,GAAG,aAAa,CAyBpD"}

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js CHANGED Viewed

@@ -4,17 +4,17 @@ import { LitestreamSyncRequester } from "./litestream-sync-requester.js";
 import { Litestream3PrometheusPoller } from "./litestream3-prometheus-poller.js";
 import { ReplicaPoller } from "./replica-poller.js";
 //#region ../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts
-function createBackupCleanupMonitor({ lc, config, replicaFile, changeStreamer, verifyBackupState }) {
+function createBackupCleanupMonitor({ lc, config, replicaFile, verifyBackupState }) {
 	const { litestream, replica } = config;
 	const backupURL = backupDestinationURL(litestream);
-	let stream;
-	if (!backupURL) stream = new ReplicaPoller(lc, replicaFile).start();
-	else if (config.litestream.backupUsingV5) stream = new LitestreamSyncRequester(lc, replicaFile, { intervalMs: litestream.incrementalBackupIntervalSeconds * 1e3 }).start();
+	let poller;
+	if (!backupURL) poller = new ReplicaPoller(lc, replicaFile);
+	else if (config.litestream.backupUsingV5) poller = new LitestreamSyncRequester(lc, replicaFile, { intervalMs: litestream.incrementalBackupIntervalSeconds * 1e3 });
 	else {
 		const { port: metricsPort } = litestream;
-		stream = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file))).start();
+		poller = new Litestream3PrometheusPoller(lc, replicaFile, backupURL, `http://localhost:${metricsPort}/metrics`, verifyBackupState ?? (() => getLastBackupTime(lc, litestream, replica.file)));
 	}
-	return new BackupMonitor(lc, stream, changeStreamer, replicaFile);
+	return new BackupMonitor(lc, poller, replicaFile);
 }
 //#endregion
 export { createBackupCleanupMonitor };

package/out/zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport ~~type~~ {~~Source} from '../../types/streams.ts';\nimport {~~\n backupDestinationURL,\n getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {type ~~BackedUpWatermark, BackupMonitor~~} from './backup-monitor.ts';\nimport ~~type~~ {~~ChangeStreamerService} from './change-streamer.ts';\nimport {~~LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n type BackupStateVerifier,\n Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n lc: LogContext;\n config: NormalizedZeroConfig;\n replicaFile: string;\n ~~changeStreamer: ChangeStreamerService;\n~~ verifyBackupState?: BackupStateVerifier \| undefined;\n};\n\nexport function createBackupCleanupMonitor({\n lc,\n config,\n replicaFile,\n ~~changeStreamer,\n~~ verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n const {litestream, replica} = config;\n const backupURL = backupDestinationURL(litestream);\n\n let ~~stream~~: ~~Source<BackedUpWatermark>;\~~n\n if (!backupURL) {\n ~~stream~~ = new ReplicaPoller(lc, replicaFile)~~.start()~~;\n } else if (config.litestream.backupUsingV5) {\n ~~stream~~ = new LitestreamSyncRequester(lc, replicaFile, {\n intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n })~~.start()~~;\n } else {\n const {port: metricsPort} = litestream;\n ~~stream~~ = new Litestream3PrometheusPoller(\n lc,\n replicaFile,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n verifyBackupState ??\n (() => getLastBackupTime(lc, litestream, replica.file)),\n )~~.start()~~;\n }\n\n return new BackupMonitor(lc, ~~stream~~, ~~changeStreamer,~~ replicaFile);\n}\n"],"mappings":";;;;;;~~AAwBA~~,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,~~gBACA,~~qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW~~,CAAC,CAAC,MAAM~~;~~MAC7C~~,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC~~,CAAC,CAAC,MAAM~~;~~MACJ~~;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD~~,CAAC,CAAC,MAAM~~;~~CACV~~;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,~~gBAAgB,~~WAAW;~~AAClE~~"}
1	+ {"version":3,"file":"backup-cleanup-monitor-factory.js","names":[],"sources":["../../../../../../zero-cache/src/services/change-streamer/backup-cleanup-monitor-factory.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport type {NormalizedZeroConfig} from '../../config/normalize.ts';\nimport {\n backupDestinationURL,\n getLastBackupTime,\n} from '../litestream/commands.ts';\nimport {BackupMonitor, type BackupPoller} from './backup-monitor.ts';\nimport {LitestreamSyncRequester} from './litestream-sync-requester.ts';\nimport {\n type BackupStateVerifier,\n Litestream3PrometheusPoller,\n} from './litestream3-prometheus-poller.ts';\nimport {ReplicaPoller} from './replica-poller.ts';\n\nexport type BackupCleanupMonitorFactoryOptions = {\n lc: LogContext;\n config: NormalizedZeroConfig;\n replicaFile: string;\n verifyBackupState?: BackupStateVerifier \| undefined;\n};\n\nexport function createBackupCleanupMonitor({\n lc,\n config,\n replicaFile,\n verifyBackupState,\n}: BackupCleanupMonitorFactoryOptions): BackupMonitor {\n const {litestream, replica} = config;\n const backupURL = backupDestinationURL(litestream);\n\n let poller: BackupPoller;\n\n if (!backupURL) {\n poller = new ReplicaPoller(lc, replicaFile);\n } else if (config.litestream.backupUsingV5) {\n poller = new LitestreamSyncRequester(lc, replicaFile, {\n intervalMs: litestream.incrementalBackupIntervalSeconds * 1000,\n });\n } else {\n const {port: metricsPort} = litestream;\n poller = new Litestream3PrometheusPoller(\n lc,\n replicaFile,\n backupURL,\n `http://localhost:${metricsPort}/metrics`,\n verifyBackupState ??\n (() => getLastBackupTime(lc, litestream, replica.file)),\n );\n }\n\n return new BackupMonitor(lc, poller, replicaFile);\n}\n"],"mappings":";;;;;;AAqBA,SAAgB,2BAA2B,EACzC,IACA,QACA,aACA,qBACoD;CACpD,MAAM,EAAC,YAAY,YAAW;CAC9B,MAAM,YAAY,qBAAqB,UAAU;CAEjD,IAAI;CAEJ,IAAI,CAAC,WACH,SAAS,IAAI,cAAc,IAAI,WAAW;MACrC,IAAI,OAAO,WAAW,eAC3B,SAAS,IAAI,wBAAwB,IAAI,aAAa,EACpD,YAAY,WAAW,mCAAmC,IAC5D,CAAC;MACI;EACL,MAAM,EAAC,MAAM,gBAAe;EAC5B,SAAS,IAAI,4BACX,IACA,aACA,WACA,oBAAoB,YAAY,WAChC,4BACS,kBAAkB,IAAI,YAAY,QAAQ,IAAI,EACzD;CACF;CAEA,OAAO,IAAI,cAAc,IAAI,QAAQ,WAAW;AAClD"}

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts CHANGED Viewed

@@ -1,17 +1,56 @@
 import type { LogContext } from '@rocicorp/logger';
 import type { Source } from '../../types/streams.ts';
 import type { SingletonService } from '../service.ts';
-import type { ChangeStreamerService } from './change-streamer.ts';
 export type BackedUpWatermark = {
     watermark: string;
     writeTimeMs?: number | undefined;
     backupTimeMs: number;
 };
+/**
+ * Determines which watermarks of the replica are backed up, by whatever means
+ * the backup (or its absence) allows.
+ */
+export interface BackupPoller {
+    /**
+     * Starts polling, returning the stream of backed-up watermarks. Must only
+     * be called once.
+     */
+    start(): Source<BackedUpWatermark>;
+    /**
+     * Requests that the replica be backed up through `watermark` promptly,
+     * rather than at the poller's regular cadence, once the replica has reached
+     * it. The resulting watermark is emitted on the stream as usual. Pollers
+     * that cannot initiate backups do not implement this.
+     */
+    requestSync?(watermark: string): void;
+}
+/**
+ * The interface through which the change-streamer consumes backed-up
+ * watermarks and requests prompt backups (see {@link BackupMonitor}).
+ */
+export type BackupSource = Pick<BackupMonitor, 'backups' | 'requestSync'>;
+/**
+ * Relays the watermarks backed up by a {@link BackupPoller} to the
+ * change-streamer, which consumes them (see {@link backups}) and requests
+ * prompt backups (see {@link requestSync}) as needed.
+ */
 export declare class BackupMonitor implements SingletonService {
     #private;
     readonly id = "backup-monitor";
-    constructor(lc: LogContext, watermarks: Source<BackedUpWatermark>, changeStreamer: ChangeStreamerService, replicaFile: string);
+    constructor(lc: LogContext, poller: BackupPoller, replicaFile: string);
     firstBackupReceived(): Promise<void>;
+    /**
+     * The stream of backed-up watermarks, each later than the last, coalesced
+     * to the latest one that has not been consumed. It ends when the monitor
+     * stops.
+     */
+    backups(): Source<BackedUpWatermark>;
+    /**
+     * Requests that the replica be backed up through `watermark` promptly, if
+     * the poller supports it (see {@link BackupPoller.requestSync}). A no-op
+     * unless the monitor is running.
+     */
+    requestSync(watermark: string): void;
     run(): Promise<void>;
     stop(): Promise<void>;
 }

package/out/zero-cache/src/services/change-streamer/backup-monitor.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;~~AACnD~~,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;~~AACpD~~,~~OAAO,KAAK,EAAC,qBAAqB,EAAC,~~MAAM,~~sBAAsB,CAAC;AAEhE,~~MAAM,~~MAAM,~~iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;~~IAiB~~/B,~~YACE~~,EAAE,EAAE,UAAU,~~EACd~~,~~UAAU~~,EAAE,MAAM,CAAC,iBAAiB,CAAC,~~EACrC~~,~~cAAc~~,~~EAAE~~,~~qBAAqB~~,~~EACrC,WAAW,~~EAAE,MAAM,~~EAMpB~~;~~IAED,mBAAmB,kBAElB;~~IAEK,GAAG,~~kBA8BR~~;IAqDD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,~~CAGpB~~;CA+BF"}
1	+ {"version":3,"file":"backup-monitor.d.ts","sourceRoot":"","sources":["../../../../../../zero-cache/src/services/change-streamer/backup-monitor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAKjD,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,wBAAwB,CAAC;AAEnD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,eAAe,CAAC;AAEpD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,KAAK,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAAC;IAEnC;;;;;OAKG;IACH,WAAW,CAAC,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,GAAG,aAAa,CAAC,CAAC;AAE1E;;;;GAIG;AACH,qBAAa,aAAc,YAAW,gBAAgB;;IACpD,QAAQ,CAAC,EAAE,oBAAoB;IAwB/B,YAAY,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,EAIpE;IAED,mBAAmB,kBAElB;IAED;;;;OAIG;IACH,OAAO,IAAI,MAAM,CAAC,iBAAiB,CAAC,CAEnC;IAED;;;;OAIG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,QAY5B;IAEK,GAAG,kBAsCR;IAqDD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAKpB;CA+BF"}