@rocicorp/zero 1.11.0-canary.2 → 1.11.0-canary.21

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js

@@ -0,0 +1,113 @@
+import { replicationSlotPrefix } from "./schema/shard.js";
+import "../../../types/pg-versions.js";
+import { dropInactiveSlotsAndReplicas } from "./replication-slots.js";
+//#region ../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts
+var DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 300 * 1e3;
+var DEFAULT_POLL_INTERVAL_MS = 60 * 1e3;
+var ReplicationSlotCleanupMonitor = class {
+	#lc;
+	/** The shard constrains which slots are monitored (e.g. zero_0_*). */
+	#shard;
+	#upstream;
+	/**
+	* This slot that this task is using. Cleanup of slots is predicated on
+	* this slot remaining active, in order to guarantee at least one healthy
+	* slot in the shard.
+	*/
+	#currentSlot;
+	/**
+	* How long a slot is inactive (while the currentSlot is active) before it
+	* is eligible to be cleaned up. Because new slots are immediately connected
+	* to, even while an initial-sync or replica-restore is in progress, slots
+	* should only be inactive if no server is using it, or if a server was
+	* disconnected (e.g. a pg restart) and is in the process of reconnecting
+	* to it (see `keepSlotActiveUntilTakenOver`).
+	*
+	* As such, this timeout should be long enough to account for a server
+	* reconnecting to upstream. Note again that the cleanup monitor only runs
+	* while at least one replication slot is active, guaranteeing that upstream
+	* replication is working.
+	*/
+	#inactiveSlotCleanupTimeoutMs;
+	#pollIntervalMs;
+	/** Manually tracked inactive times for PG < 17. null for PG 17+ */
+	#firstInactiveTimes;
+	#pollTimer;
+	constructor(lc, shard, upstream, upstreamPgServerVersionNum, currentSlot, { inactiveSlotCleanupTimeoutMs, pollIntervalMs } = {}) {
+		this.#lc = lc.withContext("component", "replication-slot-cleanup-monitor");
+		this.#shard = shard;
+		this.#upstream = upstream;
+		this.#currentSlot = currentSlot;
+		this.#inactiveSlotCleanupTimeoutMs = inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;
+		this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+		this.#firstInactiveTimes = upstreamPgServerVersionNum >= 17e4 ? null : /* @__PURE__ */ new Map();
+	}
+	start() {
+		if (this.#pollTimer) {
+			this.#lc.warn?.(`monitor already started`);
+			return;
+		}
+		this.#pollTimer = setInterval(() => this.#poll(/* @__PURE__ */ new Date()).catch((e) => this.#lc.warn?.(`error checking replication slots`, e)), this.#pollIntervalMs);
+	}
+	stop() {
+		clearInterval(this.#pollTimer);
+		this.#pollTimer = void 0;
+		this.#firstInactiveTimes?.clear();
+	}
+	async #poll(now) {
+		if (!this.#pollTimer) return;
+		const slots = await this.getSlotsToCleanup(now);
+		if (slots.length) await dropInactiveSlotsAndReplicas(this.#lc, this.#upstream, this.#shard, slots);
+	}
+	async getSlotsToCleanup(now) {
+		const slotPoolPrefix = replicationSlotPrefix(this.#shard);
+		const slotsToCleanUp = [];
+		const trackedInactiveTimes = this.#firstInactiveTimes;
+		if (trackedInactiveTimes) {
+			const shardSlots = await this.#upstream`
+        SELECT slot_name AS slot, active FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"};
+      `;
+			for (const { slot, active } of shardSlots) {
+				if (slot === this.#currentSlot && !active) {
+					this.#lc.warn?.(`current slot ${this.#currentSlot} is not active. disabling cleanup.`);
+					this.stop();
+					return [];
+				}
+				if (active) trackedInactiveTimes.delete(slot);
+				else {
+					const inactiveSince = trackedInactiveTimes.get(slot);
+					if (!inactiveSince) trackedInactiveTimes.set(slot, now);
+					else if (now.getTime() - inactiveSince.getTime() >= this.#inactiveSlotCleanupTimeoutMs) {
+						this.#lc.info?.(`slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`);
+						slotsToCleanUp.push(slot);
+					}
+				}
+			}
+		} else {
+			const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;
+			const inactiveSlots = await this.#upstream`
+        SELECT slot_name AS slot, inactive_since AS "inactiveSince" FROM pg_replication_slots
+          WHERE slot_name LIKE ${slotPoolPrefix + "%"}
+            AND active = false
+            AND inactive_since IS NOT NULL;
+      `;
+			for (const { slot, inactiveSince } of inactiveSlots) {
+				if (slot === this.#currentSlot) {
+					this.#lc.warn?.(`current slot ${this.#currentSlot} is not active. disabling cleanup.`);
+					this.stop();
+					return [];
+				}
+				if (inactiveSince <= expiration) {
+					this.#lc.info?.(`slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`);
+					slotsToCleanUp.push(slot);
+				}
+			}
+		}
+		return slotsToCleanUp;
+	}
+};
+//#endregion
+export { ReplicationSlotCleanupMonitor };
+
+//# sourceMappingURL=replication-slot-cleanup-monitor.js.map

package/out/zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replication-slot-cleanup-monitor.js","names":["#lc","#shard","#upstream","#currentSlot","#inactiveSlotCleanupTimeoutMs","#pollIntervalMs","#firstInactiveTimes","#pollTimer","#poll"],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slot-cleanup-monitor.ts"],"sourcesContent":["import type {LogContext} from '@rocicorp/logger';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport type {PostgresDB} from '../../../types/pg.ts';\nimport type {ShardID} from '../../../types/shards.ts';\nimport {dropInactiveSlotsAndReplicas} from './replication-slots.ts';\nimport {replicationSlotPrefix} from './schema/shard.ts';\n\ntype Options = {\n  inactiveSlotCleanupTimeoutMs?: number | undefined;\n  pollIntervalMs?: number | undefined;\n};\n\nconst DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS = 5 * 60 * 1000;\nconst DEFAULT_POLL_INTERVAL_MS = 60 * 1000;\n\nexport class ReplicationSlotCleanupMonitor {\n  readonly #lc: LogContext;\n\n  /** The shard constrains which slots are monitored (e.g. zero_0_*). */\n  readonly #shard: ShardID;\n  readonly #upstream: PostgresDB;\n  /**\n   * This slot that this task is using. Cleanup of slots is predicated on\n   * this slot remaining active, in order to guarantee at least one healthy\n   * slot in the shard.\n   */\n  readonly #currentSlot: string;\n\n  /**\n   * How long a slot is inactive (while the currentSlot is active) before it\n   * is eligible to be cleaned up. Because new slots are immediately connected\n   * to, even while an initial-sync or replica-restore is in progress, slots\n   * should only be inactive if no server is using it, or if a server was\n   * disconnected (e.g. a pg restart) and is in the process of reconnecting\n   * to it (see `keepSlotActiveUntilTakenOver`).\n   *\n   * As such, this timeout should be long enough to account for a server\n   * reconnecting to upstream. Note again that the cleanup monitor only runs\n   * while at least one replication slot is active, guaranteeing that upstream\n   * replication is working.\n   */\n  readonly #inactiveSlotCleanupTimeoutMs: number;\n  readonly #pollIntervalMs: number;\n\n  /** Manually tracked inactive times for PG < 17. null for PG 17+ */\n  readonly #firstInactiveTimes: Map<string, Date> | null;\n\n  #pollTimer: NodeJS.Timeout | undefined;\n\n  constructor(\n    lc: LogContext,\n    shard: ShardID,\n    upstream: PostgresDB,\n    upstreamPgServerVersionNum: number,\n    currentSlot: string,\n    {inactiveSlotCleanupTimeoutMs, pollIntervalMs}: Options = {},\n  ) {\n    this.#lc = lc.withContext('component', 'replication-slot-cleanup-monitor');\n    this.#shard = shard;\n    this.#upstream = upstream;\n    this.#currentSlot = currentSlot;\n    this.#inactiveSlotCleanupTimeoutMs =\n      inactiveSlotCleanupTimeoutMs ?? DEFAULT_INACTIVE_SLOT_CLEANUP_TIMEOUT_MS;\n    this.#pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;\n    this.#firstInactiveTimes =\n      upstreamPgServerVersionNum >= PG_17 ? null : new Map();\n  }\n\n  start() {\n    if (this.#pollTimer) {\n      this.#lc.warn?.(`monitor already started`);\n      return;\n    }\n    this.#pollTimer = setInterval(\n      () =>\n        this.#poll(new Date()).catch(e =>\n          this.#lc.warn?.(`error checking replication slots`, e),\n        ),\n      this.#pollIntervalMs,\n    );\n  }\n\n  stop() {\n    clearInterval(this.#pollTimer);\n    this.#pollTimer = undefined;\n    this.#firstInactiveTimes?.clear();\n  }\n\n  async #poll(now: Date) {\n    if (!this.#pollTimer) {\n      return; // concurrently stopped\n    }\n    const slots = await this.getSlotsToCleanup(now);\n    if (slots.length) {\n      await dropInactiveSlotsAndReplicas(\n        this.#lc,\n        this.#upstream,\n        this.#shard,\n        slots,\n      );\n    }\n  }\n\n  async getSlotsToCleanup(now: Date): Promise<string[]> {\n    const slotPoolPrefix = replicationSlotPrefix(this.#shard);\n\n    const slotsToCleanUp: string[] = [];\n    const trackedInactiveTimes = this.#firstInactiveTimes;\n\n    if (trackedInactiveTimes) {\n      // For PG <17, each poll tracks and updates earliest time the slot was\n      // observed to be inactive.\n      const shardSlots = await this.#upstream<{slot: string; active: boolean}[]>\n      /*sql*/ `\n        SELECT slot_name AS slot, active FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n      `;\n      for (const {slot, active} of shardSlots) {\n        if (slot === this.#currentSlot && !active) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        if (active) {\n          trackedInactiveTimes.delete(slot);\n        } else {\n          const inactiveSince = trackedInactiveTimes.get(slot);\n          if (!inactiveSince) {\n            trackedInactiveTimes.set(slot, now);\n          } else if (\n            now.getTime() - inactiveSince.getTime() >=\n            this.#inactiveSlotCleanupTimeoutMs\n          ) {\n            this.#lc.info?.(\n              `slot ${slot} has been inactive since ${inactiveSince.toISOString()} and is eligible for cleanup`,\n            );\n            slotsToCleanUp.push(slot);\n          }\n        }\n      }\n    } else {\n      // PG 17+ is simpler as inactivity time is tracked in the table.\n      const expiration = now.getTime() - this.#inactiveSlotCleanupTimeoutMs;\n      const inactiveSlots = await this.#upstream<\n        {slot: string; inactiveSince: number}[]\n      > /*sql*/ `\n        SELECT slot_name AS slot, inactive_since AS \"inactiveSince\" FROM pg_replication_slots\n          WHERE slot_name LIKE ${slotPoolPrefix + '%'}\n            AND active = false\n            AND inactive_since IS NOT NULL;\n      `;\n      for (const {slot, inactiveSince} of inactiveSlots) {\n        if (slot === this.#currentSlot) {\n          this.#lc.warn?.(\n            `current slot ${this.#currentSlot} is not active. disabling cleanup.`,\n          );\n          this.stop();\n          return [];\n        }\n        // The slot has been inactive since at or before the expiration cutoff\n        // (now - timeout), i.e. inactive for at least the timeout duration.\n        if (inactiveSince <= expiration) {\n          this.#lc.info?.(\n            `slot ${slot} has been inactive since ${new Date(inactiveSince).toISOString()} and is eligible for cleanup`,\n          );\n          slotsToCleanUp.push(slot);\n        }\n      }\n    }\n    return slotsToCleanUp;\n  }\n}\n"],"mappings":";;;;AAYA,IAAM,2CAA2C,MAAS;AAC1D,IAAM,2BAA2B,KAAK;AAEtC,IAAa,gCAAb,MAA2C;CACzC;;CAGA;CACA;;;;;;CAMA;;;;;;;;;;;;;;CAeA;CACA;;CAGA;CAEA;CAEA,YACE,IACA,OACA,UACA,4BACA,aACA,EAAC,8BAA8B,mBAA2B,CAAC,GAC3D;EACA,KAAKA,MAAM,GAAG,YAAY,aAAa,kCAAkC;EACzE,KAAKC,SAAS;EACd,KAAKC,YAAY;EACjB,KAAKC,eAAe;EACpB,KAAKC,gCACH,gCAAgC;EAClC,KAAKC,kBAAkB,kBAAkB;EACzC,KAAKC,sBACH,8BAAA,OAAsC,uBAAO,IAAI,IAAI;CACzD;CAEA,QAAQ;EACN,IAAI,KAAKC,YAAY;GACnB,KAAKP,IAAI,OAAO,yBAAyB;GACzC;EACF;EACA,KAAKO,aAAa,kBAEd,KAAKC,sBAAM,IAAI,KAAK,CAAC,EAAE,OAAM,MAC3B,KAAKR,IAAI,OAAO,oCAAoC,CAAC,CACvD,GACF,KAAKK,eACP;CACF;CAEA,OAAO;EACL,cAAc,KAAKE,UAAU;EAC7B,KAAKA,aAAa,KAAA;EAClB,KAAKD,qBAAqB,MAAM;CAClC;CAEA,MAAME,MAAM,KAAW;EACrB,IAAI,CAAC,KAAKD,YACR;EAEF,MAAM,QAAQ,MAAM,KAAK,kBAAkB,GAAG;EAC9C,IAAI,MAAM,QACR,MAAM,6BACJ,KAAKP,KACL,KAAKE,WACL,KAAKD,QACL,KACF;CAEJ;CAEA,MAAM,kBAAkB,KAA8B;EACpD,MAAM,iBAAiB,sBAAsB,KAAKA,MAAM;EAExD,MAAM,iBAA2B,CAAC;EAClC,MAAM,uBAAuB,KAAKK;EAElC,IAAI,sBAAsB;GAGxB,MAAM,aAAa,MAAM,KAAKJ,SACtB;;iCAEmB,iBAAiB,IAAI;;GAEhD,KAAK,MAAM,EAAC,MAAM,YAAW,YAAY;IACvC,IAAI,SAAS,KAAKC,gBAAgB,CAAC,QAAQ;KACzC,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IACA,IAAI,QACF,qBAAqB,OAAO,IAAI;SAC3B;KACL,MAAM,gBAAgB,qBAAqB,IAAI,IAAI;KACnD,IAAI,CAAC,eACH,qBAAqB,IAAI,MAAM,GAAG;UAC7B,IACL,IAAI,QAAQ,IAAI,cAAc,QAAQ,KACtC,KAAKC,+BACL;MACA,KAAKJ,IAAI,OACP,QAAQ,KAAK,2BAA2B,cAAc,YAAY,EAAE,6BACtE;MACA,eAAe,KAAK,IAAI;KAC1B;IACF;GACF;EACF,OAAO;GAEL,MAAM,aAAa,IAAI,QAAQ,IAAI,KAAKI;GACxC,MAAM,gBAAgB,MAAM,KAAKF,SAEvB;;iCAEiB,iBAAiB,IAAI;;;;GAIhD,KAAK,MAAM,EAAC,MAAM,mBAAkB,eAAe;IACjD,IAAI,SAAS,KAAKC,cAAc;KAC9B,KAAKH,IAAI,OACP,gBAAgB,KAAKG,aAAa,mCACpC;KACA,KAAK,KAAK;KACV,OAAO,CAAC;IACV;IAGA,IAAI,iBAAiB,YAAY;KAC/B,KAAKH,IAAI,OACP,QAAQ,KAAK,2BAA2B,IAAI,KAAK,aAAa,EAAE,YAAY,EAAE,6BAChF;KACA,eAAe,KAAK,IAAI;IAC1B;GACF;EACF;EACA,OAAO;CACT;AACF"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts

@@ -3,7 +3,8 @@ import type { LogContext } from '@rocicorp/logger';
 import type postgres from 'postgres';
 import { type PostgresDB } from '../../../types/pg.ts';
 import { type ShardID } from '../../../types/shards.ts';
-import { type BackupOptions } from './schema/shard.ts';
+import { type ReplicaStage } from './schema/replica-stage-enum.ts';
+import { type BackupOptions, type ReplicaState } from './schema/shard.ts';
 export type ReplicationSlot = {
     slot_name: string;
     consistent_point: string;
@@ -14,6 +15,12 @@ export type ReplicationSlotResult<T> = {
     slot: ReplicationSlot;
     capturedSnapshot: T;
     initialSession: Readable;
+    replica: ReplicaState;
+};
+export type ReservedSlot = {
+    slot: string;
+    reservation: Readable;
+    replica: ReplicaState;
 };
 export type CreateSlotSpec = {
     slotName: string;
@@ -61,22 +68,21 @@ export declare function createReplicationSlot(lc: LogContext, session: postgres.
  * PostgresChangeSource takes over the slot to process the stream in earnest.
  * It is also returned for cleanup in the case of failures (and for control
  * in unit tests).
+ *
+ * For replica's created in the `Restore` stage, the `generation` is
+ * initialized to an empty string instead of the slot's consistent point,
+ * since the slot is not being used for initial sync. It is the
+ * responsibiliy of the caller to call {@link initRestoreReplica} when
+ * the source replica has been selected for restoring, which will carry
+ * over the `generation` and other relevant values from the source.
  */
-export declare function createReplicaAndSlot<T>(lc: LogContext, sql: PostgresDB, sessionName: string, shard: ShardID, replicaID: string, failover: boolean, backupOptions: BackupOptions, captureSnapshot: (snapshot: string) => Promise<T>): Promise<ReplicationSlotResult<T>>;
+export declare function createReplicaAndSlot<T>(lc: LogContext, sql: PostgresDB, sessionName: string, shard: ShardID, epoch: number, replicaID: string, failover: boolean, backupOptions: BackupOptions, captureSnapshot: (snapshot: string) => Promise<T>, stage: ReplicaStage): Promise<ReplicationSlotResult<T>>;
 /**
- * Deletes "old" replicas (i.e. those with a lower rank than the current)
- * and attempts to drop replication slots that are not associated with any
- * replica.
- *
- * If a slot could not be dropped because there is still an active subscriber,
- * it will be reflected in the `draining` count that is returned. When there
- * are draining slots, the method should be retried until all orphaned slots
- * have been dropped.
+ * Claims an inactive slot for use by this task, opening a replication session
+ * to mark it as active. The process should follow up by taking over the slot
+ * via the PostgresChangeSource.
  */
-export declare function dropOldReplicasAndSlots(lc: LogContext, sql: PostgresDB, shard: ShardID, beforeRank: bigint): Promise<{
-    dropped: number;
-    active: number;
-    draining: number;
-}>;
+export declare function claimSlotForResumption(lc: LogContext, sql: PostgresDB, shard: ShardID, sessionName: string, slot: string): Promise<ReservedSlot | null>;
+export declare function dropInactiveSlotsAndReplicas(lc: LogContext, sql: PostgresDB, shard: ShardID, slots: string[]): Promise<void>;
 export declare function slotPoolSuffix(n: number): string;
 //# sourceMappingURL=replication-slots.d.ts.map

package/out/zero-cache/src/services/change-source/pg/replication-slots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAErC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAOtE,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAoBF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,GACrC,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CAwC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAChD,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAuFnC;AAED;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,CAAC,CAY9D;AA+DD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}
+{"version":3,"file":"replication-slots.d.ts","sourceRoot":"","sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAK1C,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AAIrC,OAAO,EAAkB,KAAK,UAAU,EAAC,MAAM,sBAAsB,CAAC;AACtE,OAAO,EAAiB,KAAK,OAAO,EAAC,MAAM,0BAA0B,CAAC;AAQtE,OAAO,EAGL,KAAK,YAAY,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAOL,KAAK,aAAa,EAClB,KAAK,YAAY,EAClB,MAAM,mBAAmB,CAAC;AAG3B,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,qBAAqB,CAAC,CAAC,IAAI;IACrC,IAAI,EAAE,eAAe,CAAC;IACtB,gBAAgB,EAAE,CAAC,CAAC;IACpB,cAAc,EAAE,QAAQ,CAAC;IACzB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,QAAQ,CAAC;IACtB,OAAO,EAAE,YAAY,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,MAAM,CAAC;IAGjB,QAAQ,CAAC,EAAE,OAAO,CAAC;IAInB,SAAS,CAAC,EAAE,OAAO,CAAC;IAGpB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAoBF,wBAAsB,qBAAqB,CACzC,EAAE,EAAE,UAAU,EACd,OAAO,EAAE,QAAQ,CAAC,GAAG,EACrB,EACE,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,WAAoC,GACrC,EAAE,cAAc,GAChB,OAAO,CAAC,eAAe,CAAC,CA8C1B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AACH,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,OAAO,EACjB,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EACjD,KAAK,EAAE,YAAY,GAClB,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CA+GnC;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAwD9B;AAED,wBAAgB,4BAA4B,CAC1C,EAAE,EAAE,UAAU,EACd,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,MAAM,EAAE,iBA+BhB;AA+DD,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,UAUvC"}

package/out/zero-cache/src/services/change-source/pg/replication-slots.js

@@ -1,8 +1,12 @@
+import { must } from "../../../../../shared/src/must.js";
 import { upstreamSchema } from "../../../types/shards.js";
 import { isPostgresError } from "../../../types/pg.js";
 import { toBigInt, toStateVersionString } from "./lsn.js";
 import { runTx } from "../../../db/run-transaction.js";
-import { createReplica, metadataPublicationName, replicationSlotExpression, replicationSlotPrefix } from "./schema/shard.js";
+import "./schema/replica-stage-enum.js";
+import { createReplica, getReplicaState, getRestoreCandidates, metadataPublicationName, replicationSlotExpression, replicationSlotPrefix } from "./schema/shard.js";
+import { AutoResetSignal } from "../../change-streamer/schema/tables.js";
+import "../../../types/pg-versions.js";
 import { orTimeout } from "../../../types/timeout.js";
 import { createReplicationSessionFor, keepSlotActiveUntilTakenOver } from "./logical-replication/stream.js";
 import { PG_CONFIGURATION_LIMIT_EXCEEDED, PG_INSUFFICIENT_PRIVILEGE } from "@drdgvhbh/postgres-error-codes";
@@ -11,8 +15,11 @@ var CREATE_REPLICATION_SLOT_TIMEOUT_MS = 3e4;
 var SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1e3;
 async function createReplicationSlot(lc, session, { slotName, failover, temporary, lockTimeout = SERVER_LOCK_TIMEOUT_MS }) {
 	await session.unsafe(`SET lock_timeout = ${lockTimeout}`);
+	const { pgVersion } = (await session.unsafe(`
+      SELECT current_setting('server_version_num') as "pgVersion";
+  `))[0];
 	const maybeTemporary = temporary ? "TEMPORARY" : "";
-	const options = failover ? "(FAILOVER)" : "";
+	const options = failover && pgVersion >= 17e4 ? "(FAILOVER)" : "";
 	const raced = await orTimeout(session.unsafe(`
     CREATE_REPLICATION_SLOT "${slotName}" ${maybeTemporary} LOGICAL pgoutput ${options}`), CREATE_REPLICATION_SLOT_TIMEOUT_MS);
 	if (raced === "timed-out") {
@@ -62,8 +69,15 @@ async function createReplicationSlot(lc, session, { slotName, failover, temporar
 * PostgresChangeSource takes over the slot to process the stream in earnest.
 * It is also returned for cleanup in the case of failures (and for control
 * in unit tests).
+*
+* For replica's created in the `Restore` stage, the `generation` is
+* initialized to an empty string instead of the slot's consistent point,
+* since the slot is not being used for initial sync. It is the
+* responsibiliy of the caller to call {@link initRestoreReplica} when
+* the source replica has been selected for restoring, which will carry
+* over the `generation` and other relevant values from the source.
 */
-async function createReplicaAndSlot(lc, sql, sessionName, shard, replicaID, failover, backupOptions, captureSnapshot) {
+async function createReplicaAndSlot(lc, sql, sessionName, shard, epoch, replicaID, failover, backupOptions, captureSnapshot, stage) {
 	const replicationSession = createReplicationSessionFor(sql, sessionName);
 	const lockName = replicationSlotManagementLock(shard);
 	const slotPoolPrefix = replicationSlotPrefix(shard);
@@ -73,6 +87,13 @@ async function createReplicaAndSlot(lc, sql, sessionName, shard, replicaID, fail
 		try {
 			return await runTx(sql, async (tx) => {
 				await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
+				if (stage === 0) {
+					const others = await getRestoreCandidates(lc, tx, shard, epoch);
+					if (others.length) {
+						const [{ id, stage, active }] = others;
+						if (stage === 0 && active) throw new AutoResetSignal(`another replica (${id}) is performing initial sync`);
+					}
+				}
 				const names = await tx`
           SELECT slot_name as name FROM pg_replication_slots
             WHERE slot_name LIKE ${slotPoolPrefix + "%"};
@@ -91,11 +112,12 @@ async function createReplicaAndSlot(lc, sql, sessionName, shard, replicaID, fail
 				});
 				const capturedSnapshot = await captureSnapshot(slot.snapshot_name);
 				const initialSession = await keepSlotActiveUntilTakenOver(lc, replicationSession, slot.slot_name, metadataPublicationName(shard.appID, shard.shardNum), toBigInt(slot.consistent_point));
-				await createReplica(tx, shard, replicaID, slot.slot_name, toStateVersionString(slot.consistent_point), backupOptions);
+				await createReplica(tx, shard, replicaID, slot.slot_name, epoch, stage === 0 ? toStateVersionString(slot.consistent_point) : "", backupOptions, stage);
 				return {
 					slot,
 					capturedSnapshot,
-					initialSession
+					initialSession,
+					replica: must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} was not created`)
 				};
 			});
 		} catch (e) {
@@ -121,26 +143,67 @@ async function createReplicaAndSlot(lc, sql, sessionName, shard, replicaID, fail
 	}
 }
 /**
-* Deletes "old" replicas (i.e. those with a lower rank than the current)
-* and attempts to drop replication slots that are not associated with any
-* replica.
-*
-* If a slot could not be dropped because there is still an active subscriber,
-* it will be reflected in the `draining` count that is returned. When there
-* are draining slots, the method should be retried until all orphaned slots
-* have been dropped.
+* Claims an inactive slot for use by this task, opening a replication session
+* to mark it as active. The process should follow up by taking over the slot
+* via the PostgresChangeSource.
 */
-async function dropOldReplicasAndSlots(lc, sql, shard, beforeRank) {
+async function claimSlotForResumption(lc, sql, shard, sessionName, slot) {
+	const replicationSession = createReplicationSessionFor(sql, sessionName);
+	const lockName = replicationSlotManagementLock(shard);
 	const replicasTable = `${upstreamSchema(shard)}.replicas`;
-	const oldReplicas = await sql`
-    SELECT id, rank::float8, slot, version, "initialSyncContext", "subscriberContext"
-     FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};
-  `;
-	if (oldReplicas.length) {
-		lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, { oldReplicas });
-		await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;
+	try {
+		const reserved = await runTx(sql, async (tx) => {
+			await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
+			const replicas = await tx`
+      SELECT slot_name as slot, active, replica.id as "replicaID", confirmed_flush_lsn as lsn
+        FROM pg_replication_slots
+        JOIN ${tx(replicasTable)} replica on slot_name = slot
+        WHERE slot_name = ${slot};
+    `;
+			if (replicas.length === 0) {
+				lc.warn?.(`no replica found for slot ${slot}`);
+				return null;
+			}
+			const [{ active, replicaID, lsn }] = replicas;
+			if (active) {
+				lc.warn?.(`replica ${replicaID} for slot ${slot} is active and cannot be claimed`);
+				return null;
+			}
+			await tx`
+        UPDATE ${tx(replicasTable)} SET "stage" = ${2} WHERE id = ${replicaID};
+      `;
+			return {
+				slot,
+				replica: must(await getReplicaState(tx, shard, replicaID), `replica ${replicaID} disappeared`),
+				reservation: await keepSlotActiveUntilTakenOver(lc, replicationSession, slot, metadataPublicationName(shard.appID, shard.shardNum), toBigInt(lsn))
+			};
+		});
+		lc.info?.(`successfully claimed slot ${slot}`);
+		return reserved;
+	} catch (e) {
+		lc.error?.(`unable to claim replication slot ${slot}`, e);
+		await replicationSession.end();
+		return null;
 	}
-	return dropUnclaimedSlots(lc, sql, shard);
+}
+function dropInactiveSlotsAndReplicas(lc, sql, shard, slots) {
+	const lockName = replicationSlotManagementLock(shard);
+	const replicasTable = `${upstreamSchema(shard)}.replicas`;
+	return runTx(sql, async (tx) => {
+		await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;
+		const dropped = await tx`
+      SELECT slot_name as slot, replica.id as "replicaID", pg_drop_replication_slot(slot_name) 
+        FROM pg_replication_slots
+        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot
+        WHERE slot_name IN ${tx(slots)} AND NOT active;
+    `;
+		if (dropped.length) lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, { dropped });
+		const replicas = dropped.map(({ replicaID }) => replicaID).filter((id) => id !== null);
+		if (replicas.length) {
+			await tx`DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;
+			lc.info?.(`deleted ${replicas.length} old replica(s)`, { replicas });
+		}
+	});
 }
 function dropUnclaimedSlots(lc, sql, shard) {
 	const lockName = replicationSlotManagementLock(shard);
@@ -190,6 +253,6 @@ function replicationSlotManagementLock(shard) {
 	return `replication-slot-management:${shard.appID}_${shard.shardNum}`;
 }
 //#endregion
-export { createReplicaAndSlot, createReplicationSlot, dropOldReplicasAndSlots };
+export { claimSlotForResumption, createReplicaAndSlot, createReplicationSlot, dropInactiveSlotsAndReplicas };
 
 //# sourceMappingURL=replication-slots.js.map

package/out/zero-cache/src/services/change-source/pg/replication-slots.js.map

@@ -1 +1 @@
-{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {\n  createReplicationSessionFor,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  createReplica,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: must be false if pgVersion < PG_17. Caller must verify.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n        `Crashing to force a clean restart.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n  // and only closed in this function if an error occurrs.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        const initialSession = await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot.slot_name,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(slot.consistent_point),\n        );\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          toStateVersionString(slot.consistent_point),\n          backupOptions,\n        );\n\n        return {slot, capturedSnapshot, initialSession};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      // Otherwise, clean up any created slot if something went wrong.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await replicationSession.end();\n        await sql`SELECT pg_drop_replication_slot(${slotName})`;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Deletes \"old\" replicas (i.e. those with a lower rank than the current)\n * and attempts to drop replication slots that are not associated with any\n * replica.\n *\n * If a slot could not be dropped because there is still an active subscriber,\n * it will be reflected in the `draining` count that is returned. When there\n * are draining slots, the method should be retried until all orphaned slots\n * have been dropped.\n */\nexport async function dropOldReplicasAndSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  beforeRank: bigint,\n): Promise<{dropped: number; active: number; draining: number}> {\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n  const oldReplicas = await sql`\n    SELECT id, rank::float8, slot, version, \"initialSyncContext\", \"subscriberContext\"\n     FROM ${sql(replicasTable)} WHERE rank < ${beforeRank};\n  `;\n  if (oldReplicas.length) {\n    lc.info?.(`Deleting ${oldReplicas.length} old replica(s)`, {oldReplicas});\n    await sql`DELETE FROM ${sql(replicasTable)} WHERE rank < ${beforeRank}`;\n  }\n\n  return dropUnclaimedSlots(lc, sql, shard);\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;AA4DA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,0BAEU;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,WAAW,eAAe;CAI1C,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EAGzB,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,qCAEjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA0CA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,WACA,UACA,eACA,iBACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAG1D,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;IACF,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,qBAAqB,KAAK,gBAAgB,GAC1C,aACF;IAEA,OAAO;KAAC;KAAM;KAAkB;IAAc;GAChD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAEA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,GAAG,mCAAmC,SAAS;GACvD;GACA,MAAM;EACR;CACF;AACF;;;;;;;;;;;AAYA,eAAsB,wBACpB,IACA,KACA,OACA,YAC8D;CAC9D,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAC/C,MAAM,cAAc,MAAM,GAAG;;YAEnB,IAAI,aAAa,EAAE,gBAAgB,WAAW;;CAExD,IAAI,YAAY,QAAQ;EACtB,GAAG,OAAO,YAAY,YAAY,OAAO,kBAAkB,EAAC,YAAW,CAAC;EACxE,MAAM,GAAG,eAAe,IAAI,aAAa,EAAE,gBAAgB;CAC7D;CAEA,OAAO,mBAAmB,IAAI,KAAK,KAAK;AAC1C;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAG3D,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}
+{"version":3,"file":"replication-slots.js","names":[],"sources":["../../../../../../../zero-cache/src/services/change-source/pg/replication-slots.ts"],"sourcesContent":["import type {Readable} from 'node:stream';\nimport {\n  PG_CONFIGURATION_LIMIT_EXCEEDED,\n  PG_INSUFFICIENT_PRIVILEGE,\n} from '@drdgvhbh/postgres-error-codes';\nimport type {LogContext} from '@rocicorp/logger';\nimport type postgres from 'postgres';\nimport {must} from '../../../../../shared/src/must.ts';\nimport {runTx} from '../../../db/run-transaction.ts';\nimport {PG_17} from '../../../types/pg-versions.ts';\nimport {isPostgresError, type PostgresDB} from '../../../types/pg.ts';\nimport {upstreamSchema, type ShardID} from '../../../types/shards.ts';\nimport {orTimeout} from '../../../types/timeout.ts';\nimport {AutoResetSignal} from '../../change-streamer/schema/tables.ts';\nimport {\n  createReplicationSessionFor,\n  keepSlotActiveUntilTakenOver,\n} from './logical-replication/stream.ts';\nimport {toBigInt, toStateVersionString} from './lsn.ts';\nimport {\n  InitialSync,\n  Restore,\n  type ReplicaStage,\n} from './schema/replica-stage-enum.ts';\nimport {\n  createReplica,\n  getReplicaState,\n  getRestoreCandidates,\n  metadataPublicationName,\n  replicationSlotExpression,\n  replicationSlotPrefix,\n  type BackupOptions,\n  type ReplicaState,\n} from './schema/shard.ts';\n\n// Record returned by `CREATE_REPLICATION_SLOT`\nexport type ReplicationSlot = {\n  slot_name: string;\n  consistent_point: string;\n  snapshot_name: string;\n  output_plugin: string;\n};\n\nexport type ReplicationSlotResult<T> = {\n  slot: ReplicationSlot;\n  capturedSnapshot: T;\n  initialSession: Readable;\n  replica: ReplicaState;\n};\n\nexport type ReservedSlot = {\n  slot: string;\n  reservation: Readable;\n  replica: ReplicaState;\n};\n\nexport type CreateSlotSpec = {\n  slotName: string;\n\n  // Note: ignored if pgVersion < PG_17.\n  failover?: boolean;\n\n  // Create a temporary slot (i.e. not persisted, and automatically\n  // cleaned up when the replication session ends).\n  temporary?: boolean;\n\n  // For overriding in tests.\n  lockTimeout?: number;\n};\n\n// When creating a replication slot, Postgres waits for open transactions\n// to complete before reserving a consistent_point (LSN) in the WAL and creating\n// a matching transaction snapshot. As such, it can technically take an arbitrary\n// amount of time (e.g. DDL operations, table-wide operations, etc.).\n//\n// However, to detect pathological situations, bound the amount of time that\n// the server waits for replication slot creation, so that a continual failure to\n// create a replication slot is surfaced by errors / alerts.\nconst CREATE_REPLICATION_SLOT_TIMEOUT_MS = 30_000;\n\n// The lock_timeout is set 1s before the client-side orTimeout so that\n// Postgres reliably aborts first and tears down the walsender cleanly.\n// The client-side timeout remains as a fallback for network-level failures.\nconst SERVER_LOCK_TIMEOUT_MS = CREATE_REPLICATION_SLOT_TIMEOUT_MS - 1_000;\n\n// Note: The replication connection does not support the extended query protocol,\n//       so all commands must be sent using sql.unsafe(). This is technically safe\n//       because all placeholder values are under our control (i.e. \"slotName\").\nexport async function createReplicationSlot(\n  lc: LogContext,\n  session: postgres.Sql,\n  {\n    slotName,\n    failover,\n    temporary,\n    lockTimeout = SERVER_LOCK_TIMEOUT_MS,\n  }: CreateSlotSpec,\n): Promise<ReplicationSlot> {\n  // CREATE_REPLICATION_SLOT can hang indefinitely waiting for long-running\n  // transactions to finish: internally it calls SnapBuildWaitSnapshot →\n  // XactLockTableWait → LockAcquire on each running XID. statement_timeout\n  // does NOT apply to replication commands, but lock_timeout does (it governs\n  // the heavyweight lock wait inside LockAcquire). Setting it here causes\n  // Postgres to raise ERRCODE_LOCK_NOT_AVAILABLE and cleanly tear down the\n  // walsender, rather than relying solely on the client-side orTimeout\n  // which can leave an orphaned backend.\n  //\n  // An orphaned walsender is actively harmful: by this point the replication\n  // slot has already been created and is pinning WAL retention and catalog_xmin.\n  // Worse, the slot is marked `active` (the walsender PID is still alive), so\n  // the existing cleanup code (which drops inactive slots on retry) can't\n  // reclaim it. Without lock_timeout the orphan persists until TCP keepalive\n  // fires (~2h default) or the blocking transaction finishes.\n  await session.unsafe(`SET lock_timeout = ${lockTimeout}`);\n\n  const {pgVersion} = (\n    await session.unsafe<{pgVersion: number}[]>(`\n      SELECT current_setting('server_version_num') as \"pgVersion\";\n  `)\n  )[0];\n\n  const maybeTemporary = temporary ? 'TEMPORARY' : '';\n  const options = failover && pgVersion >= PG_17 ? '(FAILOVER)' : '';\n  const createSlot = session.unsafe<ReplicationSlot[]>(/*sql*/ `\n    CREATE_REPLICATION_SLOT \"${slotName}\" ${maybeTemporary} LOGICAL pgoutput ${options}`);\n\n  const raced = await orTimeout(createSlot, CREATE_REPLICATION_SLOT_TIMEOUT_MS);\n  if (raced === 'timed-out') {\n    // Create slot can block indefinitely waiting for old transactions. End\n    // this connection in the background and fail fast so the process restarts.\n    void session\n      .end()\n      .catch(e =>\n        lc.warn?.(`Error closing timed out replication slot session`, e),\n      );\n    throw new Error(\n      `Timed out after ${CREATE_REPLICATION_SLOT_TIMEOUT_MS} ms creating replication slot ${slotName}. ` +\n        `Crashing to force a clean restart.`,\n    );\n  }\n  const [slot] = raced;\n  lc.info?.(`Created replication slot ${slotName}`, slot);\n  return slot;\n}\n\n/**\n * Replica and slot creation involves two sessions for proper coordination\n * with other replica management logic:\n *\n * * A normal transaction is started and acquires an advisory lock for\n *   replica slot management. This is the same lock that cleanup logic\n *   acquires before cleaning up replication slots.\n * * With the lock held, a new replication slot is created in a\n *   replication session. The API of CREATE_REPLICATION_SLOT is such\n *   that it cannot be done in a transaction, and cannot be followed by\n *   any writes, or else its snapshot (which is needed for initial sync)\n *   would be invalidated.\n * * Once the slot is created, the slot and replica information are recorded\n *   in the `replicas` table before releasing the lock.\n *\n * This locking ensures that:\n * 1. multiple replication managers attempting to create a replication slot\n *    will not use the same name for the replication slot (which is selected\n *    from a pool of reused names).\n * 2. Running replication managers (which use an earlier replica of a lower\n *    rank) will not delete the new slot during their cleanup logic, since\n *    the slot will belong to a replica of a higher rank.\n *\n * When the replication slot is created, `captureSnapshot` callback is first\n * run while the snapshot at the consistent point is available. The callback\n * must capture the snapshot in postgres (e.g. `SET TRANSACTION SNAPSHOT`)\n * in order to use it. Once the callback completes, a placeholder replication\n * session (i.e. that does not consume any messages) is started, in order to\n * reserve the slot by mark it \"active\", effectively distinguishing the\n * initializing session from a failed or abandoned session.\n *\n * Note that starting the replication session releases the initial snapshot,\n * which is why the callback must capture it synchronously, before the\n * replication session is started.\n *\n * The placeholder replication session is closed when the\n * PostgresChangeSource takes over the slot to process the stream in earnest.\n * It is also returned for cleanup in the case of failures (and for control\n * in unit tests).\n *\n * For replica's created in the `Restore` stage, the `generation` is\n * initialized to an empty string instead of the slot's consistent point,\n * since the slot is not being used for initial sync. It is the\n * responsibiliy of the caller to call {@link initRestoreReplica} when\n * the source replica has been selected for restoring, which will carry\n * over the `generation` and other relevant values from the source.\n */\nexport async function createReplicaAndSlot<T>(\n  lc: LogContext,\n  sql: PostgresDB,\n  sessionName: string,\n  shard: ShardID,\n  epoch: number,\n  replicaID: string,\n  failover: boolean,\n  backupOptions: BackupOptions,\n  captureSnapshot: (snapshot: string) => Promise<T>,\n  stage: ReplicaStage,\n): Promise<ReplicationSlotResult<T>> {\n  // Note: The replicationSession is closed by keepSlotActiveUntilTakenOver,\n  // and only closed in this function if an error occurrs.\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const slotPoolPrefix = replicationSlotPrefix(shard);\n\n  for (let first = true; ; first = false) {\n    await dropUnclaimedSlots(lc, sql, shard);\n\n    let slotName: string | undefined;\n    try {\n      return await runTx(sql, async tx => {\n        await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n        if (stage === InitialSync) {\n          // With the lock acquired, ensure that only one initial sync is\n          // active at a time. getRestoreCandidates() orders its results\n          // with stage = InitialSync (and active = true) first.\n          const others = await getRestoreCandidates(lc, tx, shard, epoch);\n          if (others.length) {\n            const [{id, stage, active}] = others;\n            if (stage === InitialSync && active) {\n              throw new AutoResetSignal(\n                `another replica (${id}) is performing initial sync`,\n              );\n            }\n          }\n        }\n\n        // Pick an available slotName from the slotPoolPrefix pool.\n        const names = await tx<{name: string}[]> /*sql*/ `\n          SELECT slot_name as name FROM pg_replication_slots\n            WHERE slot_name LIKE ${slotPoolPrefix + '%'};\n        `.values();\n        const inUse = new Set(names.flat());\n        for (let next = 0; ; next++) {\n          const candidateName = `${slotPoolPrefix}${slotPoolSuffix(next)}`;\n          if (!inUse.has(candidateName)) {\n            slotName = candidateName;\n            break;\n          }\n        }\n\n        const slot = await createReplicationSlot(lc, replicationSession, {\n          slotName,\n          failover,\n        });\n        const capturedSnapshot = await captureSnapshot(slot.snapshot_name);\n        const initialSession = await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot.slot_name,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(slot.consistent_point),\n        );\n\n        await createReplica(\n          tx,\n          shard,\n          replicaID,\n          slot.slot_name,\n          epoch,\n          stage === InitialSync\n            ? toStateVersionString(slot.consistent_point)\n            : '', // initialized with initRestoreReplica\n          backupOptions,\n          stage,\n        );\n\n        const replica = must(\n          await getReplicaState(tx, shard, replicaID),\n          `replica ${replicaID} was not created`,\n        );\n\n        return {slot, capturedSnapshot, initialSession, replica};\n      });\n    } catch (e) {\n      if (first && isPostgresError(e, PG_INSUFFICIENT_PRIVILEGE)) {\n        // Some Postgres variants (e.g. Google Cloud SQL) require that\n        // the user have the REPLICATION role in order to create a slot.\n        // Note that this must be done by the upstreamDB connection, and\n        // does not work in the replicationSession itself.\n        await sql`ALTER ROLE current_user WITH REPLICATION`;\n        lc.info?.(`Added the REPLICATION role to database user`);\n        continue;\n      }\n      // Note: This is currently manually tested since max_replication_slots\n      //       is a PG startup parameter that other tests depend on.\n      // TODO: Figure out a way to unit test this (with the full PG setup).\n      if (first && isPostgresError(e, PG_CONFIGURATION_LIMIT_EXCEEDED)) {\n        lc.warn?.(\n          `Reached max replication slots. Attempting to clean up unused slots`,\n          e,\n        );\n        // Drop any inactive replicas from failed initial syncs (e.g. inactive slots).\n        const replicasTable = `${upstreamSchema(shard)}.replicas`;\n        await sql`\n          DELETE FROM ${sql(replicasTable)} USING pg_replication_slots slots\n            WHERE replicas.slot = slots.slot_name AND NOT slots.active`;\n        continue; // then let dropUnclaimedSlots() perform its cleanup\n      }\n      // Otherwise, clean up any created slot if something went wrong.\n      if (slotName) {\n        lc.warn?.(`deleting slot ${slotName} due to error`, e);\n        await replicationSession.end();\n        await sql`SELECT pg_drop_replication_slot(${slotName})`;\n      }\n      throw e;\n    }\n  }\n}\n\n/**\n * Claims an inactive slot for use by this task, opening a replication session\n * to mark it as active. The process should follow up by taking over the slot\n * via the PostgresChangeSource.\n */\nexport async function claimSlotForResumption(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  sessionName: string,\n  slot: string,\n): Promise<ReservedSlot | null> {\n  const replicationSession = createReplicationSessionFor(sql, sessionName);\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  try {\n    const reserved = await runTx(sql, async tx => {\n      await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n      const replicas = await tx<\n        {slot: string; active: boolean; replicaID: string; lsn: string}[]\n      > /*sql*/ `\n      SELECT slot_name as slot, active, replica.id as \"replicaID\", confirmed_flush_lsn as lsn\n        FROM pg_replication_slots\n        JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name = ${slot};\n    `;\n      if (replicas.length === 0) {\n        lc.warn?.(`no replica found for slot ${slot}`);\n        return null;\n      }\n      const [{active, replicaID, lsn}] = replicas;\n      if (active) {\n        lc.warn?.(\n          `replica ${replicaID} for slot ${slot} is active and cannot be claimed`,\n        );\n        return null;\n      }\n      // Mark the replica as Restoring.\n      await tx`\n        UPDATE ${tx(replicasTable)} SET \"stage\" = ${Restore} WHERE id = ${replicaID};\n      `;\n      const replica = must(\n        await getReplicaState(tx, shard, replicaID),\n        `replica ${replicaID} disappeared`,\n      );\n\n      return {\n        slot,\n        replica,\n        reservation: await keepSlotActiveUntilTakenOver(\n          lc,\n          replicationSession,\n          slot,\n          metadataPublicationName(shard.appID, shard.shardNum),\n          toBigInt(lsn),\n        ),\n      };\n    });\n    lc.info?.(`successfully claimed slot ${slot}`);\n    return reserved;\n  } catch (e) {\n    lc.error?.(`unable to claim replication slot ${slot}`, e);\n    await replicationSession.end();\n    return null;\n  }\n}\n\nexport function dropInactiveSlotsAndReplicas(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n  slots: string[],\n) {\n  const lockName = replicationSlotManagementLock(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx<{slot: string; replicaID: string | null}[]>\n    /*sql*/ `\n      SELECT slot_name as slot, replica.id as \"replicaID\", pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name IN ${tx(slots)} AND NOT active;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped ${dropped.length} inactive replication slot(s)`, {\n        dropped,\n      });\n    }\n\n    const replicas = dropped\n      .map(({replicaID}) => replicaID)\n      .filter(id => id !== null);\n    if (replicas.length) {\n      // Delete replicas associated with the inactive (and now dropped) slots.\n      await tx\n      /*sql*/ `DELETE FROM ${tx(replicasTable)} WHERE id IN ${tx(replicas)}`;\n      lc.info?.(`deleted ${replicas.length} old replica(s)`, {replicas});\n    }\n  });\n}\n\nfunction dropUnclaimedSlots(\n  lc: LogContext,\n  sql: PostgresDB,\n  shard: ShardID,\n): Promise<{dropped: number; active: number; draining: number}> {\n  // The slot / replica cleanup happens within a transaction while holding\n  // the replication slot management lock for this shard, to ensure that no\n  // slot that belongs to a newer replica is dropped.\n  const lockName = replicationSlotManagementLock(shard);\n  const slotExpression = replicationSlotExpression(shard);\n  const replicasTable = `${upstreamSchema(shard)}.replicas`;\n\n  return runTx(sql, async tx => {\n    await tx`SELECT pg_advisory_xact_lock(hashtext(${lockName}))`;\n\n    const dropped = await tx /*sql*/ `\n      SELECT slot_name as slot, pg_drop_replication_slot(slot_name) \n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression} \n          AND NOT active\n          AND replica.id IS NULL;\n    `;\n    if (dropped.length) {\n      lc.info?.(`dropped inactive replication slots`, {dropped});\n    }\n\n    const remaining = await tx<\n      {slot: string; pid: number | null; id: string | null}[]\n    > /*sql*/ `\n      SELECT slot_name as slot, active_pid as pid, replica.id as id\n        FROM pg_replication_slots\n        LEFT JOIN ${tx(replicasTable)} replica on slot_name = slot\n        WHERE slot_name LIKE ${slotExpression};\n    `;\n    if (remaining.length) {\n      lc.info?.(`remaining replication slots`, {remaining});\n    }\n\n    let active = 0;\n    let draining = 0;\n    for (const {id} of remaining) {\n      if (id === null) {\n        draining++;\n      } else {\n        active++;\n      }\n    }\n\n    return {\n      dropped: dropped.length,\n      active,\n      draining,\n    };\n  });\n}\n\nconst ALPHABET = 'abcdefghijklmnopqrstuvwxyz';\n\n// Alphabetic notation is used as the slot pool suffix to distinguish\n// it from the (numeric) shard num that's also encoded in the slot name.\nexport function slotPoolSuffix(n: number) {\n  n++; // Adjust for 0-based indexing\n\n  let suffix = '';\n  while (n > 0) {\n    n--;\n    suffix = ALPHABET[n % 26] + suffix;\n    n = Math.floor(n / 26);\n  }\n  return suffix;\n}\n\nfunction replicationSlotManagementLock(shard: ShardID) {\n  return `replication-slot-management:${shard.appID}_${shard.shardNum}`;\n}\n"],"mappings":";;;;;;;;;;;;;AA8EA,IAAM,qCAAqC;AAK3C,IAAM,yBAAyB,qCAAqC;AAKpE,eAAsB,sBACpB,IACA,SACA,EACE,UACA,UACA,WACA,cAAc,0BAEU;CAgB1B,MAAM,QAAQ,OAAO,sBAAsB,aAAa;CAExD,MAAM,EAAC,eACL,MAAM,QAAQ,OAA8B;;GAE7C,GACC;CAEF,MAAM,iBAAiB,YAAY,cAAc;CACjD,MAAM,UAAU,YAAY,aAAA,OAAqB,eAAe;CAIhE,MAAM,QAAQ,MAAM,UAHD,QAAQ,OAAkC;+BAChC,SAAS,IAAI,eAAe,oBAAoB,SAE/C,GAAY,kCAAkC;CAC5E,IAAI,UAAU,aAAa;EAGzB,QACG,IAAI,EACJ,OAAM,MACL,GAAG,OAAO,oDAAoD,CAAC,CACjE;EACF,MAAM,IAAI,MACR,mBAAmB,mCAAmC,gCAAgC,SAAS,qCAEjG;CACF;CACA,MAAM,CAAC,QAAQ;CACf,GAAG,OAAO,4BAA4B,YAAY,IAAI;CACtD,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiDA,eAAsB,qBACpB,IACA,KACA,aACA,OACA,OACA,WACA,UACA,eACA,iBACA,OACmC;CAGnC,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,sBAAsB,KAAK;CAElD,KAAK,IAAI,QAAQ,OAAQ,QAAQ,OAAO;EACtC,MAAM,mBAAmB,IAAI,KAAK,KAAK;EAEvC,IAAI;EACJ,IAAI;GACF,OAAO,MAAM,MAAM,KAAK,OAAM,OAAM;IAClC,MAAM,EAAE,yCAAyC,SAAS;IAE1D,IAAI,UAAA,GAAuB;KAIzB,MAAM,SAAS,MAAM,qBAAqB,IAAI,IAAI,OAAO,KAAK;KAC9D,IAAI,OAAO,QAAQ;MACjB,MAAM,CAAC,EAAC,IAAI,OAAO,YAAW;MAC9B,IAAI,UAAA,KAAyB,QAC3B,MAAM,IAAI,gBACR,oBAAoB,GAAG,6BACzB;KAEJ;IACF;IAGA,MAAM,QAAQ,MAAM,EAA6B;;mCAEtB,iBAAiB,IAAI;UAC9C,OAAO;IACT,MAAM,QAAQ,IAAI,IAAI,MAAM,KAAK,CAAC;IAClC,KAAK,IAAI,OAAO,IAAK,QAAQ;KAC3B,MAAM,gBAAgB,GAAG,iBAAiB,eAAe,IAAI;KAC7D,IAAI,CAAC,MAAM,IAAI,aAAa,GAAG;MAC7B,WAAW;MACX;KACF;IACF;IAEA,MAAM,OAAO,MAAM,sBAAsB,IAAI,oBAAoB;KAC/D;KACA;IACF,CAAC;IACD,MAAM,mBAAmB,MAAM,gBAAgB,KAAK,aAAa;IACjE,MAAM,iBAAiB,MAAM,6BAC3B,IACA,oBACA,KAAK,WACL,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,KAAK,gBAAgB,CAChC;IAEA,MAAM,cACJ,IACA,OACA,WACA,KAAK,WACL,OACA,UAAA,IACI,qBAAqB,KAAK,gBAAgB,IAC1C,IACJ,eACA,KACF;IAOA,OAAO;KAAC;KAAM;KAAkB;KAAgB,SALhC,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,iBAGyB;IAAO;GACzD,CAAC;EACH,SAAS,GAAG;GACV,IAAI,SAAS,gBAAgB,GAAG,yBAAyB,GAAG;IAK1D,MAAM,GAAG;IACT,GAAG,OAAO,6CAA6C;IACvD;GACF;GAIA,IAAI,SAAS,gBAAgB,GAAG,+BAA+B,GAAG;IAChE,GAAG,OACD,sEACA,CACF;IAGA,MAAM,GAAG;wBACO,IAAI,GAFK,eAAe,KAAK,EAAE,UAEd,EAAE;;IAEnC;GACF;GAEA,IAAI,UAAU;IACZ,GAAG,OAAO,iBAAiB,SAAS,gBAAgB,CAAC;IACrD,MAAM,mBAAmB,IAAI;IAC7B,MAAM,GAAG,mCAAmC,SAAS;GACvD;GACA,MAAM;EACR;CACF;AACF;;;;;;AAOA,eAAsB,uBACpB,IACA,KACA,OACA,aACA,MAC8B;CAC9B,MAAM,qBAAqB,4BAA4B,KAAK,WAAW;CACvE,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,IAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK,OAAM,OAAM;GAC5C,MAAM,EAAE,yCAAyC,SAAS;GAE1D,MAAM,WAAW,MAAM,EAEb;;;eAGD,GAAG,aAAa,EAAE;4BACL,KAAK;;GAE3B,IAAI,SAAS,WAAW,GAAG;IACzB,GAAG,OAAO,6BAA6B,MAAM;IAC7C,OAAO;GACT;GACA,MAAM,CAAC,EAAC,QAAQ,WAAW,SAAQ;GACnC,IAAI,QAAQ;IACV,GAAG,OACD,WAAW,UAAU,YAAY,KAAK,iCACxC;IACA,OAAO;GACT;GAEA,MAAM,EAAE;iBACG,GAAG,aAAa,EAAE,iBAAA,EAAyB,cAAc,UAAU;;GAO9E,OAAO;IACL;IACA,SAPc,KACd,MAAM,gBAAgB,IAAI,OAAO,SAAS,GAC1C,WAAW,UAAU,aAKrB;IACA,aAAa,MAAM,6BACjB,IACA,oBACA,MACA,wBAAwB,MAAM,OAAO,MAAM,QAAQ,GACnD,SAAS,GAAG,CACd;GACF;EACF,CAAC;EACD,GAAG,OAAO,6BAA6B,MAAM;EAC7C,OAAO;CACT,SAAS,GAAG;EACV,GAAG,QAAQ,oCAAoC,QAAQ,CAAC;EACxD,MAAM,mBAAmB,IAAI;EAC7B,OAAO;CACT;AACF;AAEA,SAAgB,6BACd,IACA,KACA,OACA,OACA;CACA,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EACd;;;oBAGQ,GAAG,aAAa,EAAE;6BACT,GAAG,KAAK,EAAE;;EAEnC,IAAI,QAAQ,QACV,GAAG,OAAO,WAAW,QAAQ,OAAO,gCAAgC,EAClE,QACF,CAAC;EAGH,MAAM,WAAW,QACd,KAAK,EAAC,gBAAe,SAAS,EAC9B,QAAO,OAAM,OAAO,IAAI;EAC3B,IAAI,SAAS,QAAQ;GAEnB,MAAM,EACE,eAAe,GAAG,aAAa,EAAE,eAAe,GAAG,QAAQ;GACnE,GAAG,OAAO,WAAW,SAAS,OAAO,kBAAkB,EAAC,SAAQ,CAAC;EACnE;CACF,CAAC;AACH;AAEA,SAAS,mBACP,IACA,KACA,OAC8D;CAI9D,MAAM,WAAW,8BAA8B,KAAK;CACpD,MAAM,iBAAiB,0BAA0B,KAAK;CACtD,MAAM,gBAAgB,GAAG,eAAe,KAAK,EAAE;CAE/C,OAAO,MAAM,KAAK,OAAM,OAAM;EAC5B,MAAM,EAAE,yCAAyC,SAAS;EAE1D,MAAM,UAAU,MAAM,EAAW;;;oBAGjB,GAAG,aAAa,EAAE;+BACP,eAAe;;;;EAI1C,IAAI,QAAQ,QACV,GAAG,OAAO,sCAAsC,EAAC,QAAO,CAAC;EAG3D,MAAM,YAAY,MAAM,EAEd;;;oBAGM,GAAG,aAAa,EAAE;+BACP,eAAe;;EAE1C,IAAI,UAAU,QACZ,GAAG,OAAO,+BAA+B,EAAC,UAAS,CAAC;EAGtD,IAAI,SAAS;EACb,IAAI,WAAW;EACf,KAAK,MAAM,EAAC,QAAO,WACjB,IAAI,OAAO,MACT;OAEA;EAIJ,OAAO;GACL,SAAS,QAAQ;GACjB;GACA;EACF;CACF,CAAC;AACH;AAEA,IAAM,WAAW;AAIjB,SAAgB,eAAe,GAAW;CACxC;CAEA,IAAI,SAAS;CACb,OAAO,IAAI,GAAG;EACZ;EACA,SAAS,SAAS,IAAI,MAAM;EAC5B,IAAI,KAAK,MAAM,IAAI,EAAE;CACvB;CACA,OAAO;AACT;AAEA,SAAS,8BAA8B,OAAgB;CACrD,OAAO,+BAA+B,MAAM,MAAM,GAAG,MAAM;AAC7D"}

package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index-predicate.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/index-predicate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EAEd,kBAAkB,EACnB,MAAM,yBAAyB,CAAC;AAkBjC,eAAO,MAAM,2BAA2B,YACtC,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,oBAAoB,EACpB,6BAA6B,CACrB,CAAC;AAEX,MAAM,MAAM,0BAA0B,GACpC,CAAC,OAAO,2BAA2B,CAAC,CAAC,MAAM,CAAC,CAAC;AAE/C,MAAM,MAAM,oBAAoB,GAC5B;IAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAA;CAAC,GAC9D;IAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,0BAA0B,CAAA;CAAC,CAAC;AAE7E,KAAK,mBAAmB,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,GAAG;IACjE,QAAQ,CAAC,wBAAwB,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF,KAAK,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,GAAG;IAC9D,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;CACjE,CAAC;AAgEF,wBAAgB,+BAA+B,CAC7C,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,kBAAkB,GACxB,oBAAoB,CAetB"}
+{"version":3,"file":"index-predicate.d.ts","sourceRoot":"","sources":["../../../../../../../../zero-cache/src/services/change-source/pg/schema/index-predicate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,cAAc,EAEd,kBAAkB,EACnB,MAAM,yBAAyB,CAAC;AAmBjC,eAAO,MAAM,2BAA2B,YACtC,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,oBAAoB,EACpB,6BAA6B,CACrB,CAAC;AAEX,MAAM,MAAM,0BAA0B,GACpC,CAAC,OAAO,2BAA2B,CAAC,CAAC,MAAM,CAAC,CAAC;AAE/C,MAAM,MAAM,oBAAoB,GAC5B;IAAC,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAA;CAAC,GAC9D;IAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,0BAA0B,CAAA;CAAC,CAAC;AAE7E,KAAK,mBAAmB,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,GAAG;IACjE,QAAQ,CAAC,wBAAwB,CAAC,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC;CAChE,CAAC;AAEF,KAAK,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,GAAG;IAC9D,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;CACjE,CAAC;AAoEF,wBAAgB,+BAA+B,CAC7C,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,kBAAkB,GACxB,oBAAoB,CAetB"}

package/out/zero-cache/src/services/change-source/pg/schema/index-predicate.js

@@ -3,6 +3,7 @@ var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
 var INTEGER_RE = /^[+-]?\d+$/;
 var WHITESPACE_RE = /\s/;
 var OPERATOR_TOKEN_RE = /^(?:::|<>|!=|<=|>=|=|<|>)/;
+var UNSUPPORTED_OPERATOR_TOKEN_RE = /^(?:!~\*|!~|~\*|~)/;
 var INTEGER_TOKEN_RE = /^[+-]?\d+/;
 var WORD_TOKEN_RE = /^[A-Za-z_][A-Za-z0-9_$]*/;
 var SIMPLE_ESCAPES = {
@@ -80,9 +81,16 @@ var Parser = class {
 				not
 			};
 		}
+		if (this.#take("IN")) {
+			this.expect("(");
+			const values = this.#parseOperandList(")");
+			this.expect(")");
+			return comparisons(left, values);
+		}
 		const op = this.#tokens[this.#position]?.type;
 		if (isComparison(op)) {
 			this.#position++;
+			if (op === "=" && this.#takeIdentifier("ANY")) return comparisons(left, this.#parseLiteralArray());
 			return {
 				type: "comparison",
 				left,
@@ -92,9 +100,31 @@ var Parser = class {
 		}
 		const next = this.#tokens[this.#position];
 		if (next?.type === "identifier" && next.value.toUpperCase() === "COLLATE") throw new UnsupportedPredicateError("non-deterministic-collation");
+		if (next?.type === "unsupported" && UNSUPPORTED_OPERATOR_TOKEN_RE.test(next.value)) throw new UnsupportedPredicateError("unsupported-operator");
 		if (next?.type === "identifier" && unsupportedOperatorWords.has(next.value.toUpperCase())) throw new UnsupportedPredicateError("unsupported-operator");
 		return left;
 	}
+	#parseLiteralArray() {
+		this.expect("(");
+		this.#expectIdentifier("ARRAY");
+		this.expect("[");
+		const values = this.#parseOperandList("]");
+		this.expect("]");
+		if (this.#take("::")) {
+			this.#parseCastName();
+			this.expect("[");
+			this.expect("]");
+		}
+		this.expect(")");
+		return values;
+	}
+	#parseOperandList(end) {
+		const values = [];
+		if (this.#tokens[this.#position]?.type === end) return values;
+		values.push(this.#parseOperand());
+		while (this.#take(",")) values.push(this.#parseOperand());
+		return values;
+	}
 	#parseOperand() {
 		let expression = this.#parsePrimary();
 		while (this.#take("::")) expression = {
@@ -130,17 +160,27 @@ var Parser = class {
 	#parseCastName() {
 		const first = this.#expectIdentifier();
 		const parts = [first];
+		let separator = ".";
 		while (this.#take(".")) parts.push(this.#expectIdentifier());
 		const next = this.#tokens[this.#position];
-		if (first.toLowerCase() === "character" && next?.type === "identifier" && next.value.toLowerCase() === "varying") parts.push(this.#expectIdentifier());
+		if (first.toLowerCase() === "character" && next?.type === "identifier" && next.value.toLowerCase() === "varying") {
+			parts.push(this.#expectIdentifier());
+			separator = " ";
+		}
 		if (this.#tokens[this.#position]?.type === "(") throw new UnsupportedPredicateError("unsupported-cast");
-		return parts.join(".");
+		return parts.join(separator);
 	}
-	#expectIdentifier() {
+	#expectIdentifier(expected) {
 		const token = this.#tokens[this.#position++];
-		if (token?.type !== "identifier") throw new UnsupportedPredicateError("unsupported-cast");
+		if (token?.type !== "identifier" || expected !== void 0 && token.value.toUpperCase() !== expected) throw new UnsupportedPredicateError("unsupported-cast");
 		return token.value;
 	}
+	#takeIdentifier(expected) {
+		const token = this.#tokens[this.#position];
+		if (token?.type !== "identifier" || token.value.toUpperCase() !== expected) return false;
+		this.#position++;
+		return true;
+	}
 	#take(type) {
 		if (this.#tokens[this.#position]?.type !== type) return false;
 		this.#position++;
@@ -152,13 +192,24 @@ var Parser = class {
 };
 var unsupportedOperatorWords = new Set([
 	"ALL",
-	"ANY",
 	"BETWEEN",
 	"ILIKE",
-	"IN",
 	"LIKE",
 	"SIMILAR"
 ]);
+function comparisons(left, values) {
+	if (values.length === 0) throw new UnsupportedPredicateError("unsupported-syntax");
+	const expressions = values.map((right) => ({
+		type: "comparison",
+		left,
+		op: "=",
+		right
+	}));
+	return expressions.length === 1 ? expressions[0] : {
+		type: "or",
+		expressions
+	};
+}
 function normalize(expression, table) {
 	switch (expression.type) {
 		case "and":
@@ -419,7 +470,16 @@ function tokenize(sql) {
 			i += operator.length;
 			continue;
 		}
-		if (char === "(" || char === ")" || char === ".") {
+		const unsupportedOperator = UNSUPPORTED_OPERATOR_TOKEN_RE.exec(sql.slice(i))?.[0];
+		if (unsupportedOperator) {
+			tokens.push({
+				type: "unsupported",
+				value: unsupportedOperator
+			});
+			i += unsupportedOperator.length;
+			continue;
+		}
+		if (char === "(" || char === ")" || char === "[" || char === "]" || char === "," || char === ".") {
 			tokens.push({ type: char });
 			i++;
 			continue;
@@ -448,7 +508,7 @@ function tokenize(sql) {
 				literalType: "null",
 				value: null
 			});
-			else if (keyword === "AND" || keyword === "OR" || keyword === "NOT" || keyword === "IS") tokens.push({ type: keyword });
+			else if (keyword === "AND" || keyword === "OR" || keyword === "NOT" || keyword === "IN" || keyword === "IS") tokens.push({ type: keyword });
 			else tokens.push({
 				type: "identifier",
 				value: word.toLowerCase()