@rotorsoft/act-pg 1.13.1 → 1.13.2

package/dist/index.js

@@ -34,6 +34,8 @@ var SAFE_IDENTIFIER = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
 var PG_UNIQUE_VIOLATION = "23505";
 var NOTIFY_CHANNEL_PREFIX = "act_commit";
 var NOTIFY_MAX_PAYLOAD_BYTES = 8e3;
+var NOTIFY_RECONNECT_BASE_MS = 250;
+var NOTIFY_RECONNECT_MAX_MS = 3e4;
 function notify_channel(schema, table) {
   return `${NOTIFY_CHANNEL_PREFIX}_${schema}_${table}`;
 }
@@ -106,6 +108,34 @@ var PostgresStore = class {
    * connection would re-fire the stale handler.
    */
   _listen_handler;
+  /**
+   * Error listener attached to the active LISTEN client. node-postgres
+   * removes its idle-error guard on checkout, so a checked-out client
+   * that emits `error` (backend restart, failover, network drop) with no
+   * listener is an uncaught exception — a process crash (#1189). Tracked
+   * alongside `_listen_handler` so teardown detaches it in lockstep.
+   */
+  _listen_error_handler;
+  /**
+   * The caller's notification handler for the active subscription, kept
+   * so the self-healing reconnect path (#1189) can re-establish LISTEN
+   * on a fresh client after the dedicated one emits `error`. Cleared by
+   * `_teardown_listen`, which is what makes disposal cancel any pending
+   * reconnect.
+   */
+  _notify_handler;
+  /**
+   * Pending reconnect timer, if a LISTEN client error scheduled one.
+   * Tracked so `_teardown_listen` (and therefore `dispose()`) can cancel
+   * it — a reconnect must never fire after teardown.
+   */
+  _reconnect_timer;
+  /**
+   * Consecutive reconnect attempts since the last healthy LISTEN, used to
+   * grow the capped exponential backoff. Reset to 0 once a re-LISTEN
+   * succeeds.
+   */
+  _reconnect_attempts = 0;
   /**
    * Cross-process commit subscription. **Present only when
    * `config.notify === true`** — the orchestrator's auto-wire path
@@ -173,16 +203,30 @@ var PostgresStore = class {
     await this._pool.end();
   }
   /**
-   * Tear down the active LISTEN subscription if any: detach the
-   * notification listener, run UNLISTEN, and destroy the dedicated
-   * client (do not return it to the pool — its listener is removed but
-   * destroying belt-and-braces guards against any future change in
-   * pg-pool semantics that could re-issue a half-clean client).
+   * Tear down the active LISTEN subscription if any: cancel any pending
+   * reconnect, forget the caller's handler (so no reconnect can fire
+   * after teardown), detach the notification + error listeners, run
+   * UNLISTEN, and destroy the dedicated client (do not return it to the
+   * pool — its listeners are removed but destroying belt-and-braces
+   * guards against any future change in pg-pool semantics that could
+   * re-issue a half-clean client).
+   *
+   * Clearing `_notify_handler` and the reconnect timer here is what makes
+   * `dispose()` safe during a pending reconnect (#1189): a scheduled
+   * `_reconnect` bails the moment it finds no handler.
    */
   async _teardown_listen() {
+    if (this._reconnect_timer) {
+      clearTimeout(this._reconnect_timer);
+      this._reconnect_timer = void 0;
+    }
+    this._notify_handler = void 0;
+    this._reconnect_attempts = 0;
     if (!this._listen_client) return;
     this._listen_client.removeListener("notification", this._listen_handler);
+    this._listen_client.removeListener("error", this._listen_error_handler);
     this._listen_handler = void 0;
+    this._listen_error_handler = void 0;
     try {
       await this._listen_client.query(`UNLISTEN ${this._channel}`);
     } catch {
@@ -246,7 +290,7 @@ var PostgresStore = class {
           stream varchar(100) COLLATE pg_catalog."default" PRIMARY KEY,
           source varchar(100) COLLATE pg_catalog."default",
           at int NOT NULL DEFAULT -1,
-          retry smallint NOT NULL DEFAULT -1,
+          retry int NOT NULL DEFAULT -1,
           blocked boolean NOT NULL DEFAULT false,
           error text,
           leased_by text,
@@ -268,6 +312,21 @@ var PostgresStore = class {
         `ALTER TABLE ${this._fqs}
          ADD COLUMN IF NOT EXISTS deferred_at timestamptz;`
       );
+      await client.query(
+        `DO $$
+         BEGIN
+           IF EXISTS (
+             SELECT 1 FROM information_schema.columns
+             WHERE table_schema = '${this.config.schema}'
+               AND table_name = '${this.config.table}_streams'
+               AND column_name = 'retry'
+               AND data_type = 'smallint'
+           ) THEN
+             EXECUTE 'ALTER TABLE ${this._fqs} ALTER COLUMN retry TYPE integer';
+           END IF;
+         END
+         $$;`
+      );
       await client.query(
         `DROP INDEX IF EXISTS "${this.config.schema}"."${this.config.table}_streams_fetch_ix"`
       );
@@ -1154,11 +1213,39 @@ var PostgresStore = class {
    * client; pool disposal also tears the subscription down as a safety
    * net.
    *
+   * The subscription is **self-healing** (#1189): the dedicated client
+   * has an `error` listener that, on a connection blip (backend restart,
+   * failover, network drop), tears the dead client down and re-LISTENs
+   * on a fresh one with capped exponential backoff — degrading to the
+   * poll path in between. A pending reconnect is cancelled by disposal,
+   * so no reconnect fires after teardown.
+   *
    * @param handler Called for each cross-process commit notification.
    * @returns Disposer that releases the LISTEN client.
    */
   async _subscribe_notifications(handler) {
     await this._teardown_listen();
+    this._notify_handler = handler;
+    try {
+      await this._open_listen(handler);
+    } catch (err) {
+      this._notify_handler = void 0;
+      throw err;
+    }
+    return async () => {
+      if (this._notify_handler !== handler) return;
+      await this._teardown_listen();
+    };
+  }
+  /**
+   * Check out a dedicated client, attach the notification + error
+   * listeners, and run `LISTEN`. Shared by the initial subscription and
+   * every reconnect (#1189). On any failure before `LISTEN` succeeds the
+   * client is detached and destroyed so nothing leaks — the caller
+   * decides whether to propagate (initial subscribe) or reschedule
+   * (reconnect).
+   */
+  async _open_listen(handler) {
     const client = await this._client("notify");
     const on_notification = (msg) => {
       if (msg.channel !== this._channel) return;
@@ -1197,20 +1284,59 @@ var PostgresStore = class {
         logger.error(err, "act_commit: handler threw, listener preserved");
       }
     };
+    const on_error = (err) => {
+      logger.error(err, "act_commit: LISTEN client errored, reconnecting");
+      this._reconnect();
+    };
     client.on("notification", on_notification);
+    client.on("error", on_error);
     try {
       await client.query(`LISTEN ${this._channel}`);
     } catch (err) {
       client.removeListener("notification", on_notification);
+      client.removeListener("error", on_error);
       client.release(true);
       throw err;
     }
     this._listen_client = client;
     this._listen_handler = on_notification;
-    return async () => {
-      if (this._listen_client !== client) return;
-      await this._teardown_listen();
-    };
+    this._listen_error_handler = on_error;
+    this._reconnect_attempts = 0;
+  }
+  /**
+   * Self-heal the LISTEN subscription after the dedicated client emitted
+   * `error` (#1189). Detaches and destroys the dead client, then
+   * reconnects on a fresh one with capped exponential backoff. Bails
+   * immediately if the subscription was disposed while a reconnect was
+   * pending (`_notify_handler` cleared by `_teardown_listen`), so no
+   * reconnect ever fires after teardown.
+   */
+  _reconnect() {
+    const handler = this._notify_handler;
+    if (!handler) return;
+    if (this._listen_client) {
+      this._listen_client.removeListener("notification", this._listen_handler);
+      this._listen_client.removeListener("error", this._listen_error_handler);
+      this._listen_handler = void 0;
+      this._listen_error_handler = void 0;
+      this._listen_client.release(true);
+      this._listen_client = void 0;
+    }
+    const delay = Math.min(
+      NOTIFY_RECONNECT_MAX_MS,
+      NOTIFY_RECONNECT_BASE_MS * 2 ** this._reconnect_attempts
+    );
+    this._reconnect_attempts++;
+    this._reconnect_timer = setTimeout(() => {
+      this._reconnect_timer = void 0;
+      const current = this._notify_handler;
+      if (!current) return;
+      this._open_listen(current).catch((err) => {
+        logger.error(err, "act_commit: LISTEN reconnect failed, retrying");
+        this._reconnect();
+      });
+    }, delay);
+    this._reconnect_timer.unref?.();
   }
   /**
    * Atomically truncates streams and seeds each with a snapshot or tombstone.