@rotorsoft/act-pg 1.13.0 → 1.13.2

package/dist/index.js

@@ -34,6 +34,8 @@ var SAFE_IDENTIFIER = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
 var PG_UNIQUE_VIOLATION = "23505";
 var NOTIFY_CHANNEL_PREFIX = "act_commit";
 var NOTIFY_MAX_PAYLOAD_BYTES = 8e3;
+var NOTIFY_RECONNECT_BASE_MS = 250;
+var NOTIFY_RECONNECT_MAX_MS = 3e4;
 function notify_channel(schema, table) {
   return `${NOTIFY_CHANNEL_PREFIX}_${schema}_${table}`;
 }
@@ -106,6 +108,34 @@ var PostgresStore = class {
    * connection would re-fire the stale handler.
    */
   _listen_handler;
+  /**
+   * Error listener attached to the active LISTEN client. node-postgres
+   * removes its idle-error guard on checkout, so a checked-out client
+   * that emits `error` (backend restart, failover, network drop) with no
+   * listener is an uncaught exception — a process crash (#1189). Tracked
+   * alongside `_listen_handler` so teardown detaches it in lockstep.
+   */
+  _listen_error_handler;
+  /**
+   * The caller's notification handler for the active subscription, kept
+   * so the self-healing reconnect path (#1189) can re-establish LISTEN
+   * on a fresh client after the dedicated one emits `error`. Cleared by
+   * `_teardown_listen`, which is what makes disposal cancel any pending
+   * reconnect.
+   */
+  _notify_handler;
+  /**
+   * Pending reconnect timer, if a LISTEN client error scheduled one.
+   * Tracked so `_teardown_listen` (and therefore `dispose()`) can cancel
+   * it — a reconnect must never fire after teardown.
+   */
+  _reconnect_timer;
+  /**
+   * Consecutive reconnect attempts since the last healthy LISTEN, used to
+   * grow the capped exponential backoff. Reset to 0 once a re-LISTEN
+   * succeeds.
+   */
+  _reconnect_attempts = 0;
   /**
    * Cross-process commit subscription. **Present only when
    * `config.notify === true`** — the orchestrator's auto-wire path
@@ -173,16 +203,30 @@ var PostgresStore = class {
     await this._pool.end();
   }
   /**
-   * Tear down the active LISTEN subscription if any: detach the
-   * notification listener, run UNLISTEN, and destroy the dedicated
-   * client (do not return it to the pool — its listener is removed but
-   * destroying belt-and-braces guards against any future change in
-   * pg-pool semantics that could re-issue a half-clean client).
+   * Tear down the active LISTEN subscription if any: cancel any pending
+   * reconnect, forget the caller's handler (so no reconnect can fire
+   * after teardown), detach the notification + error listeners, run
+   * UNLISTEN, and destroy the dedicated client (do not return it to the
+   * pool — its listeners are removed but destroying belt-and-braces
+   * guards against any future change in pg-pool semantics that could
+   * re-issue a half-clean client).
+   *
+   * Clearing `_notify_handler` and the reconnect timer here is what makes
+   * `dispose()` safe during a pending reconnect (#1189): a scheduled
+   * `_reconnect` bails the moment it finds no handler.
    */
   async _teardown_listen() {
+    if (this._reconnect_timer) {
+      clearTimeout(this._reconnect_timer);
+      this._reconnect_timer = void 0;
+    }
+    this._notify_handler = void 0;
+    this._reconnect_attempts = 0;
     if (!this._listen_client) return;
     this._listen_client.removeListener("notification", this._listen_handler);
+    this._listen_client.removeListener("error", this._listen_error_handler);
     this._listen_handler = void 0;
+    this._listen_error_handler = void 0;
     try {
       await this._listen_client.query(`UNLISTEN ${this._channel}`);
     } catch {
@@ -246,7 +290,7 @@ var PostgresStore = class {
           stream varchar(100) COLLATE pg_catalog."default" PRIMARY KEY,
           source varchar(100) COLLATE pg_catalog."default",
           at int NOT NULL DEFAULT -1,
-          retry smallint NOT NULL DEFAULT -1,
+          retry int NOT NULL DEFAULT -1,
           blocked boolean NOT NULL DEFAULT false,
           error text,
           leased_by text,
@@ -268,6 +312,21 @@ var PostgresStore = class {
         `ALTER TABLE ${this._fqs}
          ADD COLUMN IF NOT EXISTS deferred_at timestamptz;`
       );
+      await client.query(
+        `DO $$
+         BEGIN
+           IF EXISTS (
+             SELECT 1 FROM information_schema.columns
+             WHERE table_schema = '${this.config.schema}'
+               AND table_name = '${this.config.table}_streams'
+               AND column_name = 'retry'
+               AND data_type = 'smallint'
+           ) THEN
+             EXECUTE 'ALTER TABLE ${this._fqs} ALTER COLUMN retry TYPE integer';
+           END IF;
+         END
+         $$;`
+      );
       await client.query(
         `DROP INDEX IF EXISTS "${this.config.schema}"."${this.config.table}_streams_fetch_ix"`
       );
@@ -413,16 +472,13 @@ var PostgresStore = class {
   async commit(stream, msgs, meta, expectedVersion) {
     if (msgs.length === 0) return [];
     const client = await this._client("commit");
-    let version = -1;
     try {
-      await client.query("BEGIN");
       const last = await client.query(
-        `SELECT version
-        FROM ${this._fqt}
-        WHERE stream=$1 ORDER BY version DESC LIMIT 1`,
+        `SELECT version FROM ${this._fqt}
+         WHERE stream=$1 ORDER BY version DESC LIMIT 1`,
         [stream]
       );
-      version = last.rowCount ? last.rows[0].version : -1;
+      let version = last.rows.at(0)?.version ?? -1;
       if (typeof expectedVersion === "number" && version !== expectedVersion)
         throw new ConcurrencyError(
           stream,
@@ -430,52 +486,69 @@ var PostgresStore = class {
           msgs,
           expectedVersion
         );
-      const committed = [];
+      const base_version = version;
+      const names = [];
+      const datas = [];
+      const piis = [];
+      const versions = [];
       for (const { name, data, pii } of msgs) {
         version++;
-        const sql = `
+        names.push(name);
+        datas.push(JSON.stringify(data));
+        piis.push(
+          this._resolve_pii_key && pii != null ? JSON.stringify(await encrypt(pii, this._resolve_pii_key)) : pii != null ? JSON.stringify(pii) : null
+        );
+        versions.push(version);
+      }
+      const insert_select = msgs.length === 1 ? `SELECT $1, $2::jsonb, $3::jsonb, $5, $4::int, $6 FROM l` : `SELECT u.name, u.data::jsonb, u.pii::jsonb, $5, u.version, $6
+             FROM l, unnest($1::text[], $2::text[], $3::text[], $4::int[])
+               WITH ORDINALITY AS u(name, data, pii, version, ord)
+             ORDER BY u.ord`;
+      const notify_ctes = this.config.notify ? `,
+           payload AS (
+             SELECT json_build_object(
+               'stream', $5::text,
+               'events', json_agg(json_build_object('id', ins.id, 'name', ins.name) ORDER BY ins.version),
+               'by', $9::text
+             )::text AS p
+             FROM ins
+           ),
+           n AS (
+             SELECT pg_notify($8, payload.p) FROM payload
+             WHERE octet_length(payload.p) < $10
+           )` : "";
+      const final_select = this.config.notify ? "SELECT ins.* FROM ins LEFT JOIN n ON true ORDER BY ins.version" : "SELECT * FROM ins ORDER BY version";
+      const sql = `WITH l AS (SELECT pg_advisory_xact_lock(hashtext($7))),
+        ins AS (
           INSERT INTO ${this._fqt}(name, data, pii, stream, version, meta)
-          VALUES($1, $2, $3, $4, $5, $6) RETURNING *`;
-        const pii_for_write = this._resolve_pii_key && pii != null ? JSON.stringify(await encrypt(pii, this._resolve_pii_key)) : pii ?? null;
-        const vals = [name, data, pii_for_write, stream, version, meta];
-        try {
-          const { rows } = await client.query(sql, vals);
-          const row = rows.at(0);
-          if (this._resolve_pii_key && typeof row.pii === "string") {
-            const decrypted = await decrypt(row.pii, this._resolve_pii_key);
-            row.pii = decrypted;
-          }
-          committed.push(row);
-        } catch (error) {
-          if (error?.code === PG_UNIQUE_VIOLATION) {
-            throw new ConcurrencyError(
-              stream,
-              version - 1,
-              msgs,
-              expectedVersion ?? -1
-            );
+          ${insert_select}
+          RETURNING *
+        )${notify_ctes}
+        ${final_select}`;
+      const base_params = msgs.length === 1 ? [names[0], datas[0], piis[0], versions[0], stream, meta, this._fqt] : [names, datas, piis, versions, stream, meta, this._fqt];
+      const params = this.config.notify ? [...base_params, this._channel, this._by, NOTIFY_MAX_PAYLOAD_BYTES] : base_params;
+      try {
+        const { rows } = await client.query(sql, params);
+        if (this._resolve_pii_key) {
+          for (const row of rows) {
+            if (typeof row.pii === "string") {
+              const decrypted = await decrypt(row.pii, this._resolve_pii_key);
+              row.pii = decrypted;
+            }
           }
-          throw error;
         }
+        return rows;
+      } catch (error) {
+        if (error?.code === PG_UNIQUE_VIOLATION) {
+          throw new ConcurrencyError(
+            stream,
+            base_version,
+            msgs,
+            expectedVersion ?? -1
+          );
+        }
+        throw error;
       }
-      if (this.config.notify) {
-        const payload = JSON.stringify({
-          stream,
-          events: committed.map((c) => ({ id: c.id, name: c.name })),
-          by: this._by
-        });
-        if (Buffer.byteLength(payload, "utf8") < NOTIFY_MAX_PAYLOAD_BYTES)
-          await client.query(`SELECT pg_notify($1, $2)`, [
-            this._channel,
-            payload
-          ]);
-      }
-      await client.query("COMMIT");
-      return committed;
-    } catch (error) {
-      await client.query("ROLLBACK").catch(() => {
-      });
-      throw error;
     } finally {
       client.release();
     }
@@ -1140,11 +1213,39 @@ var PostgresStore = class {
    * client; pool disposal also tears the subscription down as a safety
    * net.
    *
+   * The subscription is **self-healing** (#1189): the dedicated client
+   * has an `error` listener that, on a connection blip (backend restart,
+   * failover, network drop), tears the dead client down and re-LISTENs
+   * on a fresh one with capped exponential backoff — degrading to the
+   * poll path in between. A pending reconnect is cancelled by disposal,
+   * so no reconnect fires after teardown.
+   *
    * @param handler Called for each cross-process commit notification.
    * @returns Disposer that releases the LISTEN client.
    */
   async _subscribe_notifications(handler) {
     await this._teardown_listen();
+    this._notify_handler = handler;
+    try {
+      await this._open_listen(handler);
+    } catch (err) {
+      this._notify_handler = void 0;
+      throw err;
+    }
+    return async () => {
+      if (this._notify_handler !== handler) return;
+      await this._teardown_listen();
+    };
+  }
+  /**
+   * Check out a dedicated client, attach the notification + error
+   * listeners, and run `LISTEN`. Shared by the initial subscription and
+   * every reconnect (#1189). On any failure before `LISTEN` succeeds the
+   * client is detached and destroyed so nothing leaks — the caller
+   * decides whether to propagate (initial subscribe) or reschedule
+   * (reconnect).
+   */
+  async _open_listen(handler) {
     const client = await this._client("notify");
     const on_notification = (msg) => {
       if (msg.channel !== this._channel) return;
@@ -1183,20 +1284,59 @@ var PostgresStore = class {
         logger.error(err, "act_commit: handler threw, listener preserved");
       }
     };
+    const on_error = (err) => {
+      logger.error(err, "act_commit: LISTEN client errored, reconnecting");
+      this._reconnect();
+    };
     client.on("notification", on_notification);
+    client.on("error", on_error);
     try {
       await client.query(`LISTEN ${this._channel}`);
     } catch (err) {
       client.removeListener("notification", on_notification);
+      client.removeListener("error", on_error);
       client.release(true);
       throw err;
     }
     this._listen_client = client;
     this._listen_handler = on_notification;
-    return async () => {
-      if (this._listen_client !== client) return;
-      await this._teardown_listen();
-    };
+    this._listen_error_handler = on_error;
+    this._reconnect_attempts = 0;
+  }
+  /**
+   * Self-heal the LISTEN subscription after the dedicated client emitted
+   * `error` (#1189). Detaches and destroys the dead client, then
+   * reconnects on a fresh one with capped exponential backoff. Bails
+   * immediately if the subscription was disposed while a reconnect was
+   * pending (`_notify_handler` cleared by `_teardown_listen`), so no
+   * reconnect ever fires after teardown.
+   */
+  _reconnect() {
+    const handler = this._notify_handler;
+    if (!handler) return;
+    if (this._listen_client) {
+      this._listen_client.removeListener("notification", this._listen_handler);
+      this._listen_client.removeListener("error", this._listen_error_handler);
+      this._listen_handler = void 0;
+      this._listen_error_handler = void 0;
+      this._listen_client.release(true);
+      this._listen_client = void 0;
+    }
+    const delay = Math.min(
+      NOTIFY_RECONNECT_MAX_MS,
+      NOTIFY_RECONNECT_BASE_MS * 2 ** this._reconnect_attempts
+    );
+    this._reconnect_attempts++;
+    this._reconnect_timer = setTimeout(() => {
+      this._reconnect_timer = void 0;
+      const current = this._notify_handler;
+      if (!current) return;
+      this._open_listen(current).catch((err) => {
+        logger.error(err, "act_commit: LISTEN reconnect failed, retrying");
+        this._reconnect();
+      });
+    }, delay);
+    this._reconnect_timer.unref?.();
   }
   /**
    * Atomically truncates streams and seeds each with a snapshot or tombstone.
@@ -1214,6 +1354,9 @@ var PostgresStore = class {
     const client = await this._client("truncate");
     try {
       await client.query("BEGIN");
+      await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [
+        this._fqt
+      ]);
       const result = /* @__PURE__ */ new Map();
       if (full.length) {
         await client.query(`DELETE FROM ${this._fqs} WHERE stream = ANY($1)`, [