@rotorsoft/act-pg 1.15.1 → 1.17.0

package/dist/index.js

@@ -16,7 +16,6 @@ import {
 } from "@rotorsoft/act-crypto";
 import pg from "pg";
 var logger = log();
-var SOURCE_METACHARACTER_CLASS = "[]^$.*+?()[{}|\\\\]";
 var { Pool, types } = pg;
 var JSONB_OID = types.builtins.JSONB;
 var scopedTypes = {
@@ -336,6 +335,11 @@ var PostgresStore = class {
         `ALTER TABLE ${this._fqs}
          ADD COLUMN IF NOT EXISTS correlated_at int;`
       );
+      await client.query(
+        `UPDATE ${this._fqs}
+         SET correlated_at = (SELECT COALESCE(MAX(id), -1) FROM ${this._fqt})
+         WHERE correlated_at IS NULL;`
+      );
       await client.query(
         `DO $$
          BEGIN
@@ -648,38 +652,32 @@ var PostgresStore = class {
         -- and lock EVERY claimable stream for this transaction, starving
         -- overlapping competing consumers. We only lock the small
         -- lagging+leading candidate slice, down in the "locked" CTE.
-        -- Eligibility, split by SOURCE CLASS rather than expressed as one
-        -- disjunction (#1448). The three classes are mutually exclusive, so
-        -- the UNION ALL returns exactly what the old OR-chain did \u2014 but each
-        -- arm now carries a single, planner-legible predicate.
         --
-        -- That matters for one arm in particular. Written as
-        -- "s.source IS NULL OR (... AND e.stream = s.source) OR (... ~ ...)",
-        -- the equality is buried inside a disjunction and is NOT sargable:
-        -- no index on (stream, id) can serve it, so every probe degenerated
-        -- into a forward scan of the pk from the stream's watermark. With
-        -- the classes separated, the literal arm \u2014 every per-aggregate
-        -- .to(e => ({target: e.stream})) reaction, i.e. the overwhelming
-        -- majority \u2014 becomes an index seek on (stream, id).
+        -- Eligibility is a pure subscription-table predicate (#1488).
+        -- claim does not read the event log at all: correlate marks the
+        -- highest event id that resolves to a target, and at <
+        -- correlated_at is the whole question. The probe this replaced ran
+        -- an EXISTS against the events table once per eligible row, which
+        -- cost O(subscribed streams) per claim per worker no matter how
+        -- little work was pending.
         --
-        -- Measured at 10k subscribed streams with 10 pending: 5,792 ms
-        -- before, 10.3 ms after. See libs/act-pg/PERFORMANCE.md for #1448.
-        eligible AS (
-          SELECT stream, source, at, priority, lane, correlated_at
-          FROM ${this._fqs} s
-          WHERE blocked = false
-            ${lane_clause}
-            AND (leased_by IS NULL OR leased_until <= NOW())
-            AND (deferred_at IS NULL OR deferred_at <= NOW())
-        ),
+        -- Read from the base table so the planner can use the partial index
+        -- built for exactly this predicate:
+        --   (lane, priority DESC, at) WHERE blocked = false
+        --                               AND at < correlated_at
+        -- It contains only streams with work, so LIMIT pushes into it and a
+        -- claim scans at most lagging + leading rows. A stream leaves the
+        -- index when ack advances at to the mark, and re-enters when
+        -- correlate raises it.
+        --
+        -- The comparison is NULL-safe by SQL's own rules: an unmarked row
+        -- compares unknown and is excluded. That is definitional now
+        -- (#1446) \u2014 a subscription is claimable iff a mark says so \u2014 where
+        -- before #1488 it meant "unknown, fall through to the probe". An
+        -- install upgrading from before the column needs one correlate pass
+        -- from a rewound checkpoint to mark its rows; see the runbook in
+        -- docs/docs/guides/production-checklist.md.
         available AS (
-          -- Fast arm (#1485): a marked stream answers from the subscription
-          -- row alone. Read from the base table, NOT from the eligible CTE:
-          -- it is referenced several times so PG materializes it, and a
-          -- materialized scan cannot use the partial index this predicate was
-          -- built for (WHERE blocked = false AND at < correlated_at).
-          -- The comparison is NULL-safe: an unmarked row compares
-          -- unknown and is excluded here, falling through to the legacy arms.
           SELECT stream, source, at, priority, lane
           FROM ${this._fqs} s
           WHERE s.blocked = false
@@ -687,53 +685,6 @@ var PostgresStore = class {
             ${lane_clause}
             AND (s.leased_by IS NULL OR s.leased_until <= NOW())
             AND (s.deferred_at IS NULL OR s.deferred_at <= NOW())
-          UNION ALL
-          -- Legacy arms, gated on an absent mark. NULL means "unknown", so
-          -- an install that predates the column probes the event log exactly
-          -- as it did before, and each row migrates to the fast arm the first
-          -- time correlate marks it. Deleted once correlate marks universally.
-          --
-          -- Every arm below is watermark-agnostic (#1446): a fresh
-          -- subscription sits at -1, and e.id > s.at already answers
-          -- correctly there, since the first event has a greater id.
-          -- There used to be a fourth arm claiming at < 0 unconditionally,
-          -- which handed out an empty lease and a no-op ack for every
-          -- subscription with no matching events yet.
-          -- Source-less subscription: any non-snapshot event past the
-          -- watermark counts, so the id index alone answers it.
-          SELECT stream, source, at, priority, lane FROM eligible s
-          WHERE s.correlated_at IS NULL
-            AND s.source IS NULL
-            AND EXISTS (
-              SELECT 1 FROM ${this._fqt} e
-              WHERE e.id > s.at AND e.name <> '${SNAP_EVENT}' LIMIT 1
-            )
-          UNION ALL
-          -- Literal source (no regex metacharacter): exact equality, so
-          -- "s1" never claims "s12", and (stream, id) is usable.
-          SELECT stream, source, at, priority, lane FROM eligible s
-          WHERE s.correlated_at IS NULL
-            AND s.source IS NOT NULL
-            AND s.source !~ '${SOURCE_METACHARACTER_CLASS}'
-            AND EXISTS (
-              SELECT 1 FROM ${this._fqt} e
-              WHERE e.stream = s.source AND e.id > s.at
-                AND e.name <> '${SNAP_EVENT}' LIMIT 1
-            )
-          UNION ALL
-          -- Pattern source (e.g. '^(A|B)$'): POSIX match, so the
-          -- calculator's static regex reaction is claimed for every stream
-          -- it anchors. Unavoidably a scan \u2014 but only for the subscriptions
-          -- that actually declare a pattern.
-          SELECT stream, source, at, priority, lane FROM eligible s
-          WHERE s.correlated_at IS NULL
-            AND s.source IS NOT NULL
-            AND s.source ~ '${SOURCE_METACHARACTER_CLASS}'
-            AND EXISTS (
-              SELECT 1 FROM ${this._fqt} e
-              WHERE e.id > s.at AND e.name <> '${SNAP_EVENT}'
-                AND e.stream ~ s.source LIMIT 1
-            )
         ),
         -- Priority lanes (ACT-102): higher priority first, then
         -- lagging-watermark order. With everyone at priority=0 the
@@ -852,36 +803,19 @@ var PostgresStore = class {
         await client.query(
           `
           UPDATE ${this._fqs} t
-          SET priority = COALESCE((s->>'priority')::int, 0)
+          SET priority = GREATEST(t.priority, COALESCE((s->>'priority')::int, 0)),
+              lane = COALESCE(s->>'lane', 'default'),
+              correlated_at = GREATEST(t.correlated_at, (s->>'correlated_at')::int)
           FROM jsonb_array_elements($1::jsonb) AS s
           WHERE t.stream = s->>'stream'
-            AND COALESCE((s->>'priority')::int, 0) > t.priority
+            AND (COALESCE((s->>'priority')::int, 0) > t.priority
+                 OR t.lane <> COALESCE(s->>'lane', 'default')
+                 OR (s->>'correlated_at' IS NOT NULL
+                     AND (t.correlated_at IS NULL
+                          OR t.correlated_at < (s->>'correlated_at')::int)))
           `,
           [JSON.stringify(streams)]
         );
-        await client.query(
-          `
-          UPDATE ${this._fqs} t
-          SET lane = COALESCE(s->>'lane', 'default')
-          FROM jsonb_array_elements($1::jsonb) AS s
-          WHERE t.stream = s->>'stream'
-            AND t.lane <> COALESCE(s->>'lane', 'default')
-          `,
-          [JSON.stringify(streams)]
-        );
-        if (streams.some((s) => s.correlated_at !== void 0))
-          await client.query(
-            `
-          UPDATE ${this._fqs} t
-          SET correlated_at = (s->>'correlated_at')::int
-          FROM jsonb_array_elements($1::jsonb) AS s
-          WHERE t.stream = s->>'stream'
-            AND s->>'correlated_at' IS NOT NULL
-            AND (t.correlated_at IS NULL
-                 OR t.correlated_at < (s->>'correlated_at')::int)
-          `,
-            [JSON.stringify(streams)]
-          );
       }
       if (correlated_at !== void 0)
         await client.query(
@@ -1187,7 +1121,7 @@ var PostgresStore = class {
       values.push(query.after);
       conditions.push(`stream > $${values.length}`);
     }
-    let sql = `SELECT stream, source, at, retry, blocked, error, leased_by, leased_until, priority, lane, deferred_at FROM ${this._fqs}`;
+    let sql = `SELECT stream, source, at, retry, blocked, error, leased_by, leased_until, priority, lane, deferred_at, correlated_at FROM ${this._fqs}`;
     if (conditions.length) sql += " WHERE " + conditions.join(" AND ");
     values.push(limit);
     sql += ` ORDER BY stream LIMIT $${values.length}`;
@@ -1214,7 +1148,9 @@ var PostgresStore = class {
           lane: row.lane,
           // Persisted as timestamptz; surface as ms since epoch (#1221) so
           // the cold-start re-seed can re-arm the drain at the due-time.
-          deferred_at: row.deferred_at ? row.deferred_at.getTime() : void 0
+          deferred_at: row.deferred_at ? row.deferred_at.getTime() : void 0,
+          // NULL means "no mark yet" — unknown, not "no work" (#1485).
+          correlated_at: row.correlated_at ?? void 0
         });
         count++;
       }