@rotorsoft/act-pg 1.16.1 → 1.17.1

package/dist/index.js

@@ -16,7 +16,6 @@ import {
 } from "@rotorsoft/act-crypto";
 import pg from "pg";
 var logger = log();
-var SOURCE_METACHARACTER_CLASS = "[]^$.*+?()[{}|\\\\]";
 var { Pool, types } = pg;
 var JSONB_OID = types.builtins.JSONB;
 var scopedTypes = {
@@ -336,6 +335,11 @@ var PostgresStore = class {
         `ALTER TABLE ${this._fqs}
          ADD COLUMN IF NOT EXISTS correlated_at int;`
       );
+      await client.query(
+        `UPDATE ${this._fqs}
+         SET correlated_at = (SELECT COALESCE(MAX(id), -1) FROM ${this._fqt})
+         WHERE correlated_at IS NULL;`
+      );
       await client.query(
         `DO $$
          BEGIN
@@ -389,6 +393,11 @@ var PostgresStore = class {
         ON ${this._fqs} (lane, priority DESC, at)
         WHERE blocked = false AND at < correlated_at;`
       );
+      await client.query(
+        `CREATE INDEX IF NOT EXISTS "${this.config.table}_streams_at_ix"
+        ON ${this._fqs} (at)
+        WHERE blocked = false AND at < correlated_at;`
+      );
       await client.query("COMMIT");
       logger.info(
         `Seeded schema "${this.config.schema}" with table "${this.config.table}"`
@@ -648,129 +657,73 @@ var PostgresStore = class {
         -- and lock EVERY claimable stream for this transaction, starving
         -- overlapping competing consumers. We only lock the small
         -- lagging+leading candidate slice, down in the "locked" CTE.
-        -- Eligibility, split by SOURCE CLASS rather than expressed as one
-        -- disjunction (#1448). The three classes are mutually exclusive, so
-        -- the UNION ALL returns exactly what the old OR-chain did \u2014 but each
-        -- arm now carries a single, planner-legible predicate.
         --
-        -- That matters for one arm in particular. Written as
-        -- "s.source IS NULL OR (... AND e.stream = s.source) OR (... ~ ...)",
-        -- the equality is buried inside a disjunction and is NOT sargable:
-        -- no index on (stream, id) can serve it, so every probe degenerated
-        -- into a forward scan of the pk from the stream's watermark. With
-        -- the classes separated, the literal arm \u2014 every per-aggregate
-        -- .to(e => ({target: e.stream})) reaction, i.e. the overwhelming
-        -- majority \u2014 becomes an index seek on (stream, id).
+        -- Eligibility is a pure subscription-table predicate (#1488).
+        -- claim does not read the event log at all: correlate marks the
+        -- highest event id that resolves to a target, and at <
+        -- correlated_at is the whole question. The probe this replaced ran
+        -- an EXISTS against the events table once per eligible row, which
+        -- cost O(subscribed streams) per claim per worker no matter how
+        -- little work was pending.
+        --
+        -- The predicate is repeated in each arm below rather than factored
+        -- into a shared CTE, and that repetition is load-bearing (#1510). A
+        -- CTE referenced more than once is materialized, so LIMIT 8 was
+        -- applied to a fully-built 100,000-row result instead of pushing into
+        -- the index: measured at 75.6 ms for a claim that returns 8 rows.
+        -- Reading the base table in each arm lets the planner stop after the
+        -- limit \u2014 19.9 ms with the existing index, 2.0 ms once the
+        -- at-ordered partial index below serves the two watermark arms.
         --
-        -- Measured at 10k subscribed streams with 10 pending: 5,792 ms
-        -- before, 10.3 ms after. See libs/act-pg/PERFORMANCE.md for #1448.
-        eligible AS (
-          SELECT stream, source, at, priority, lane, correlated_at
+        -- A row with no mark compares unknown and is excluded by SQL's own
+        -- rules. That is definitional (#1446): a subscription is claimable iff
+        -- a mark says so. seed() marks rows that predate the column.
+        --
+        -- Priority lanes (ACT-102): higher priority first, then
+        -- lagging-watermark order. With everyone at priority=0 the ORDER BY
+        -- collapses to plain at ASC, so existing workloads see no change.
+        --
+        -- The lagging frontier is a UNION of two portions (ACT-1223): the
+        -- priority portion takes the first (lagging - fair) slots by
+        -- priority DESC, at ASC; a fairness reserve then fills fair more
+        -- by pure at ASC (priority ignored), excluding the ones already
+        -- chosen, so a default-priority lagging stream is never starved out
+        -- by sustained higher-priority load.
+        prio AS (
+          SELECT stream, source, at, lane, TRUE AS lagging
           FROM ${this._fqs} s
-          WHERE blocked = false
+          WHERE s.blocked = false
+            AND s.at < s.correlated_at
             ${lane_clause}
-            AND (leased_by IS NULL OR leased_until <= NOW())
-            AND (deferred_at IS NULL OR deferred_at <= NOW())
+            AND (s.leased_by IS NULL OR s.leased_until <= NOW())
+            AND (s.deferred_at IS NULL OR s.deferred_at <= NOW())
+          ORDER BY s.priority DESC, s.at ASC
+          LIMIT ($1::int - $5::int)
         ),
-        available AS (
-          -- Fast arm (#1485): a marked stream answers from the subscription
-          -- row alone. Read from the base table, NOT from the eligible CTE:
-          -- it is referenced several times so PG materializes it, and a
-          -- materialized scan cannot use the partial index this predicate was
-          -- built for (WHERE blocked = false AND at < correlated_at).
-          -- The comparison is NULL-safe: an unmarked row compares
-          -- unknown and is excluded here, falling through to the legacy arms.
-          SELECT stream, source, at, priority, lane
+        fair AS (
+          SELECT stream, source, at, lane, TRUE AS lagging
           FROM ${this._fqs} s
           WHERE s.blocked = false
             AND s.at < s.correlated_at
             ${lane_clause}
             AND (s.leased_by IS NULL OR s.leased_until <= NOW())
             AND (s.deferred_at IS NULL OR s.deferred_at <= NOW())
-          UNION ALL
-          -- Legacy arms, gated on an absent mark. NULL means "unknown", so
-          -- an install that predates the column probes the event log exactly
-          -- as it did before, and each row migrates to the fast arm the first
-          -- time correlate marks it. Deleted once correlate marks universally.
-          --
-          -- Every arm below is watermark-agnostic (#1446): a fresh
-          -- subscription sits at -1, and e.id > s.at already answers
-          -- correctly there, since the first event has a greater id.
-          -- There used to be a fourth arm claiming at < 0 unconditionally,
-          -- which handed out an empty lease and a no-op ack for every
-          -- subscription with no matching events yet.
-          -- Source-less subscription: any non-snapshot event past the
-          -- watermark counts, so the id index alone answers it.
-          SELECT stream, source, at, priority, lane FROM eligible s
-          WHERE s.correlated_at IS NULL
-            AND s.source IS NULL
-            AND EXISTS (
-              SELECT 1 FROM ${this._fqt} e
-              WHERE e.id > s.at AND e.name <> '${SNAP_EVENT}' LIMIT 1
-            )
-          UNION ALL
-          -- Literal source (no regex metacharacter): exact equality, so
-          -- "s1" never claims "s12", and (stream, id) is usable.
-          SELECT stream, source, at, priority, lane FROM eligible s
-          WHERE s.correlated_at IS NULL
-            AND s.source IS NOT NULL
-            AND s.source !~ '${SOURCE_METACHARACTER_CLASS}'
-            AND EXISTS (
-              SELECT 1 FROM ${this._fqt} e
-              WHERE e.stream = s.source AND e.id > s.at
-                AND e.name <> '${SNAP_EVENT}' LIMIT 1
-            )
-          UNION ALL
-          -- Pattern source (e.g. '^(A|B)$'): POSIX match, so the
-          -- calculator's static regex reaction is claimed for every stream
-          -- it anchors. Unavoidably a scan \u2014 but only for the subscriptions
-          -- that actually declare a pattern.
-          SELECT stream, source, at, priority, lane FROM eligible s
-          WHERE s.correlated_at IS NULL
-            AND s.source IS NOT NULL
-            AND s.source ~ '${SOURCE_METACHARACTER_CLASS}'
-            AND EXISTS (
-              SELECT 1 FROM ${this._fqt} e
-              WHERE e.id > s.at AND e.name <> '${SNAP_EVENT}'
-                AND e.stream ~ s.source LIMIT 1
-            )
+            AND s.stream NOT IN (SELECT stream FROM prio)
+          ORDER BY s.at ASC
+          LIMIT $5
         ),
-        -- Priority lanes (ACT-102): higher priority first, then
-        -- lagging-watermark order. With everyone at priority=0 the
-        -- ORDER BY collapses to plain at ASC so existing workloads
-        -- see no behavior change.
-        --
-        -- The lagging frontier is a UNION of two portions (ACT-1223): the
-        -- priority-ordered portion takes the first (lagging - fair) slots
-        -- by priority DESC, at ASC; a fairness reserve then fills fair more
-        -- slots by pure at ASC (priority ignored), excluding the ones
-        -- already chosen, so a default-priority lagging stream is never
-        -- starved out by sustained higher-priority load. With all
-        -- priorities equal both portions order by at, a no-op merge.
         lag AS (
-          (
-            SELECT stream, source, at, lane, TRUE AS lagging
-            FROM available
-            ORDER BY priority DESC, at ASC
-            LIMIT ($1::int - $5::int)
-          )
-          UNION
-          (
-            SELECT stream, source, at, lane, TRUE AS lagging
-            FROM available
-            WHERE stream NOT IN (
-              SELECT stream FROM available
-              ORDER BY priority DESC, at ASC
-              LIMIT ($1::int - $5::int)
-            )
-            ORDER BY at ASC
-            LIMIT $5
-          )
+          SELECT * FROM prio UNION SELECT * FROM fair
         ),
         lead AS (
           SELECT stream, source, at, lane, FALSE AS lagging
-          FROM available
-          ORDER BY at DESC
+          FROM ${this._fqs} s
+          WHERE s.blocked = false
+            AND s.at < s.correlated_at
+            ${lane_clause}
+            AND (s.leased_by IS NULL OR s.leased_until <= NOW())
+            AND (s.deferred_at IS NULL OR s.deferred_at <= NOW())
+          ORDER BY s.at DESC
           LIMIT $2
         ),
         combined AS (