@rotorsoft/act-pg 1.15.1 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/@types/postgres-store.d.ts.map +1 -1
- package/dist/index.cjs +40 -104
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +40 -104
- package/dist/index.js.map +1 -1
- package/package.json +7 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"postgres-store.d.ts","sourceRoot":"","sources":["../../src/postgres-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,YAAY,EACZ,SAAS,EACT,SAAS,EACT,KAAK,EAEL,OAAO,EACP,cAAc,EACd,KAAK,EACL,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,MAAM,EACN,OAAO,EACP,KAAK,EACL,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,WAAW,EACX,cAAc,EACf,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"postgres-store.d.ts","sourceRoot":"","sources":["../../src/postgres-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,YAAY,EACZ,SAAS,EACT,SAAS,EACT,KAAK,EAEL,OAAO,EACP,cAAc,EACd,KAAK,EACL,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,MAAM,EACN,OAAO,EACP,KAAK,EACL,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,WAAW,EACX,cAAc,EACf,MAAM,gBAAgB,CAAC;AAUxB,OAAO,EAEL,KAAK,UAAU,EAGhB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AA0BpB,KAAK,MAAM,GAAG,QAAQ,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,cAAc,CAAC,EAAE,UAAU,CAAC;CAC7B,CAAC,GACA,EAAE,CAAC,UAAU,CAAC;AAqGhB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2GG;AACH,qBAAa,aAAc,YAAW,KAAK;IACzC,OAAO,CAAC,KAAK,CAAC;IACd,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,IAAI,CAAS;IACrB,0DAA0D;IAC1D,OAAO,CAAC,IAAI,CAAS;IACrB;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAwB;IAC5C;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,8DAA8D;IAC9D,OAAO,CAAC,cAAc,CAA4B;IAClD;;;;;OAKG;IACH,OAAO,CAAC,eAAe,CAA+C;IACtE;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB,CAAqC;IAClE;;;;;;OAMG;IACH,OAAO,CAAC,eAAe,CAET;IACd;;;;OAIG;IACH,OAAO,CAAC,gBAAgB,CAA4C;IACpE;;;;OAIG;IACH,OAAO,CAAC,mBAAmB,CAAK;IAChC;;;;;;;;;OASG;IACH,MAAM,CAAC,EAAE,CACP,OAAO,EAAE,CAAC,YAAY,EAAE,iBAAiB,KAAK,IAAI,KAC/C,OAAO,CAAC,cAAc,CAAC,CAAC;IAE7B;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAsC;IAEvE;;;OAGG;IACH,YAAY,MAAM,GAAE,OAAO,CAAC,MAAM,CAAM,EAyBvC;IAED;;;;;;;OAOG;YACW,OAAO;IAQrB;;;;OAIG;IACG,OAAO,kBAGZ;IAED;;;;;;;;;;;;OAYG;YACW,gBAAgB;IAwB9B;;;;OAIG;IACG,IAAI,kBAmRT;IAED;;;OAGG;IACG,IAAI,kBAkBT;IAED;;;;;;;;;OASG;IACG,KAAK,CAAC,CAAC,SAAS,OAAO,EAC3B,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,IAAI,EAChD,KAAK,CAAC,EAAE,KAAK,mBA2Gd;IAED;;;;;;;;;OASG;IACG,MAAM,CAAC,CAAC,SAAS,OAAO,EAC5B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,OAAO,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,EAC3B,IAAI,EAAE,SAAS,EACf,eAAe,CAAC,EAAE,MAAM,oCAqKzB;IAED;;;;;;;;;;;;;OAaG;IACG,KAAK,CACT,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,KAAK,EAAE,CAAC,CAqJlB;IAED;;;;;;OAMG;IACG,SAAS,CACb,OAAO,EAAE,cAAc,EAAE,EACzB,aAAa,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAC,CA8E3E;IAED;;;;;OAKG;IACG,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CA8D3C;IAED;;;;OAIG;IACG,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA6C3D;IAED;;;;;;;;;OASG;IACG,KAAK,CACT,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,EAC9B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,CAAC,CAkBjB;IAED;;;;;OAKG;IACH;;;;;OAKG;IACH,OAAO,CAAC,cAAc;IAqChB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAiB3D;IAED;;;;;;;;;;;;;;OAcG;IACG,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAwB7D;IAED;;;;;;;;;;;;;OAaG;IACG,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMxE;IAED;;;;;;;;;OASG;IACG,aAAa,CACjB,QAAQ,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,EAC5C,KAAK,CAAC,EAAE,YAAY,GACnB,OAAO,CAAC,kBAAkB,CAAC,CAoG7B;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACG,WAAW,CAAC,CAAC,SAAS,OAAO,EACjC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,GAAG,cAAc,CAAC,EAC/D,OAAO,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAC7B,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAyEtC;IAED;;;;OAIG;YACW,uBAAuB;IA6CrC;;;;OAIG;YACW,sBAAsB;IA2GpC;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;YACW,wBAAwB;IA2BtC;;;;;;;OAOG;YACW,YAAY;IA2F1B;;;;;;;OAOG;IACH,OAAO,CAAC,UAAU;IAiDlB;;;;;;;;OAQG;IACG,QAAQ,CACZ,OAAO,EAAE,KAAK,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,SAAS,CAAC;QACjB,MAAM,CAAC,EAAE,IAAI,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,GACD,OAAO,CACR,GAAG,CACD,MAAM,EACN;QACE,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,CAAC;QAC7C,MAAM,CAAC,EAAE,IAAI,CAAC;KACf,CACF,CACF,CAsFA;IAED;;;;;;;;;;;OAWG;IACG,OAAO,CACX,MAAM,EAAE,CACN,QAAQ,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,KACpE,OAAO,CAAC,IAAI,CAAC,GACjB,OAAO,CAAC,IAAI,CAAC,CA4Cf;IAED;;;;;;;;;;;;;;;OAeG;IACG,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAMhD;CACF"}
|
package/dist/index.cjs
CHANGED
|
@@ -40,7 +40,6 @@ var import_act = require("@rotorsoft/act");
|
|
|
40
40
|
var import_act_crypto = require("@rotorsoft/act-crypto");
|
|
41
41
|
var import_pg = __toESM(require("pg"), 1);
|
|
42
42
|
var logger = (0, import_act.log)();
|
|
43
|
-
var SOURCE_METACHARACTER_CLASS = "[]^$.*+?()[{}|\\\\]";
|
|
44
43
|
var { Pool, types } = import_pg.default;
|
|
45
44
|
var JSONB_OID = types.builtins.JSONB;
|
|
46
45
|
var scopedTypes = {
|
|
@@ -360,6 +359,11 @@ var PostgresStore = class {
|
|
|
360
359
|
`ALTER TABLE ${this._fqs}
|
|
361
360
|
ADD COLUMN IF NOT EXISTS correlated_at int;`
|
|
362
361
|
);
|
|
362
|
+
await client.query(
|
|
363
|
+
`UPDATE ${this._fqs}
|
|
364
|
+
SET correlated_at = (SELECT COALESCE(MAX(id), -1) FROM ${this._fqt})
|
|
365
|
+
WHERE correlated_at IS NULL;`
|
|
366
|
+
);
|
|
363
367
|
await client.query(
|
|
364
368
|
`DO $$
|
|
365
369
|
BEGIN
|
|
@@ -672,38 +676,32 @@ var PostgresStore = class {
|
|
|
672
676
|
-- and lock EVERY claimable stream for this transaction, starving
|
|
673
677
|
-- overlapping competing consumers. We only lock the small
|
|
674
678
|
-- lagging+leading candidate slice, down in the "locked" CTE.
|
|
675
|
-
-- Eligibility, split by SOURCE CLASS rather than expressed as one
|
|
676
|
-
-- disjunction (#1448). The three classes are mutually exclusive, so
|
|
677
|
-
-- the UNION ALL returns exactly what the old OR-chain did \u2014 but each
|
|
678
|
-
-- arm now carries a single, planner-legible predicate.
|
|
679
679
|
--
|
|
680
|
-
--
|
|
681
|
-
--
|
|
682
|
-
--
|
|
683
|
-
--
|
|
684
|
-
--
|
|
685
|
-
--
|
|
686
|
-
--
|
|
687
|
-
-- majority \u2014 becomes an index seek on (stream, id).
|
|
680
|
+
-- Eligibility is a pure subscription-table predicate (#1488).
|
|
681
|
+
-- claim does not read the event log at all: correlate marks the
|
|
682
|
+
-- highest event id that resolves to a target, and at <
|
|
683
|
+
-- correlated_at is the whole question. The probe this replaced ran
|
|
684
|
+
-- an EXISTS against the events table once per eligible row, which
|
|
685
|
+
-- cost O(subscribed streams) per claim per worker no matter how
|
|
686
|
+
-- little work was pending.
|
|
688
687
|
--
|
|
689
|
-
--
|
|
690
|
-
--
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
688
|
+
-- Read from the base table so the planner can use the partial index
|
|
689
|
+
-- built for exactly this predicate:
|
|
690
|
+
-- (lane, priority DESC, at) WHERE blocked = false
|
|
691
|
+
-- AND at < correlated_at
|
|
692
|
+
-- It contains only streams with work, so LIMIT pushes into it and a
|
|
693
|
+
-- claim scans at most lagging + leading rows. A stream leaves the
|
|
694
|
+
-- index when ack advances at to the mark, and re-enters when
|
|
695
|
+
-- correlate raises it.
|
|
696
|
+
--
|
|
697
|
+
-- The comparison is NULL-safe by SQL's own rules: an unmarked row
|
|
698
|
+
-- compares unknown and is excluded. That is definitional now
|
|
699
|
+
-- (#1446) \u2014 a subscription is claimable iff a mark says so \u2014 where
|
|
700
|
+
-- before #1488 it meant "unknown, fall through to the probe". An
|
|
701
|
+
-- install upgrading from before the column needs one correlate pass
|
|
702
|
+
-- from a rewound checkpoint to mark its rows; see the runbook in
|
|
703
|
+
-- docs/docs/guides/production-checklist.md.
|
|
699
704
|
available AS (
|
|
700
|
-
-- Fast arm (#1485): a marked stream answers from the subscription
|
|
701
|
-
-- row alone. Read from the base table, NOT from the eligible CTE:
|
|
702
|
-
-- it is referenced several times so PG materializes it, and a
|
|
703
|
-
-- materialized scan cannot use the partial index this predicate was
|
|
704
|
-
-- built for (WHERE blocked = false AND at < correlated_at).
|
|
705
|
-
-- The comparison is NULL-safe: an unmarked row compares
|
|
706
|
-
-- unknown and is excluded here, falling through to the legacy arms.
|
|
707
705
|
SELECT stream, source, at, priority, lane
|
|
708
706
|
FROM ${this._fqs} s
|
|
709
707
|
WHERE s.blocked = false
|
|
@@ -711,53 +709,6 @@ var PostgresStore = class {
|
|
|
711
709
|
${lane_clause}
|
|
712
710
|
AND (s.leased_by IS NULL OR s.leased_until <= NOW())
|
|
713
711
|
AND (s.deferred_at IS NULL OR s.deferred_at <= NOW())
|
|
714
|
-
UNION ALL
|
|
715
|
-
-- Legacy arms, gated on an absent mark. NULL means "unknown", so
|
|
716
|
-
-- an install that predates the column probes the event log exactly
|
|
717
|
-
-- as it did before, and each row migrates to the fast arm the first
|
|
718
|
-
-- time correlate marks it. Deleted once correlate marks universally.
|
|
719
|
-
--
|
|
720
|
-
-- Every arm below is watermark-agnostic (#1446): a fresh
|
|
721
|
-
-- subscription sits at -1, and e.id > s.at already answers
|
|
722
|
-
-- correctly there, since the first event has a greater id.
|
|
723
|
-
-- There used to be a fourth arm claiming at < 0 unconditionally,
|
|
724
|
-
-- which handed out an empty lease and a no-op ack for every
|
|
725
|
-
-- subscription with no matching events yet.
|
|
726
|
-
-- Source-less subscription: any non-snapshot event past the
|
|
727
|
-
-- watermark counts, so the id index alone answers it.
|
|
728
|
-
SELECT stream, source, at, priority, lane FROM eligible s
|
|
729
|
-
WHERE s.correlated_at IS NULL
|
|
730
|
-
AND s.source IS NULL
|
|
731
|
-
AND EXISTS (
|
|
732
|
-
SELECT 1 FROM ${this._fqt} e
|
|
733
|
-
WHERE e.id > s.at AND e.name <> '${import_act.SNAP_EVENT}' LIMIT 1
|
|
734
|
-
)
|
|
735
|
-
UNION ALL
|
|
736
|
-
-- Literal source (no regex metacharacter): exact equality, so
|
|
737
|
-
-- "s1" never claims "s12", and (stream, id) is usable.
|
|
738
|
-
SELECT stream, source, at, priority, lane FROM eligible s
|
|
739
|
-
WHERE s.correlated_at IS NULL
|
|
740
|
-
AND s.source IS NOT NULL
|
|
741
|
-
AND s.source !~ '${SOURCE_METACHARACTER_CLASS}'
|
|
742
|
-
AND EXISTS (
|
|
743
|
-
SELECT 1 FROM ${this._fqt} e
|
|
744
|
-
WHERE e.stream = s.source AND e.id > s.at
|
|
745
|
-
AND e.name <> '${import_act.SNAP_EVENT}' LIMIT 1
|
|
746
|
-
)
|
|
747
|
-
UNION ALL
|
|
748
|
-
-- Pattern source (e.g. '^(A|B)$'): POSIX match, so the
|
|
749
|
-
-- calculator's static regex reaction is claimed for every stream
|
|
750
|
-
-- it anchors. Unavoidably a scan \u2014 but only for the subscriptions
|
|
751
|
-
-- that actually declare a pattern.
|
|
752
|
-
SELECT stream, source, at, priority, lane FROM eligible s
|
|
753
|
-
WHERE s.correlated_at IS NULL
|
|
754
|
-
AND s.source IS NOT NULL
|
|
755
|
-
AND s.source ~ '${SOURCE_METACHARACTER_CLASS}'
|
|
756
|
-
AND EXISTS (
|
|
757
|
-
SELECT 1 FROM ${this._fqt} e
|
|
758
|
-
WHERE e.id > s.at AND e.name <> '${import_act.SNAP_EVENT}'
|
|
759
|
-
AND e.stream ~ s.source LIMIT 1
|
|
760
|
-
)
|
|
761
712
|
),
|
|
762
713
|
-- Priority lanes (ACT-102): higher priority first, then
|
|
763
714
|
-- lagging-watermark order. With everyone at priority=0 the
|
|
@@ -876,36 +827,19 @@ var PostgresStore = class {
|
|
|
876
827
|
await client.query(
|
|
877
828
|
`
|
|
878
829
|
UPDATE ${this._fqs} t
|
|
879
|
-
SET priority = COALESCE((s->>'priority')::int, 0)
|
|
830
|
+
SET priority = GREATEST(t.priority, COALESCE((s->>'priority')::int, 0)),
|
|
831
|
+
lane = COALESCE(s->>'lane', 'default'),
|
|
832
|
+
correlated_at = GREATEST(t.correlated_at, (s->>'correlated_at')::int)
|
|
880
833
|
FROM jsonb_array_elements($1::jsonb) AS s
|
|
881
834
|
WHERE t.stream = s->>'stream'
|
|
882
|
-
AND COALESCE((s->>'priority')::int, 0) > t.priority
|
|
835
|
+
AND (COALESCE((s->>'priority')::int, 0) > t.priority
|
|
836
|
+
OR t.lane <> COALESCE(s->>'lane', 'default')
|
|
837
|
+
OR (s->>'correlated_at' IS NOT NULL
|
|
838
|
+
AND (t.correlated_at IS NULL
|
|
839
|
+
OR t.correlated_at < (s->>'correlated_at')::int)))
|
|
883
840
|
`,
|
|
884
841
|
[JSON.stringify(streams)]
|
|
885
842
|
);
|
|
886
|
-
await client.query(
|
|
887
|
-
`
|
|
888
|
-
UPDATE ${this._fqs} t
|
|
889
|
-
SET lane = COALESCE(s->>'lane', 'default')
|
|
890
|
-
FROM jsonb_array_elements($1::jsonb) AS s
|
|
891
|
-
WHERE t.stream = s->>'stream'
|
|
892
|
-
AND t.lane <> COALESCE(s->>'lane', 'default')
|
|
893
|
-
`,
|
|
894
|
-
[JSON.stringify(streams)]
|
|
895
|
-
);
|
|
896
|
-
if (streams.some((s) => s.correlated_at !== void 0))
|
|
897
|
-
await client.query(
|
|
898
|
-
`
|
|
899
|
-
UPDATE ${this._fqs} t
|
|
900
|
-
SET correlated_at = (s->>'correlated_at')::int
|
|
901
|
-
FROM jsonb_array_elements($1::jsonb) AS s
|
|
902
|
-
WHERE t.stream = s->>'stream'
|
|
903
|
-
AND s->>'correlated_at' IS NOT NULL
|
|
904
|
-
AND (t.correlated_at IS NULL
|
|
905
|
-
OR t.correlated_at < (s->>'correlated_at')::int)
|
|
906
|
-
`,
|
|
907
|
-
[JSON.stringify(streams)]
|
|
908
|
-
);
|
|
909
843
|
}
|
|
910
844
|
if (correlated_at !== void 0)
|
|
911
845
|
await client.query(
|
|
@@ -1211,7 +1145,7 @@ var PostgresStore = class {
|
|
|
1211
1145
|
values.push(query.after);
|
|
1212
1146
|
conditions.push(`stream > $${values.length}`);
|
|
1213
1147
|
}
|
|
1214
|
-
let sql = `SELECT stream, source, at, retry, blocked, error, leased_by, leased_until, priority, lane, deferred_at FROM ${this._fqs}`;
|
|
1148
|
+
let sql = `SELECT stream, source, at, retry, blocked, error, leased_by, leased_until, priority, lane, deferred_at, correlated_at FROM ${this._fqs}`;
|
|
1215
1149
|
if (conditions.length) sql += " WHERE " + conditions.join(" AND ");
|
|
1216
1150
|
values.push(limit);
|
|
1217
1151
|
sql += ` ORDER BY stream LIMIT $${values.length}`;
|
|
@@ -1238,7 +1172,9 @@ var PostgresStore = class {
|
|
|
1238
1172
|
lane: row.lane,
|
|
1239
1173
|
// Persisted as timestamptz; surface as ms since epoch (#1221) so
|
|
1240
1174
|
// the cold-start re-seed can re-arm the drain at the due-time.
|
|
1241
|
-
deferred_at: row.deferred_at ? row.deferred_at.getTime() : void 0
|
|
1175
|
+
deferred_at: row.deferred_at ? row.deferred_at.getTime() : void 0,
|
|
1176
|
+
// NULL means "no mark yet" — unknown, not "no work" (#1485).
|
|
1177
|
+
correlated_at: row.correlated_at ?? void 0
|
|
1242
1178
|
});
|
|
1243
1179
|
count++;
|
|
1244
1180
|
}
|