@kairos-es/store-postgres 0.0.0 → 1.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/config.js +2 -1
- package/dist/esm/config.js.map +1 -1
- package/package.json +2 -2
- package/dist/cjs/internal/subscribe.js +0 -120
- package/dist/cjs/internal/subscribe.js.map +0 -1
- package/dist/dts/internal/subscribe.d.ts +0 -87
- package/dist/dts/internal/subscribe.d.ts.map +0 -1
- package/dist/esm/internal/subscribe.js +0 -113
- package/dist/esm/internal/subscribe.js.map +0 -1
package/dist/esm/config.js
CHANGED
|
@@ -73,7 +73,8 @@ export const PostgresStoreConfig = /*#__PURE__*/Schema.Struct({
|
|
|
73
73
|
// The two subscribe knobs are core's field definitions, not this engine's: see
|
|
74
74
|
// `subscriptionTuningFields`.
|
|
75
75
|
...subscriptionTuningFields
|
|
76
|
-
}).pipe(
|
|
76
|
+
}).pipe(
|
|
77
|
+
/*#__PURE__*/
|
|
77
78
|
// Cross-field guard: the derived table/index/function/channel names must all
|
|
78
79
|
// fit Postgres's 63-byte identifier limit. Returning the message string fails
|
|
79
80
|
// the decode with it; `undefined`/`true` passes.
|
package/dist/esm/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","names":["subscriptionTuningFields","Schema","oversizedIdentifier","IdentifierAtom","String","pipe","nonEmptyString","pattern","identifier","description","PostgresStoreConfig","Struct","schema","optionalWith","exact","tablePrefix","lockTimeout","Number","nonNegative","finite","filter","config","decodePostgresStoreConfig","decodeUnknownSync"],"sources":["../../src/config.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;AAoBA,SAASA,wBAAwB,QAAQ,iBAAiB;AAC1D,SAASC,MAAM,QAAQ,QAAQ;AAC/B,SAASC,mBAAmB,QAAQ,mBAAgB;AAEpD;;;;;;AAMA,MAAMC,cAAc,gBAAGF,MAAM,CAACG,MAAM,CAACC,IAAI,cACvCJ,MAAM,CAACK,cAAc,EAAE,eACvBL,MAAM,CAACM,OAAO,CAAC,SAAS,EAAE;EACxBC,UAAU,EAAE,wBAAwB;EACpCC,WAAW,EAAE;CACd,CAAC,CACH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;AA0BA,OAAO,MAAMC,mBAAmB,gBAAGT,MAAM,CAACU,MAAM,CAAC;EAC/C;EACA;EACA;EACAC,MAAM,EAAEX,MAAM,CAACY,YAAY,CAACV,cAAc,EAAE;IAAEW,KAAK,EAAE;EAAI,CAAE,CAAC;EAC5DC,WAAW,EAAEd,MAAM,CAACY,YAAY,CAACV,cAAc,EAAE;IAAEW,KAAK,EAAE;EAAI,CAAE,CAAC;EACjEE,WAAW,EAAEf,MAAM,CAACY,YAAY,CAC9BZ,MAAM,CAACgB,MAAM,CAACZ,IAAI,CAACJ,MAAM,CAACiB,WAAW,EAAE,EAAEjB,MAAM,CAACkB,MAAM,EAAE,CAAC,EACzD;IAAEL,KAAK,EAAE;EAAI,CAAE,CAChB;EACD;EACA;EACA,GAAGd;CACJ,CAAC,CAACK,IAAI;AACL;AACA;AACA;AACAJ,MAAM,CAACmB,MAAM,CAAEC,MAAM,IAAKnB,mBAAmB,CAACmB,MAAM,CAAC,IAAI,IAAI,CAAC,CAC/D;AAGD;;;;;AAKA,OAAO,MAAMC,yBAAyB,gBACpCrB,MAAM,CAACsB,iBAAiB,CAACb,mBAAmB,CAAC","ignoreList":[]}
|
|
1
|
+
{"version":3,"file":"config.js","names":["subscriptionTuningFields","Schema","oversizedIdentifier","IdentifierAtom","String","pipe","nonEmptyString","pattern","identifier","description","PostgresStoreConfig","Struct","schema","optionalWith","exact","tablePrefix","lockTimeout","Number","nonNegative","finite","filter","config","decodePostgresStoreConfig","decodeUnknownSync"],"sources":["../../src/config.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;AAoBA,SAASA,wBAAwB,QAAQ,iBAAiB;AAC1D,SAASC,MAAM,QAAQ,QAAQ;AAC/B,SAASC,mBAAmB,QAAQ,mBAAgB;AAEpD;;;;;;AAMA,MAAMC,cAAc,gBAAGF,MAAM,CAACG,MAAM,CAACC,IAAI,cACvCJ,MAAM,CAACK,cAAc,EAAE,eACvBL,MAAM,CAACM,OAAO,CAAC,SAAS,EAAE;EACxBC,UAAU,EAAE,wBAAwB;EACpCC,WAAW,EAAE;CACd,CAAC,CACH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;AA0BA,OAAO,MAAMC,mBAAmB,gBAAGT,MAAM,CAACU,MAAM,CAAC;EAC/C;EACA;EACA;EACAC,MAAM,EAAEX,MAAM,CAACY,YAAY,CAACV,cAAc,EAAE;IAAEW,KAAK,EAAE;EAAI,CAAE,CAAC;EAC5DC,WAAW,EAAEd,MAAM,CAACY,YAAY,CAACV,cAAc,EAAE;IAAEW,KAAK,EAAE;EAAI,CAAE,CAAC;EACjEE,WAAW,EAAEf,MAAM,CAACY,YAAY,CAC9BZ,MAAM,CAACgB,MAAM,CAACZ,IAAI,CAACJ,MAAM,CAACiB,WAAW,EAAE,EAAEjB,MAAM,CAACkB,MAAM,EAAE,CAAC,EACzD;IAAEL,KAAK,EAAE;EAAI,CAAE,CAChB;EACD;EACA;EACA,GAAGd;CACJ,CAAC,CAACK,IAAI;AAAA;AACL;AACA;AACA;AACAJ,MAAM,CAACmB,MAAM,CAAEC,MAAM,IAAKnB,mBAAmB,CAACmB,MAAM,CAAC,IAAI,IAAI,CAAC,CAC/D;AAGD;;;;;AAKA,OAAO,MAAMC,yBAAyB,gBACpCrB,MAAM,CAACsB,iBAAiB,CAACb,mBAAmB,CAAC","ignoreList":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kairos-es/store-postgres",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "1.0.0-alpha.1",
|
|
4
4
|
"description": "Postgres DcbEventStore backend for kairos-es",
|
|
5
5
|
"license": "BSD-3-Clause",
|
|
6
6
|
"repository": {
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"@effect/sql": "^0.52.0",
|
|
17
17
|
"@effect/sql-pg": "^0.53.0",
|
|
18
18
|
"effect": "^3.22.0",
|
|
19
|
-
"@kairos-es/core": "^0.0.
|
|
19
|
+
"@kairos-es/core": "^1.0.0-alpha.1"
|
|
20
20
|
},
|
|
21
21
|
"main": "./dist/cjs/index.js",
|
|
22
22
|
"module": "./dist/esm/index.js",
|
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.subscribeStream = exports.DEFAULT_POLL_INTERVAL_MILLIS = exports.DEFAULT_CATCH_UP_PAGE = void 0;
|
|
7
|
-
var _core = require("@kairos-es/core");
|
|
8
|
-
var _effect = require("effect");
|
|
9
|
-
/**
|
|
10
|
-
* A short catch-up page size default for the reconcile loop. Reads page by page
|
|
11
|
-
* until a short page signals the tail is drained, so a large backlog is streamed
|
|
12
|
-
* in bounded chunks rather than one huge SELECT. Overridable via
|
|
13
|
-
* `config.catchUpPageSize`.
|
|
14
|
-
*/
|
|
15
|
-
const DEFAULT_CATCH_UP_PAGE = exports.DEFAULT_CATCH_UP_PAGE = 512;
|
|
16
|
-
/**
|
|
17
|
-
* The default upper bound a live subscriber waits for a wake before re-reading
|
|
18
|
-
* from its cursor ANYWAY (milliseconds; overridable via `config.pollInterval`).
|
|
19
|
-
* This periodic position-based poll — not `NOTIFY` — is the correctness
|
|
20
|
-
* guarantee: every committed position is reconciled within this interval
|
|
21
|
-
* regardless of whether its `NOTIFY` was heard, which covers a `NOTIFY` fired
|
|
22
|
-
* before `LISTEN` was fully established, a coalesced/lost `NOTIFY`, and a listen
|
|
23
|
-
* connection that dropped SILENTLY (the `@effect/sql-pg` listen client swallows
|
|
24
|
-
* connection errors and does not auto-reconnect). `NOTIFY` only lowers delivery
|
|
25
|
-
* latency below this interval. Upstream `postgres_tt.py` uses the same 1-second
|
|
26
|
-
* fallback poll (`conn.wait(..., interval=1)`).
|
|
27
|
-
*/
|
|
28
|
-
const DEFAULT_POLL_INTERVAL_MILLIS = exports.DEFAULT_POLL_INTERVAL_MILLIS = 1000;
|
|
29
|
-
/**
|
|
30
|
-
* Retry a TRANSIENT poll fault (a `SqlError` from one reconcile read) on a capped
|
|
31
|
-
* exponential backoff rather than letting `Stream.orDie` kill a subscription that
|
|
32
|
-
* may have been alive for days over a single connection blip. A fault that
|
|
33
|
-
* PERSISTS past the cap propagates (the decorator dies it) — infrastructure stays
|
|
34
|
-
* off the contract channel while the stream survives ordinary transients.
|
|
35
|
-
*/
|
|
36
|
-
const POLL_RETRY_SCHEDULE = /*#__PURE__*/_effect.Schedule.exponential('100 millis', 2).pipe(/*#__PURE__*/_effect.Schedule.union(/*#__PURE__*/_effect.Schedule.spaced('2 seconds')), /*#__PURE__*/_effect.Schedule.intersect(/*#__PURE__*/_effect.Schedule.recurs(20)));
|
|
37
|
-
/**
|
|
38
|
-
* Build the subscription `Stream`: catch up from `after`, then track the live
|
|
39
|
-
* tail. The cursor threads through the `unfoldChunkEffect` unfold state, starting
|
|
40
|
-
* at `after` and advancing to the last delivered position each batch; the stream
|
|
41
|
-
* never completes on its own (always `Option.some`), so the enclosing scope
|
|
42
|
-
* interrupts it on close. A `SqlError` mid-stream stays on the channel;
|
|
43
|
-
* `makeContractStore` dies it (subscribe is `E = never` on the contract).
|
|
44
|
-
*/
|
|
45
|
-
const subscribeStream = (deps, after) => _effect.Stream.unwrapScoped(_effect.Effect.gen(function* () {
|
|
46
|
-
// The wake primitive: a sliding(1) queue holding at most one pending
|
|
47
|
-
// signal. The listener `offer`s an opaque wake on every NOTIFY; the poll
|
|
48
|
-
// loop `take`s (with a timeout). The lost-wakeup guard is STRUCTURAL —
|
|
49
|
-
// drain the queue BEFORE reading, so a NOTIFY that lands mid-read leaves
|
|
50
|
-
// a fresh signal that makes the next `take` return at once rather than
|
|
51
|
-
// sleeping through a real change.
|
|
52
|
-
const wake = yield* _effect.Queue.sliding(1);
|
|
53
|
-
// Tap every notification into the wake queue. If the listen stream DOES
|
|
54
|
-
// surface an error, re-subscribe with capped-exponential backoff. In
|
|
55
|
-
// practice the @effect/sql-pg listen client usually swallows a dropped
|
|
56
|
-
// connection SILENTLY rather than erroring, so this retry rarely fires —
|
|
57
|
-
// the poll fallback below, not this retry, is the real reconnect guarantee:
|
|
58
|
-
// the reconcile loop re-reads from the cursor on every poll, so a silently-
|
|
59
|
-
// dropped (and later re-established) listener loses no positions.
|
|
60
|
-
const listener = deps.listen.pipe(_effect.Stream.tap(() => _effect.Queue.offer(wake, undefined)), _effect.Stream.retry(_effect.Schedule.exponential('50 millis').pipe(_effect.Schedule.union(_effect.Schedule.spaced('1 seconds')))));
|
|
61
|
-
// Drain the listener into the wake queue on a background fibre; the scope
|
|
62
|
-
// releases it. We never read its elements directly — every notification is
|
|
63
|
-
// an opaque wake signal.
|
|
64
|
-
yield* listener.pipe(_effect.Stream.runDrain, _effect.Effect.forkScoped);
|
|
65
|
-
// Reconcile: page-read from `from` until a short page, advancing to the last
|
|
66
|
-
// delivered position. Retries a TRANSIENT poll fault on a capped schedule so
|
|
67
|
-
// a single connection blip does not kill a days-long subscription; a
|
|
68
|
-
// persistent fault propagates and dies.
|
|
69
|
-
const drainFrom = from => _effect.Effect.gen(function* () {
|
|
70
|
-
const collected = [];
|
|
71
|
-
let cursor = from;
|
|
72
|
-
// Loop pages until a short page signals the tail is drained.
|
|
73
|
-
while (true) {
|
|
74
|
-
const events = yield* deps.fetch(cursor, _core.ReadLimit.make(deps.catchUpPage));
|
|
75
|
-
for (const event of events) {
|
|
76
|
-
collected.push(event);
|
|
77
|
-
}
|
|
78
|
-
const lastEvent = events.at(-1);
|
|
79
|
-
if (lastEvent === undefined || events.length < deps.catchUpPage) {
|
|
80
|
-
break;
|
|
81
|
-
}
|
|
82
|
-
cursor = lastEvent.position;
|
|
83
|
-
}
|
|
84
|
-
const last = collected.at(-1)?.position ?? from;
|
|
85
|
-
return {
|
|
86
|
-
events: collected,
|
|
87
|
-
last
|
|
88
|
-
};
|
|
89
|
-
}).pipe(_effect.Effect.retry(POLL_RETRY_SCHEDULE));
|
|
90
|
-
/**
|
|
91
|
-
* Produce the NEXT non-empty batch from a cursor position, together with
|
|
92
|
-
* the advanced cursor — a pure `Position → Effect<[Chunk, next]>`. The
|
|
93
|
-
* `> boundary` de-dup is enforced by `fetch`'s exclusive lower bound, so a
|
|
94
|
-
* live re-read after a wake never re-emits an already-delivered position.
|
|
95
|
-
*
|
|
96
|
-
* Drain the wake queue BEFORE reading (lost-wakeup guard), then drain from
|
|
97
|
-
* the cursor. If a drain yields nothing, wait for a wake but only up to the
|
|
98
|
-
* poll interval, then loop and re-read ANYWAY — the periodic position-based
|
|
99
|
-
* poll that makes delivery correct independently of NOTIFY. This effect only
|
|
100
|
-
* completes by returning a non-empty chunk; the scope interrupts it on close.
|
|
101
|
-
*/
|
|
102
|
-
const nextBatch = from => _effect.Effect.gen(function* () {
|
|
103
|
-
while (true) {
|
|
104
|
-
yield* _effect.Queue.takeAll(wake);
|
|
105
|
-
const {
|
|
106
|
-
events,
|
|
107
|
-
last
|
|
108
|
-
} = yield* drainFrom(from);
|
|
109
|
-
if (events.length > 0) {
|
|
110
|
-
return [_effect.Chunk.fromIterable(events), last];
|
|
111
|
-
}
|
|
112
|
-
// Nothing new: block on a wake, but only up to the poll interval — on
|
|
113
|
-
// timeout we fall through and the loop re-reads regardless.
|
|
114
|
-
yield* _effect.Effect.ignore(_effect.Effect.timeout(_effect.Queue.take(wake), deps.pollInterval));
|
|
115
|
-
}
|
|
116
|
-
});
|
|
117
|
-
return _effect.Stream.unfoldChunkEffect(after ?? _core.ORIGIN, from => _effect.Effect.map(nextBatch(from), _effect.Option.some));
|
|
118
|
-
}));
|
|
119
|
-
exports.subscribeStream = subscribeStream;
|
|
120
|
-
//# sourceMappingURL=subscribe.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"subscribe.js","names":["_core","require","_effect","DEFAULT_CATCH_UP_PAGE","exports","DEFAULT_POLL_INTERVAL_MILLIS","POLL_RETRY_SCHEDULE","Schedule","exponential","pipe","union","spaced","intersect","recurs","subscribeStream","deps","after","Stream","unwrapScoped","Effect","gen","wake","Queue","sliding","listener","listen","tap","offer","undefined","retry","runDrain","forkScoped","drainFrom","from","collected","cursor","events","fetch","ReadLimit","make","catchUpPage","event","push","lastEvent","at","length","position","last","nextBatch","takeAll","Chunk","fromIterable","ignore","timeout","take","pollInterval","unfoldChunkEffect","ORIGIN","map","Option","some"],"sources":["../../../src/internal/subscribe.ts"],"sourcesContent":[null],"mappings":";;;;;;AAgCA,IAAAA,KAAA,GAAAC,OAAA;AACA,IAAAC,OAAA,GAAAD,OAAA;AAUA;;;;;;AAMO,MAAME,qBAAqB,GAAAC,OAAA,CAAAD,qBAAA,GAAG,GAAG;AAExC;;;;;;;;;;;;AAYO,MAAME,4BAA4B,GAAAD,OAAA,CAAAC,4BAAA,GAAG,IAAI;AAEhD;;;;;;;AAOA,MAAMC,mBAAmB,gBAAGC,gBAAQ,CAACC,WAAW,CAAC,YAAY,EAAE,CAAC,CAAC,CAACC,IAAI,cACpEF,gBAAQ,CAACG,KAAK,cAACH,gBAAQ,CAACI,MAAM,CAAC,WAAW,CAAC,CAAC,eAC5CJ,gBAAQ,CAACK,SAAS,cAACL,gBAAQ,CAACM,MAAM,CAAC,EAAE,CAAC,CAAC,CACxC;AA6BD;;;;;;;;AAQO,MAAMC,eAAe,GAAGA,CAC7BC,IAAsB,EACtBC,KAA2B,KAE3BC,cAAM,CAACC,YAAY,CACjBC,cAAM,CAACC,GAAG,CAAC,aAAS;EAClB;EACA;EACA;EACA;EACA;EACA;EACA,MAAMC,IAAI,GAAG,OAAOC,aAAK,CAACC,OAAO,CAAO,CAAC,CAAC;EAE1C;EACA;EACA;EACA;EACA;EACA;EACA;EACA,MAAMC,QAAQ,GAAGT,IAAI,CAACU,MAAM,CAAChB,IAAI,CAC/BQ,cAAM,CAACS,GAAG,CAAC,MAAMJ,aAAK,CAACK,KAAK,CAACN,IAAI,EAAEO,SAAS,CAAC,CAAC,EAC9CX,cAAM,CAACY,KAAK,CACVtB,gBAAQ,CAACC,WAAW,CAAC,WAAW,CAAC,CAACC,IAAI,CACpCF,gBAAQ,CAACG,KAAK,CAACH,gBAAQ,CAACI,MAAM,CAAC,WAAW,CAAC,CAAC,CAC7C,CACF,CACF;EACD;EACA;EACA;EACA,OAAOa,QAAQ,CAACf,IAAI,CAACQ,cAAM,CAACa,QAAQ,EAAEX,cAAM,CAACY,UAAU,CAAC;EAExD;EACA;EACA;EACA;EACA,MAAMC,SAAS,GACbC,IAAc,IAQdd,cAAM,CAACC,GAAG,CAAC,aAAS;IAClB,MAAMc,SAAS,GAA0B,EAAE;IAC3C,IAAIC,MAAM,GAAGF,IAAI;IACjB;IACA,OAAO,IAAI,EAAE;MACX,MAAMG,MAAM,GAAG,OAAOrB,IAAI,CAACsB,KAAK,CAC9BF,MAAM,EACNG,eAAS,CAACC,IAAI,CAACxB,IAAI,CAACyB,WAAW,CAAC,CACjC;MACD,KAAK,MAAMC,KAAK,IAAIL,MAAM,EAAE;QAC1BF,SAAS,CAACQ,IAAI,CAACD,KAAK,CAAC;MACvB;MACA,MAAME,SAAS,GAAGP,MAAM,CAACQ,EAAE,CAAC,CAAC,CAAC,CAAC;MAC/B,IAAID,SAAS,KAAKf,SAAS,IAAIQ,MAAM,CAACS,MAAM,GAAG9B,IAAI,CAACyB,WAAW,EAAE;QAC/D;MACF;MACAL,MAAM,GAAGQ,SAAS,CAACG,QAAQ;IAC7B;IACA,MAAMC,IAAI,GAAGb,SAAS,CAACU,EAAE,CAAC,CAAC,CAAC,CAAC,EAAEE,QAAQ,IAAIb,IAAI;IAC/C,OAAO;MAAEG,MAAM,EAAEF,SAAS;MAAEa;IAAI,CAAE;EACpC,CAAC,CAAC,CAACtC,IAAI,CAACU,cAAM,CAACU,KAAK,CAACvB,mBAAmB,CAAC,CAAC;EAE5C;;;;;;;;;;;;EAYA,MAAM0C,SAAS,GACbf,IAAc,IAKdd,cAAM,CAACC,GAAG,CAAC,aAAS;IAClB,OAAO,IAAI,EAAE;MACX,OAAOE,aAAK,CAAC2B,OAAO,CAAC5B,IAAI,CAAC;MAC1B,MAAM;QAAEe,MAAM;QAAEW;MAAI,CAAE,GAAG,OAAOf,SAAS,CAACC,IAAI,CAAC;MAC/C,IAAIG,MAAM,CAACS,MAAM,GAAG,CAAC,EAAE;QACrB,OAAO,CAACK,aAAK,CAACC,YAAY,CAACf,MAAM,CAAC,EAAEW,IAAI,CAAU;MACpD;MACA;MACA;MACA,OAAO5B,cAAM,CAACiC,MAAM,CAClBjC,cAAM,CAACkC,OAAO,CAAC/B,aAAK,CAACgC,IAAI,CAACjC,IAAI,CAAC,EAAEN,IAAI,CAACwC,YAAY,CAAC,CACpD;IACH;EACF,CAAC,CAAC;EAEJ,OAAOtC,cAAM,CAACuC,iBAAiB,CAACxC,KAAK,IAAIyC,YAAM,EAAGxB,IAAI,IACpDd,cAAM,CAACuC,GAAG,CAACV,SAAS,CAACf,IAAI,CAAC,EAAE0B,cAAM,CAACC,IAAI,CAAC,CACzC;AACH,CAAC,CAAC,CACH;AAAAxD,OAAA,CAAAU,eAAA,GAAAA,eAAA","ignoreList":[]}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The subscribe machine for the Postgres engine, extracted from `store.ts`. It owns
|
|
3
|
-
* the wake queue, the listener drain, the paged catch-up reconcile, the `> boundary`
|
|
4
|
-
* de-dup (via the cursor's exclusive lower bound), and the poll loop — all behind
|
|
5
|
-
* two STUBBABLE seams (`fetch`, `listen`), so the wake/poll/dedup logic is
|
|
6
|
-
* UNIT-TESTABLE with no live database (see `test/subscribe.test.ts`). `store.ts`
|
|
7
|
-
* supplies production `fetch`/`listen` built from the pooled/direct clients.
|
|
8
|
-
*
|
|
9
|
-
* Correctness rests on the periodic position-based reconcile — a re-read from the
|
|
10
|
-
* cursor every poll interval — NOT on `NOTIFY`, which is only a latency
|
|
11
|
-
* optimisation. Exclusive-lock commit order equals sequence order, so an
|
|
12
|
-
* ascending re-read never skips a temporary gap; permanent bigserial gaps are
|
|
13
|
-
* simply absent positions and are fine to skip.
|
|
14
|
-
*
|
|
15
|
-
* ## `catchUpPageSize` bounds the STATEMENT, not the emission
|
|
16
|
-
*
|
|
17
|
-
* `drainFrom` below PAGES the catch-up read — `catchUpPageSize` bounds each
|
|
18
|
-
* `SELECT` — but it COLLECTS every page into one array, which is emitted as a
|
|
19
|
-
* SINGLE chunk. So peak memory at subscription start is O(backlog) rather than
|
|
20
|
-
* O(page): a subscriber joining behind a large log materialises the whole backlog
|
|
21
|
-
* before its consumer sees the first element.
|
|
22
|
-
*
|
|
23
|
-
* That is a memory characteristic and nothing more, which is why ONE chunk is
|
|
24
|
-
* enough here: downstream commit boundaries do not depend on the upstream chunking
|
|
25
|
-
* at all, because the read side's `Stream.groupedWithin` re-chunks the stream into
|
|
26
|
-
* bounded micro-batches whatever arrives (ADR-0007), so a projection's transactions
|
|
27
|
-
* stay small even against a one-chunk catch-up. The cost therefore lands on backlog
|
|
28
|
-
* memory alone, and it lands hardest on the durable-log-with-in-memory-view mode,
|
|
29
|
-
* where an ephemeral view re-catches-up the entire log on every boot.
|
|
30
|
-
*/
|
|
31
|
-
import type { SqlError } from '@effect/sql';
|
|
32
|
-
import type { Position, SequencedEvent } from '@kairos-es/core';
|
|
33
|
-
import { ReadLimit } from '@kairos-es/core';
|
|
34
|
-
import { Effect, type Scope, Stream } from 'effect';
|
|
35
|
-
/**
|
|
36
|
-
* A short catch-up page size default for the reconcile loop. Reads page by page
|
|
37
|
-
* until a short page signals the tail is drained, so a large backlog is streamed
|
|
38
|
-
* in bounded chunks rather than one huge SELECT. Overridable via
|
|
39
|
-
* `config.catchUpPageSize`.
|
|
40
|
-
*/
|
|
41
|
-
export declare const DEFAULT_CATCH_UP_PAGE = 512;
|
|
42
|
-
/**
|
|
43
|
-
* The default upper bound a live subscriber waits for a wake before re-reading
|
|
44
|
-
* from its cursor ANYWAY (milliseconds; overridable via `config.pollInterval`).
|
|
45
|
-
* This periodic position-based poll — not `NOTIFY` — is the correctness
|
|
46
|
-
* guarantee: every committed position is reconciled within this interval
|
|
47
|
-
* regardless of whether its `NOTIFY` was heard, which covers a `NOTIFY` fired
|
|
48
|
-
* before `LISTEN` was fully established, a coalesced/lost `NOTIFY`, and a listen
|
|
49
|
-
* connection that dropped SILENTLY (the `@effect/sql-pg` listen client swallows
|
|
50
|
-
* connection errors and does not auto-reconnect). `NOTIFY` only lowers delivery
|
|
51
|
-
* latency below this interval. Upstream `postgres_tt.py` uses the same 1-second
|
|
52
|
-
* fallback poll (`conn.wait(..., interval=1)`).
|
|
53
|
-
*/
|
|
54
|
-
export declare const DEFAULT_POLL_INTERVAL_MILLIS = 1000;
|
|
55
|
-
/**
|
|
56
|
-
* The two seams the subscribe machine runs over, plus the two tuning knobs. In
|
|
57
|
-
* production `fetch` is a compiled read plan run against the pooled client and
|
|
58
|
-
* mapped to `SequencedEvent`s, and `listen` is `PgClientDirect.listen(channel)`;
|
|
59
|
-
* a unit test stubs both with plain in-memory values.
|
|
60
|
-
*/
|
|
61
|
-
export interface SubscriptionDeps {
|
|
62
|
-
/**
|
|
63
|
-
* Page-read the events strictly after `from`, up to `limit`, ascending by
|
|
64
|
-
* position. Returning FEWER than `limit` signals the tail is drained.
|
|
65
|
-
*/
|
|
66
|
-
readonly fetch: (from: Position, limit: ReadLimit) => Effect.Effect<ReadonlyArray<SequencedEvent>, SqlError.SqlError>;
|
|
67
|
-
/**
|
|
68
|
-
* The raw wake source: each element is an opaque wake signal (a `NOTIFY`). The
|
|
69
|
-
* machine taps it into its own wake queue and drains it on a background fibre;
|
|
70
|
-
* it never reads the elements. `Stream.never` is a valid poll-only source.
|
|
71
|
-
*/
|
|
72
|
-
readonly listen: Stream.Stream<unknown, SqlError.SqlError>;
|
|
73
|
-
/** The poll interval in milliseconds (see `DEFAULT_POLL_INTERVAL_MILLIS`). */
|
|
74
|
-
readonly pollInterval: number;
|
|
75
|
-
/** The catch-up page size (see `DEFAULT_CATCH_UP_PAGE`). */
|
|
76
|
-
readonly catchUpPage: number;
|
|
77
|
-
}
|
|
78
|
-
/**
|
|
79
|
-
* Build the subscription `Stream`: catch up from `after`, then track the live
|
|
80
|
-
* tail. The cursor threads through the `unfoldChunkEffect` unfold state, starting
|
|
81
|
-
* at `after` and advancing to the last delivered position each batch; the stream
|
|
82
|
-
* never completes on its own (always `Option.some`), so the enclosing scope
|
|
83
|
-
* interrupts it on close. A `SqlError` mid-stream stays on the channel;
|
|
84
|
-
* `makeContractStore` dies it (subscribe is `E = never` on the contract).
|
|
85
|
-
*/
|
|
86
|
-
export declare const subscribeStream: (deps: SubscriptionDeps, after: Position | undefined) => Stream.Stream<SequencedEvent, SqlError.SqlError, Scope.Scope>;
|
|
87
|
-
//# sourceMappingURL=subscribe.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"subscribe.d.ts","sourceRoot":"","sources":["../../../src/internal/subscribe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC/D,OAAO,EAAU,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAEL,MAAM,EAIN,KAAK,KAAK,EACV,MAAM,EACP,MAAM,QAAQ,CAAA;AAEf;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAA;AAExC;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,4BAA4B,OAAO,CAAA;AAchD;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,CACd,IAAI,EAAE,QAAQ,EACd,KAAK,EAAE,SAAS,KACb,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACpE;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAC1D,8EAA8E;IAC9E,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,4DAA4D;IAC5D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,GAC1B,MAAM,gBAAgB,EACtB,OAAO,QAAQ,GAAG,SAAS,KAC1B,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAuG5D,CAAA"}
|
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
import { ORIGIN, ReadLimit } from '@kairos-es/core';
|
|
2
|
-
import { Chunk, Effect, Option, Queue, Schedule, Stream } from 'effect';
|
|
3
|
-
/**
|
|
4
|
-
* A short catch-up page size default for the reconcile loop. Reads page by page
|
|
5
|
-
* until a short page signals the tail is drained, so a large backlog is streamed
|
|
6
|
-
* in bounded chunks rather than one huge SELECT. Overridable via
|
|
7
|
-
* `config.catchUpPageSize`.
|
|
8
|
-
*/
|
|
9
|
-
export const DEFAULT_CATCH_UP_PAGE = 512;
|
|
10
|
-
/**
|
|
11
|
-
* The default upper bound a live subscriber waits for a wake before re-reading
|
|
12
|
-
* from its cursor ANYWAY (milliseconds; overridable via `config.pollInterval`).
|
|
13
|
-
* This periodic position-based poll — not `NOTIFY` — is the correctness
|
|
14
|
-
* guarantee: every committed position is reconciled within this interval
|
|
15
|
-
* regardless of whether its `NOTIFY` was heard, which covers a `NOTIFY` fired
|
|
16
|
-
* before `LISTEN` was fully established, a coalesced/lost `NOTIFY`, and a listen
|
|
17
|
-
* connection that dropped SILENTLY (the `@effect/sql-pg` listen client swallows
|
|
18
|
-
* connection errors and does not auto-reconnect). `NOTIFY` only lowers delivery
|
|
19
|
-
* latency below this interval. Upstream `postgres_tt.py` uses the same 1-second
|
|
20
|
-
* fallback poll (`conn.wait(..., interval=1)`).
|
|
21
|
-
*/
|
|
22
|
-
export const DEFAULT_POLL_INTERVAL_MILLIS = 1000;
|
|
23
|
-
/**
|
|
24
|
-
* Retry a TRANSIENT poll fault (a `SqlError` from one reconcile read) on a capped
|
|
25
|
-
* exponential backoff rather than letting `Stream.orDie` kill a subscription that
|
|
26
|
-
* may have been alive for days over a single connection blip. A fault that
|
|
27
|
-
* PERSISTS past the cap propagates (the decorator dies it) — infrastructure stays
|
|
28
|
-
* off the contract channel while the stream survives ordinary transients.
|
|
29
|
-
*/
|
|
30
|
-
const POLL_RETRY_SCHEDULE = /*#__PURE__*/Schedule.exponential('100 millis', 2).pipe(/*#__PURE__*/Schedule.union(/*#__PURE__*/Schedule.spaced('2 seconds')), /*#__PURE__*/Schedule.intersect(/*#__PURE__*/Schedule.recurs(20)));
|
|
31
|
-
/**
|
|
32
|
-
* Build the subscription `Stream`: catch up from `after`, then track the live
|
|
33
|
-
* tail. The cursor threads through the `unfoldChunkEffect` unfold state, starting
|
|
34
|
-
* at `after` and advancing to the last delivered position each batch; the stream
|
|
35
|
-
* never completes on its own (always `Option.some`), so the enclosing scope
|
|
36
|
-
* interrupts it on close. A `SqlError` mid-stream stays on the channel;
|
|
37
|
-
* `makeContractStore` dies it (subscribe is `E = never` on the contract).
|
|
38
|
-
*/
|
|
39
|
-
export const subscribeStream = (deps, after) => Stream.unwrapScoped(Effect.gen(function* () {
|
|
40
|
-
// The wake primitive: a sliding(1) queue holding at most one pending
|
|
41
|
-
// signal. The listener `offer`s an opaque wake on every NOTIFY; the poll
|
|
42
|
-
// loop `take`s (with a timeout). The lost-wakeup guard is STRUCTURAL —
|
|
43
|
-
// drain the queue BEFORE reading, so a NOTIFY that lands mid-read leaves
|
|
44
|
-
// a fresh signal that makes the next `take` return at once rather than
|
|
45
|
-
// sleeping through a real change.
|
|
46
|
-
const wake = yield* Queue.sliding(1);
|
|
47
|
-
// Tap every notification into the wake queue. If the listen stream DOES
|
|
48
|
-
// surface an error, re-subscribe with capped-exponential backoff. In
|
|
49
|
-
// practice the @effect/sql-pg listen client usually swallows a dropped
|
|
50
|
-
// connection SILENTLY rather than erroring, so this retry rarely fires —
|
|
51
|
-
// the poll fallback below, not this retry, is the real reconnect guarantee:
|
|
52
|
-
// the reconcile loop re-reads from the cursor on every poll, so a silently-
|
|
53
|
-
// dropped (and later re-established) listener loses no positions.
|
|
54
|
-
const listener = deps.listen.pipe(Stream.tap(() => Queue.offer(wake, undefined)), Stream.retry(Schedule.exponential('50 millis').pipe(Schedule.union(Schedule.spaced('1 seconds')))));
|
|
55
|
-
// Drain the listener into the wake queue on a background fibre; the scope
|
|
56
|
-
// releases it. We never read its elements directly — every notification is
|
|
57
|
-
// an opaque wake signal.
|
|
58
|
-
yield* listener.pipe(Stream.runDrain, Effect.forkScoped);
|
|
59
|
-
// Reconcile: page-read from `from` until a short page, advancing to the last
|
|
60
|
-
// delivered position. Retries a TRANSIENT poll fault on a capped schedule so
|
|
61
|
-
// a single connection blip does not kill a days-long subscription; a
|
|
62
|
-
// persistent fault propagates and dies.
|
|
63
|
-
const drainFrom = from => Effect.gen(function* () {
|
|
64
|
-
const collected = [];
|
|
65
|
-
let cursor = from;
|
|
66
|
-
// Loop pages until a short page signals the tail is drained.
|
|
67
|
-
while (true) {
|
|
68
|
-
const events = yield* deps.fetch(cursor, ReadLimit.make(deps.catchUpPage));
|
|
69
|
-
for (const event of events) {
|
|
70
|
-
collected.push(event);
|
|
71
|
-
}
|
|
72
|
-
const lastEvent = events.at(-1);
|
|
73
|
-
if (lastEvent === undefined || events.length < deps.catchUpPage) {
|
|
74
|
-
break;
|
|
75
|
-
}
|
|
76
|
-
cursor = lastEvent.position;
|
|
77
|
-
}
|
|
78
|
-
const last = collected.at(-1)?.position ?? from;
|
|
79
|
-
return {
|
|
80
|
-
events: collected,
|
|
81
|
-
last
|
|
82
|
-
};
|
|
83
|
-
}).pipe(Effect.retry(POLL_RETRY_SCHEDULE));
|
|
84
|
-
/**
|
|
85
|
-
* Produce the NEXT non-empty batch from a cursor position, together with
|
|
86
|
-
* the advanced cursor — a pure `Position → Effect<[Chunk, next]>`. The
|
|
87
|
-
* `> boundary` de-dup is enforced by `fetch`'s exclusive lower bound, so a
|
|
88
|
-
* live re-read after a wake never re-emits an already-delivered position.
|
|
89
|
-
*
|
|
90
|
-
* Drain the wake queue BEFORE reading (lost-wakeup guard), then drain from
|
|
91
|
-
* the cursor. If a drain yields nothing, wait for a wake but only up to the
|
|
92
|
-
* poll interval, then loop and re-read ANYWAY — the periodic position-based
|
|
93
|
-
* poll that makes delivery correct independently of NOTIFY. This effect only
|
|
94
|
-
* completes by returning a non-empty chunk; the scope interrupts it on close.
|
|
95
|
-
*/
|
|
96
|
-
const nextBatch = from => Effect.gen(function* () {
|
|
97
|
-
while (true) {
|
|
98
|
-
yield* Queue.takeAll(wake);
|
|
99
|
-
const {
|
|
100
|
-
events,
|
|
101
|
-
last
|
|
102
|
-
} = yield* drainFrom(from);
|
|
103
|
-
if (events.length > 0) {
|
|
104
|
-
return [Chunk.fromIterable(events), last];
|
|
105
|
-
}
|
|
106
|
-
// Nothing new: block on a wake, but only up to the poll interval — on
|
|
107
|
-
// timeout we fall through and the loop re-reads regardless.
|
|
108
|
-
yield* Effect.ignore(Effect.timeout(Queue.take(wake), deps.pollInterval));
|
|
109
|
-
}
|
|
110
|
-
});
|
|
111
|
-
return Stream.unfoldChunkEffect(after ?? ORIGIN, from => Effect.map(nextBatch(from), Option.some));
|
|
112
|
-
}));
|
|
113
|
-
//# sourceMappingURL=subscribe.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"subscribe.js","names":["ORIGIN","ReadLimit","Chunk","Effect","Option","Queue","Schedule","Stream","DEFAULT_CATCH_UP_PAGE","DEFAULT_POLL_INTERVAL_MILLIS","POLL_RETRY_SCHEDULE","exponential","pipe","union","spaced","intersect","recurs","subscribeStream","deps","after","unwrapScoped","gen","wake","sliding","listener","listen","tap","offer","undefined","retry","runDrain","forkScoped","drainFrom","from","collected","cursor","events","fetch","make","catchUpPage","event","push","lastEvent","at","length","position","last","nextBatch","takeAll","fromIterable","ignore","timeout","take","pollInterval","unfoldChunkEffect","map","some"],"sources":["../../../src/internal/subscribe.ts"],"sourcesContent":[null],"mappings":"AAgCA,SAASA,MAAM,EAAEC,SAAS,QAAQ,iBAAiB;AACnD,SACEC,KAAK,EACLC,MAAM,EACNC,MAAM,EACNC,KAAK,EACLC,QAAQ,EAERC,MAAM,QACD,QAAQ;AAEf;;;;;;AAMA,OAAO,MAAMC,qBAAqB,GAAG,GAAG;AAExC;;;;;;;;;;;;AAYA,OAAO,MAAMC,4BAA4B,GAAG,IAAI;AAEhD;;;;;;;AAOA,MAAMC,mBAAmB,gBAAGJ,QAAQ,CAACK,WAAW,CAAC,YAAY,EAAE,CAAC,CAAC,CAACC,IAAI,cACpEN,QAAQ,CAACO,KAAK,cAACP,QAAQ,CAACQ,MAAM,CAAC,WAAW,CAAC,CAAC,eAC5CR,QAAQ,CAACS,SAAS,cAACT,QAAQ,CAACU,MAAM,CAAC,EAAE,CAAC,CAAC,CACxC;AA6BD;;;;;;;;AAQA,OAAO,MAAMC,eAAe,GAAGA,CAC7BC,IAAsB,EACtBC,KAA2B,KAE3BZ,MAAM,CAACa,YAAY,CACjBjB,MAAM,CAACkB,GAAG,CAAC,aAAS;EAClB;EACA;EACA;EACA;EACA;EACA;EACA,MAAMC,IAAI,GAAG,OAAOjB,KAAK,CAACkB,OAAO,CAAO,CAAC,CAAC;EAE1C;EACA;EACA;EACA;EACA;EACA;EACA;EACA,MAAMC,QAAQ,GAAGN,IAAI,CAACO,MAAM,CAACb,IAAI,CAC/BL,MAAM,CAACmB,GAAG,CAAC,MAAMrB,KAAK,CAACsB,KAAK,CAACL,IAAI,EAAEM,SAAS,CAAC,CAAC,EAC9CrB,MAAM,CAACsB,KAAK,CACVvB,QAAQ,CAACK,WAAW,CAAC,WAAW,CAAC,CAACC,IAAI,CACpCN,QAAQ,CAACO,KAAK,CAACP,QAAQ,CAACQ,MAAM,CAAC,WAAW,CAAC,CAAC,CAC7C,CACF,CACF;EACD;EACA;EACA;EACA,OAAOU,QAAQ,CAACZ,IAAI,CAACL,MAAM,CAACuB,QAAQ,EAAE3B,MAAM,CAAC4B,UAAU,CAAC;EAExD;EACA;EACA;EACA;EACA,MAAMC,SAAS,GACbC,IAAc,IAQd9B,MAAM,CAACkB,GAAG,CAAC,aAAS;IAClB,MAAMa,SAAS,GAA0B,EAAE;IAC3C,IAAIC,MAAM,GAAGF,IAAI;IACjB;IACA,OAAO,IAAI,EAAE;MACX,MAAMG,MAAM,GAAG,OAAOlB,IAAI,CAACmB,KAAK,CAC9BF,MAAM,EACNlC,SAAS,CAACqC,IAAI,CAACpB,IAAI,CAACqB,WAAW,CAAC,CACjC;MACD,KAAK,MAAMC,KAAK,IAAIJ,MAAM,EAAE;QAC1BF,SAAS,CAACO,IAAI,CAACD,KAAK,CAAC;MACvB;MACA,MAAME,SAAS,GAAGN,MAAM,CAACO,EAAE,CAAC,CAAC,CAAC,CAAC;MAC/B,IAAID,SAAS,KAAKd,SAAS,IAAIQ,MAAM,CAACQ,MAAM,GAAG1B,IAAI,CAACqB,WAAW,EAAE;QAC/D;MACF;MACAJ,MAAM,GAAGO,SAAS,CAACG,QAAQ;IAC7B;IACA,MAAMC,IAAI,GAAGZ,SAAS,CAACS,EAAE,CAAC,CAAC,CAAC,CAAC,EAAEE,QAAQ,IAAIZ,IAAI;IAC/C,OAAO;MAAEG,MAAM,EAAEF,SAAS;MAAEY;IAAI,CAAE;EACpC,CAAC,CAAC,CAAClC,IAAI,CAACT,MAAM,CAAC0B,KAAK,CAACnB,mBAAmB,CAAC,CAAC;EAE5C;;;;;;;;;;;;EAYA,MAAMqC,SAAS,GACbd,IAAc,IAKd9B,MAAM,CAACkB,GAAG,CAAC,aAAS;IAClB,OAAO,IAAI,EAAE;MACX,OAAOhB,KAAK,CAAC2C,OAAO,CAAC1B,IAAI,CAAC;MAC1B,MAAM;QAAEc,MAAM;QAAEU;MAAI,CAAE,GAAG,OAAOd,SAAS,CAACC,IAAI,CAAC;MAC/C,IAAIG,MAAM,CAACQ,MAAM,GAAG,CAAC,EAAE;QACrB,OAAO,CAAC1C,KAAK,CAAC+C,YAAY,CAACb,MAAM,CAAC,EAAEU,IAAI,CAAU;MACpD;MACA;MACA;MACA,OAAO3C,MAAM,CAAC+C,MAAM,CAClB/C,MAAM,CAACgD,OAAO,CAAC9C,KAAK,CAAC+C,IAAI,CAAC9B,IAAI,CAAC,EAAEJ,IAAI,CAACmC,YAAY,CAAC,CACpD;IACH;EACF,CAAC,CAAC;EAEJ,OAAO9C,MAAM,CAAC+C,iBAAiB,CAACnC,KAAK,IAAInB,MAAM,EAAGiC,IAAI,IACpD9B,MAAM,CAACoD,GAAG,CAACR,SAAS,CAACd,IAAI,CAAC,EAAE7B,MAAM,CAACoD,IAAI,CAAC,CACzC;AACH,CAAC,CAAC,CACH","ignoreList":[]}
|