@kairos-es/feed 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +109 -2
- package/dist/cjs/Feed.js +6 -0
- package/dist/cjs/Feed.js.map +1 -0
- package/dist/cjs/FeedSink.js +54 -0
- package/dist/cjs/FeedSink.js.map +1 -0
- package/dist/cjs/FeedSyncStore.js +10 -0
- package/dist/cjs/FeedSyncStore.js.map +1 -0
- package/dist/cjs/deliver.js +86 -0
- package/dist/cjs/deliver.js.map +1 -0
- package/dist/cjs/errors.js +114 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/identifiers.js +63 -0
- package/dist/cjs/identifiers.js.map +1 -0
- package/dist/cjs/inMemoryFeed.js +328 -0
- package/dist/cjs/inMemoryFeed.js.map +1 -0
- package/dist/cjs/inMemorySink.js +142 -0
- package/dist/cjs/inMemorySink.js.map +1 -0
- package/dist/cjs/index.js +184 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/inspect.js +68 -0
- package/dist/cjs/inspect.js.map +1 -0
- package/dist/cjs/readiness.js +100 -0
- package/dist/cjs/readiness.js.map +1 -0
- package/dist/cjs/reconcile.js +164 -0
- package/dist/cjs/reconcile.js.map +1 -0
- package/dist/cjs/runFeedSync.js +603 -0
- package/dist/cjs/runFeedSync.js.map +1 -0
- package/dist/cjs/tuning.js +60 -0
- package/dist/cjs/tuning.js.map +1 -0
- package/dist/dts-cjs/Feed.d.ts +99 -0
- package/dist/dts-cjs/Feed.d.ts.map +1 -0
- package/dist/dts-cjs/FeedSink.d.ts +189 -0
- package/dist/dts-cjs/FeedSink.d.ts.map +1 -0
- package/dist/dts-cjs/FeedSyncStore.d.ts +281 -0
- package/dist/dts-cjs/FeedSyncStore.d.ts.map +1 -0
- package/dist/dts-cjs/deliver.d.ts +16 -0
- package/dist/dts-cjs/deliver.d.ts.map +1 -0
- package/dist/dts-cjs/errors.d.ts +131 -0
- package/dist/dts-cjs/errors.d.ts.map +1 -0
- package/dist/dts-cjs/identifiers.d.ts +67 -0
- package/dist/dts-cjs/identifiers.d.ts.map +1 -0
- package/dist/dts-cjs/inMemoryFeed.d.ts +32 -0
- package/dist/dts-cjs/inMemoryFeed.d.ts.map +1 -0
- package/dist/dts-cjs/inMemorySink.d.ts +61 -0
- package/dist/dts-cjs/inMemorySink.d.ts.map +1 -0
- package/dist/dts-cjs/index.d.ts +63 -0
- package/dist/dts-cjs/index.d.ts.map +1 -0
- package/dist/dts-cjs/inspect.d.ts +71 -0
- package/dist/dts-cjs/inspect.d.ts.map +1 -0
- package/dist/dts-cjs/package.json +3 -0
- package/dist/dts-cjs/readiness.d.ts +84 -0
- package/dist/dts-cjs/readiness.d.ts.map +1 -0
- package/dist/dts-cjs/reconcile.d.ts +68 -0
- package/dist/dts-cjs/reconcile.d.ts.map +1 -0
- package/dist/dts-cjs/runFeedSync.d.ts +220 -0
- package/dist/dts-cjs/runFeedSync.d.ts.map +1 -0
- package/dist/dts-cjs/tuning.d.ts +40 -0
- package/dist/dts-cjs/tuning.d.ts.map +1 -0
- package/dist/dts-esm/Feed.d.ts +99 -0
- package/dist/dts-esm/Feed.d.ts.map +1 -0
- package/dist/dts-esm/FeedSink.d.ts +189 -0
- package/dist/dts-esm/FeedSink.d.ts.map +1 -0
- package/dist/dts-esm/FeedSyncStore.d.ts +281 -0
- package/dist/dts-esm/FeedSyncStore.d.ts.map +1 -0
- package/dist/dts-esm/deliver.d.ts +16 -0
- package/dist/dts-esm/deliver.d.ts.map +1 -0
- package/dist/dts-esm/errors.d.ts +131 -0
- package/dist/dts-esm/errors.d.ts.map +1 -0
- package/dist/dts-esm/identifiers.d.ts +67 -0
- package/dist/dts-esm/identifiers.d.ts.map +1 -0
- package/dist/dts-esm/inMemoryFeed.d.ts +32 -0
- package/dist/dts-esm/inMemoryFeed.d.ts.map +1 -0
- package/dist/dts-esm/inMemorySink.d.ts +61 -0
- package/dist/dts-esm/inMemorySink.d.ts.map +1 -0
- package/dist/dts-esm/index.d.ts +63 -0
- package/dist/dts-esm/index.d.ts.map +1 -0
- package/dist/dts-esm/inspect.d.ts +71 -0
- package/dist/dts-esm/inspect.d.ts.map +1 -0
- package/dist/dts-esm/package.json +3 -0
- package/dist/dts-esm/readiness.d.ts +84 -0
- package/dist/dts-esm/readiness.d.ts.map +1 -0
- package/dist/dts-esm/reconcile.d.ts +68 -0
- package/dist/dts-esm/reconcile.d.ts.map +1 -0
- package/dist/dts-esm/runFeedSync.d.ts +220 -0
- package/dist/dts-esm/runFeedSync.d.ts.map +1 -0
- package/dist/dts-esm/tuning.d.ts +40 -0
- package/dist/dts-esm/tuning.d.ts.map +1 -0
- package/dist/esm/Feed.js +2 -0
- package/dist/esm/Feed.js.map +1 -0
- package/dist/esm/FeedSink.js +46 -0
- package/dist/esm/FeedSink.js.map +1 -0
- package/dist/esm/FeedSyncStore.js +3 -0
- package/dist/esm/FeedSyncStore.js.map +1 -0
- package/dist/esm/deliver.js +78 -0
- package/dist/esm/deliver.js.map +1 -0
- package/dist/esm/errors.js +98 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/identifiers.js +56 -0
- package/dist/esm/identifiers.js.map +1 -0
- package/dist/esm/inMemoryFeed.js +320 -0
- package/dist/esm/inMemoryFeed.js.map +1 -0
- package/dist/esm/inMemorySink.js +134 -0
- package/dist/esm/inMemorySink.js.map +1 -0
- package/dist/esm/index.js +62 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/inspect.js +58 -0
- package/dist/esm/inspect.js.map +1 -0
- package/dist/esm/package.json +4 -0
- package/dist/esm/readiness.js +92 -0
- package/dist/esm/readiness.js.map +1 -0
- package/dist/esm/reconcile.js +153 -0
- package/dist/esm/reconcile.js.map +1 -0
- package/dist/esm/runFeedSync.js +595 -0
- package/dist/esm/runFeedSync.js.map +1 -0
- package/dist/esm/tuning.js +51 -0
- package/dist/esm/tuning.js.map +1 -0
- package/package.json +34 -3
- package/src/Feed.ts +111 -0
- package/src/FeedSink.ts +130 -0
- package/src/FeedSyncStore.ts +326 -0
- package/src/deliver.ts +108 -0
- package/src/errors.ts +134 -0
- package/src/identifiers.ts +92 -0
- package/src/inMemoryFeed.ts +582 -0
- package/src/inMemorySink.ts +215 -0
- package/src/index.ts +136 -0
- package/src/inspect.ts +114 -0
- package/src/readiness.ts +137 -0
- package/src/reconcile.ts +224 -0
- package/src/runFeedSync.ts +882 -0
- package/src/tuning.ts +65 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reconcile: delete what a sink holds that the feed no longer has.
|
|
3
|
+
*
|
|
4
|
+
* ## Why a feed needs it at all
|
|
5
|
+
*
|
|
6
|
+
* The steady state deletes through tombstones, and a tombstone lives only until
|
|
7
|
+
* every sink registered on the feed has passed it. A record can still be left
|
|
8
|
+
* behind in a sink: a sink restored from a backup, a key written into the index by
|
|
9
|
+
* hand, or a record deleted before a reset, whose tombstone the reset cleared with
|
|
10
|
+
* every other row, followed by an in-place resync that, having no atomic swap,
|
|
11
|
+
* sends only the rows the new generation holds and so never deletes it. Reconcile is the one operation that compares the sink with the feed
|
|
12
|
+
* rather than replaying the feed into the sink, and so the only one that finds
|
|
13
|
+
* those orphans.
|
|
14
|
+
*
|
|
15
|
+
* ## The order of the two reads, and why it is safe
|
|
16
|
+
*
|
|
17
|
+
* The sink's keys are listed FIRST and the feed's live keys read SECOND. A key the
|
|
18
|
+
* sink lists that has a live row by the time the feed is read is kept, so a record
|
|
19
|
+
* written while reconcile runs is never deleted by it. A key whose row turns into
|
|
20
|
+
* a tombstone after the feed is read is kept too, and its tombstone deletes it on
|
|
21
|
+
* the next pass. And a key whose row comes back to life after the feed is read is
|
|
22
|
+
* deleted here and re-sent by the next pass, because its new `seq` is above the
|
|
23
|
+
* cursor. Reconcile runs inside the worker that holds the lease, between batches,
|
|
24
|
+
* so nothing else is sending to the sink while it does (ADR-0019 §8).
|
|
25
|
+
*
|
|
26
|
+
* ## What it costs
|
|
27
|
+
*
|
|
28
|
+
* It holds every key the sink lists and every live key of the feed in memory at
|
|
29
|
+
* once, and reads every live document to find the keys. It is a periodic repair,
|
|
30
|
+
* not a steady-state path: `reconcileEvery` should be measured in hours.
|
|
31
|
+
*/
|
|
32
|
+
import { Effect } from 'effect';
|
|
33
|
+
import type { FeedUnreadable, FeedWriteFailed, LeaseHeld, LeaseLost, SinkHalted, SinkRetryable } from './errors.js';
|
|
34
|
+
import type { FeedSource } from './Feed.js';
|
|
35
|
+
import { type FeedReconcile, type FeedSink } from './FeedSink.js';
|
|
36
|
+
import type { FeedSyncStore } from './FeedSyncStore.js';
|
|
37
|
+
import { type FeedKey } from './identifiers.js';
|
|
38
|
+
/** What one reconcile did. */
|
|
39
|
+
export interface ReconcileReport {
|
|
40
|
+
/** How many keys the sink listed. */
|
|
41
|
+
readonly listed: number;
|
|
42
|
+
/** The orphans deleted: listed by the sink, with no live row in the feed. */
|
|
43
|
+
readonly deleted: ReadonlyArray<FeedKey>;
|
|
44
|
+
/** Orphans whose delete the sink refused, with its reason. They stay in the sink. */
|
|
45
|
+
readonly refused: ReadonlyArray<{
|
|
46
|
+
readonly key: FeedKey;
|
|
47
|
+
readonly reason: string;
|
|
48
|
+
}>;
|
|
49
|
+
}
|
|
50
|
+
/** Every key of `source` whose row is live, read page by page from the origin. */
|
|
51
|
+
export declare const liveKeys: <D>(source: FeedSource<D>, pageSize: number) => Effect.Effect<ReadonlySet<FeedKey>, FeedUnreadable>;
|
|
52
|
+
/** Options for {@link reconcileFeed}. */
|
|
53
|
+
export interface ReconcileFeedOptions<D, Receipt> {
|
|
54
|
+
readonly source: FeedSource<D>;
|
|
55
|
+
readonly sink: FeedSink<D, Receipt> & {
|
|
56
|
+
readonly reconcile: FeedReconcile;
|
|
57
|
+
};
|
|
58
|
+
readonly store: FeedSyncStore;
|
|
59
|
+
/** How many rows to read, and deletes to send, at a time. */
|
|
60
|
+
readonly pageSize?: number;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Reconcile `sink` against `source` once, now. It takes the worker lease itself,
|
|
64
|
+
* so it fails `LeaseHeld` while a sync worker for the same feed and sink runs —
|
|
65
|
+
* reconcile through that worker's `reconcileEvery` instead, or stop it first.
|
|
66
|
+
*/
|
|
67
|
+
export declare const reconcileFeed: <D, Receipt>(options: ReconcileFeedOptions<D, Receipt>) => Effect.Effect<ReconcileReport, LeaseHeld | LeaseLost | FeedUnreadable | FeedWriteFailed | SinkRetryable | SinkHalted>;
|
|
68
|
+
//# sourceMappingURL=reconcile.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../../src/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,OAAO,EAAgB,MAAM,EAAkB,MAAM,QAAQ,CAAA;AAG7D,OAAO,KAAK,EACV,cAAc,EACd,eAAe,EACf,SAAS,EACT,SAAS,EACT,UAAU,EACV,aAAa,EACd,MAAM,UAAU,CAAA;AACjB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,EAAc,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,CAAA;AAC1E,OAAO,KAAK,EAAa,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/D,OAAO,EAAe,KAAK,OAAO,EAAmB,MAAM,eAAe,CAAA;AAG1E,8BAA8B;AAC9B,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;IACxC,qFAAqF;IACrF,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;QACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KACxB,CAAC,CAAA;CACH;AAED,kFAAkF;AAClF,eAAO,MAAM,QAAQ,GAAI,CAAC,EACxB,QAAQ,UAAU,CAAC,CAAC,CAAC,EACrB,UAAU,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,cAAc,CAoBjD,CAAA;AAmGJ,yCAAyC;AACzC,MAAM,WAAW,oBAAoB,CAAC,CAAC,EAAE,OAAO;IAC9C,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG;QAAE,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAA;KAAE,CAAA;IAC3E,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,GAAI,CAAC,EAAE,OAAO,EACtC,SAAS,oBAAoB,CAAC,CAAC,EAAE,OAAO,CAAC,KACxC,MAAM,CAAC,MAAM,CACd,eAAe,EACb,SAAS,GACT,SAAS,GACT,cAAc,GACd,eAAe,GACf,aAAa,GACb,UAAU,CAgBX,CAAA"}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `runFeedSync` — the sync worker: one per feed and sink, holding the lease,
|
|
3
|
+
* copying the feed's rows to the sink and moving its cursor only once the sink
|
|
4
|
+
* confirms them (ADR-0019).
|
|
5
|
+
*
|
|
6
|
+
* ## The lifecycle
|
|
7
|
+
*
|
|
8
|
+
* A worker runs in SESSIONS. A session takes the lease, then loops:
|
|
9
|
+
*
|
|
10
|
+
* 1. Read the cursor and the feed's generation. No cursor, or a cursor at another
|
|
11
|
+
* generation, means the sink holds a copy of something the feed no longer is,
|
|
12
|
+
* so the session RESYNCS (below) and loops.
|
|
13
|
+
* 2. Otherwise it TAILS: the rows after the cursor, through core's generic
|
|
14
|
+
* subscribe machine with `emit: 'page'`, so each page is sent as it is read and
|
|
15
|
+
* the worker holds one page in memory, never the backlog. Each page is checked
|
|
16
|
+
* against the generation first, and so is every poll interval on its own, since
|
|
17
|
+
* a reset that writes nothing yields no page; a generation that moved since the
|
|
18
|
+
* cursor was saved ends the tail, and step 1 sees the mismatch.
|
|
19
|
+
*
|
|
20
|
+
* Every page goes through one delivery: send, park the keys the sink refuses with
|
|
21
|
+
* its reasons, send the rest again, confirm. Then ONE compare-and-set moves the
|
|
22
|
+
* cursor past the whole page and moves the parked list with it — the refused keys
|
|
23
|
+
* parked, every key that was taken unparked — and fenced on the lease holder, so a
|
|
24
|
+
* worker that lost its lease moves nothing. Then tombstones every sink has passed
|
|
25
|
+
* are collected.
|
|
26
|
+
*
|
|
27
|
+
* The rules that sequence keeps, each a consequence of a sink not being
|
|
28
|
+
* transactional:
|
|
29
|
+
*
|
|
30
|
+
* - **The cursor moves only after `confirm`.** A crash between `apply` and
|
|
31
|
+
* `confirm` resends the batch; a sink is idempotent, so that costs a resend and
|
|
32
|
+
* nothing else.
|
|
33
|
+
* - **A refused key does not hold the feed up.** It is parked and the cursor moves
|
|
34
|
+
* past it. It is retried when its row next changes, because a change gives the
|
|
35
|
+
* row a `seq` above the cursor, and it leaves the parked list once it is sent
|
|
36
|
+
* successfully — a tombstone for a parked key included, which is how a refused
|
|
37
|
+
* record that is then deleted stops being parked without anyone acting.
|
|
38
|
+
* - **A retryable failure keeps the cursor.** The session ends, and supervision
|
|
39
|
+
* (below) starts a new one after a back-off no shorter than the sink's
|
|
40
|
+
* `retryAfter`; the new session sends the same page again.
|
|
41
|
+
*
|
|
42
|
+
* ## Resync
|
|
43
|
+
*
|
|
44
|
+
* Every row, page by page from the origin, goes into `resync.begin`'s fresh
|
|
45
|
+
* target; then the generation is checked once more, the target is published over
|
|
46
|
+
* the live copy, and the cursor is saved at the new generation with the parked list
|
|
47
|
+
* REPLACED by the refusals the resync met. Resync progress is held in memory only,
|
|
48
|
+
* so a crash restarts it from the origin, with a new `begin` that drops the
|
|
49
|
+
* half-built target the crash left.
|
|
50
|
+
*
|
|
51
|
+
* ### Waiting for the materialisation
|
|
52
|
+
*
|
|
53
|
+
* The feed's end is not the rebuild's end. After a reset the materialisation
|
|
54
|
+
* replays the log and writes the feed as it goes, so a resync that reaches the
|
|
55
|
+
* feed's last row has reached wherever the replay had got to, and publishing there
|
|
56
|
+
* swaps a complete live copy for a partial one. `readiness.ts` owns that argument.
|
|
57
|
+
* So `runFeedSync` takes an optional `readiness`: asked once as each attempt
|
|
58
|
+
* starts, it hands back a probe, and the attempt does not publish until the probe
|
|
59
|
+
* says the materialisation has caught up. Until then the attempt polls: once per
|
|
60
|
+
* poll interval it checks the generation, abandoning the attempt as stale if a
|
|
61
|
+
* reset moved it again, asks the probe, and reads the feed on from where it left
|
|
62
|
+
* off, loading what the replay wrote meanwhile. The probe is asked BEFORE that
|
|
63
|
+
* read, so the read that follows a `true` sees every row the materialisation had
|
|
64
|
+
* committed by the time it answered, since a feed row commits with the checkpoint
|
|
65
|
+
* that produced it. The lease is still checked immediately before `publish`, and
|
|
66
|
+
* the generation once more just before that, however long the wait was.
|
|
67
|
+
*
|
|
68
|
+
* With no `readiness` the attempt publishes at the feed's end, which is exactly the
|
|
69
|
+
* behaviour above, and search can serve a PARTIAL copy for as long as the replay
|
|
70
|
+
* takes. An in-place rebuild has no publish; there the wait holds the closing
|
|
71
|
+
* reconcile and the cursor save instead, so the reconcile never deletes a record
|
|
72
|
+
* the replay has not reached yet. Without a `readiness` an in-place rebuild
|
|
73
|
+
* reconciles at the feed's end too, and so deletes from the live copy every record
|
|
74
|
+
* the replay had not yet written again; the tail sends each back as the replay
|
|
75
|
+
* reaches it.
|
|
76
|
+
*
|
|
77
|
+
* The wait is UNBOUNDED. A probe that never says yes holds the attempt for good,
|
|
78
|
+
* with no error and no restart, because publishing early is the very failure the
|
|
79
|
+
* wait exists to prevent and no timeout could tell a stuck replay from a long one.
|
|
80
|
+
* It is made visible instead: `inspectFeed` reports the sink `resyncDue` for as long
|
|
81
|
+
* as it lasts, since the cursor stays at the old generation, and every
|
|
82
|
+
* `HELD_WAIT_WARN_POLLS` poll intervals (`tuning.ts`) the attempt logs a warning,
|
|
83
|
+
* "feed sync rebuild held on readiness", with the feed, the sink, the generation,
|
|
84
|
+
* the polls so far and how long it has been held.
|
|
85
|
+
*
|
|
86
|
+
* Each `begin` names a target of its own, never one an earlier attempt used, and
|
|
87
|
+
* that is what bounds a lease lapse here. A worker that has lost its lease can
|
|
88
|
+
* still land one call after the new holder's (`FeedSyncStore.ts` says when). Were
|
|
89
|
+
* the target named after its generation alone, that call could be a `publish` of
|
|
90
|
+
* the NEW holder's half-built target, which then goes on loading into a fresh one
|
|
91
|
+
* holding only the later pages, publishes it, and saves its cursor at the head:
|
|
92
|
+
* whole pages lost, and nothing to bring them back. With a target per attempt, a
|
|
93
|
+
* stale `publish` moves only the stale worker's own target — or fails, retryably,
|
|
94
|
+
* when a newer `begin` has dropped it — and a stale `begin` drops the new holder's
|
|
95
|
+
* target, whose `publish` then fails retryably and whose resync starts again.
|
|
96
|
+
* What a stale `publish` of its own target still costs is `FeedSyncStore.ts`'s to
|
|
97
|
+
* say, with the rest of the lease's window.
|
|
98
|
+
*
|
|
99
|
+
* `begin`, `load`, `apply` and `publish` each run behind `ensureHeld`; `confirm`
|
|
100
|
+
* and a reconcile's key listing change nothing, and do not.
|
|
101
|
+
*
|
|
102
|
+
* Tombstones are sent as deletes, although the target starts empty. The feed does
|
|
103
|
+
* not stand still while a resync pages through it: a key loaded on an early page
|
|
104
|
+
* can be deleted by the read model before the resync is done, and its tombstone
|
|
105
|
+
* then sits at a `seq` a LATER page reads. Dropping it would load the key, save
|
|
106
|
+
* the cursor past its delete, and publish a target that holds a record the feed no
|
|
107
|
+
* longer has, with nothing ever coming to remove it. A tombstone for a key the
|
|
108
|
+
* target never held costs a send and changes nothing, since a delete of an absent
|
|
109
|
+
* key is a no-op by the sink laws.
|
|
110
|
+
*
|
|
111
|
+
* The same movement decides the parked list. A key can be met more than once in
|
|
112
|
+
* one resync — refused on an early page, then changed and met again later — so
|
|
113
|
+
* the list saved is what each key's LAST change in the resync came to: parked if
|
|
114
|
+
* that change was refused, absent if it was taken. Keeping every refusal the
|
|
115
|
+
* resync met would leave parked a key the target holds, or one the feed deleted.
|
|
116
|
+
*
|
|
117
|
+
* A sink with no `resync` is resynced IN PLACE: every row is sent to the live copy,
|
|
118
|
+
* live rows as upserts and tombstones as deletes, and then a reconcile deletes what
|
|
119
|
+
* the feed no longer holds. The live copy serves a mixture of old and new from the
|
|
120
|
+
* first page on, which is the price of a target with no atomic swap. Its refusals
|
|
121
|
+
* land in the parked list it saves like any resync's, with two consequences of
|
|
122
|
+
* writing over the live copy: a refused upsert leaves the PREVIOUS generation's
|
|
123
|
+
* document of that key live, and an orphan whose delete the closing reconcile was
|
|
124
|
+
* refused stays live too. Both are parked with the sink's reason, the orphan at the
|
|
125
|
+
* origin `seq` since it has no row, and the closing reconcile's report goes to
|
|
126
|
+
* `onReconcile` like a scheduled one's. A sink with neither part could never be
|
|
127
|
+
* rebuilt, and is refused at construction.
|
|
128
|
+
*
|
|
129
|
+
* ## Supervision
|
|
130
|
+
*
|
|
131
|
+
* Every failure a session meets ends it, and the worker's own loop decides what
|
|
132
|
+
* next. It follows `@kairos-es/automation`'s daemon rather than generalising the
|
|
133
|
+
* read side's supervisor, whose own doc says a progress signal the caller owns does
|
|
134
|
+
* not qualify — and the signal here, a cursor advance, is the worker's own.
|
|
135
|
+
*
|
|
136
|
+
* - `LeaseHeld`: another worker is running. This one waits a poll interval and
|
|
137
|
+
* tries again, and the wait is not a failure: a standby worker is a healthy one,
|
|
138
|
+
* and it takes over within a poll interval of the first worker's lease going.
|
|
139
|
+
* - Anything else: a back-off of `restartMinDelay` doubling to `restartMaxDelay`,
|
|
140
|
+
* scaled by a jitter drawn from `Random` so a fleet of workers does not retry in
|
|
141
|
+
* lockstep, and never shorter than a `SinkRetryable`'s `retryAfter`. A restart
|
|
142
|
+
* that follows no cursor advance since the previous restart counts towards
|
|
143
|
+
* `maxNoProgressRestarts`; reaching it ends the worker with `FeedSyncStalled`,
|
|
144
|
+
* for the caller to report. Any advance resets the count and the curve.
|
|
145
|
+
*
|
|
146
|
+
* Defects are not caught: they are bugs, in the driver or in a sink or store that
|
|
147
|
+
* should have classified its fault, and supervision must not retry a bug.
|
|
148
|
+
*
|
|
149
|
+
* ## The scope
|
|
150
|
+
*
|
|
151
|
+
* The worker never returns. Run it in a fibre of its own beside the projections,
|
|
152
|
+
* and close the scope to stop it: each session runs in a child scope of the
|
|
153
|
+
* caller's, so closing the caller's scope releases the lease.
|
|
154
|
+
*/
|
|
155
|
+
import { Duration, Effect, Scope } from 'effect';
|
|
156
|
+
import { FeedSyncStalled } from './errors.js';
|
|
157
|
+
import type { FeedSource } from './Feed.js';
|
|
158
|
+
import { type FeedSink } from './FeedSink.js';
|
|
159
|
+
import type { FeedSyncStore } from './FeedSyncStore.js';
|
|
160
|
+
import { type FeedName, type SinkName } from './identifiers.js';
|
|
161
|
+
import type { FeedReadiness } from './readiness.js';
|
|
162
|
+
import { type ReconcileReport } from './reconcile.js';
|
|
163
|
+
/** What `onRestart` is told before each restart's back-off. */
|
|
164
|
+
export interface FeedSyncRestart {
|
|
165
|
+
readonly feed: FeedName;
|
|
166
|
+
readonly sink: SinkName;
|
|
167
|
+
/** Consecutive restarts with no cursor advance, this one included. */
|
|
168
|
+
readonly restarts: number;
|
|
169
|
+
readonly delay: Duration.Duration;
|
|
170
|
+
/** The failure that ended the session. */
|
|
171
|
+
readonly reason: unknown;
|
|
172
|
+
}
|
|
173
|
+
/** The worker's knobs. Every one has a default; see `tuning.ts`'s module doc. */
|
|
174
|
+
export interface FeedSyncTuning {
|
|
175
|
+
/** The upper bound on an idle sleep. Default 1 second. */
|
|
176
|
+
readonly pollInterval?: Duration.Input;
|
|
177
|
+
/** Rows read, and changes sent, per page. Default core's catch-up page, 512. */
|
|
178
|
+
readonly pageSize?: number;
|
|
179
|
+
/** The first restart back-off. Default 100 ms. */
|
|
180
|
+
readonly restartMinDelay?: Duration.Input;
|
|
181
|
+
/** The restart back-off's cap, before jitter. Default 30 seconds. */
|
|
182
|
+
readonly restartMaxDelay?: Duration.Input;
|
|
183
|
+
/** Consecutive restarts with no cursor advance before `FeedSyncStalled`. Default 40. */
|
|
184
|
+
readonly maxNoProgressRestarts?: number;
|
|
185
|
+
readonly onRestart?: (info: FeedSyncRestart) => Effect.Effect<void>;
|
|
186
|
+
/**
|
|
187
|
+
* Told what each reconcile the worker runs did: each scheduled one, and the one
|
|
188
|
+
* that closes an in-place rebuild.
|
|
189
|
+
*/
|
|
190
|
+
readonly onReconcile?: (report: ReconcileReport) => Effect.Effect<void>;
|
|
191
|
+
}
|
|
192
|
+
/** Options for {@link runFeedSync}. */
|
|
193
|
+
export interface RunFeedSyncOptions<D, Receipt> {
|
|
194
|
+
readonly source: FeedSource<D>;
|
|
195
|
+
readonly sink: FeedSink<D, Receipt>;
|
|
196
|
+
readonly store: FeedSyncStore;
|
|
197
|
+
readonly tuning?: FeedSyncTuning;
|
|
198
|
+
/**
|
|
199
|
+
* Reconcile the sink against the feed this often, between batches, under the
|
|
200
|
+
* worker's own lease. Needs the sink's `reconcile`; absent, the worker
|
|
201
|
+
* reconciles only as part of an in-place resync.
|
|
202
|
+
*/
|
|
203
|
+
readonly reconcileEvery?: Duration.Input;
|
|
204
|
+
/**
|
|
205
|
+
* Hold each resync's `publish` — or, in place, its closing reconcile — until the
|
|
206
|
+
* materialisation writing the feed has caught up with the log, for as long as
|
|
207
|
+
* that takes; see the module doc's Resync section. Absent, a resync publishes at
|
|
208
|
+
* the feed's end, and search can be PARTIAL while a rebuild's replay runs.
|
|
209
|
+
* `materialisationCaughtUp` builds the common case from the read model.
|
|
210
|
+
*/
|
|
211
|
+
readonly readiness?: FeedReadiness;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Run the sync worker for `options.source` and `options.sink`; see the module doc.
|
|
215
|
+
* Dies at once, before taking the lease, on a sink that has neither `resync` nor
|
|
216
|
+
* `reconcile`, on a `reconcileEvery` for a sink with no `reconcile`, and on tuning
|
|
217
|
+
* that is out of range.
|
|
218
|
+
*/
|
|
219
|
+
export declare const runFeedSync: <D, Receipt>(options: RunFeedSyncOptions<D, Receipt>) => Effect.Effect<never, FeedSyncStalled, Scope.Scope>;
|
|
220
|
+
//# sourceMappingURL=runFeedSync.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runFeedSync.d.ts","sourceRoot":"","sources":["../../src/runFeedSync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyJG;AAGH,OAAO,EAKL,QAAQ,EACR,MAAM,EAIN,KAAK,EAEN,MAAM,QAAQ,CAAA;AAGf,OAAO,EAEL,eAAe,EAQhB,MAAM,UAAU,CAAA;AACjB,OAAO,KAAK,EAAW,UAAU,EAAE,MAAM,QAAQ,CAAA;AACjD,OAAO,EAKL,KAAK,QAAQ,EACd,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAGV,aAAa,EAGd,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAGL,KAAK,QAAQ,EAGb,KAAK,QAAQ,EACd,MAAM,eAAe,CAAA;AACtB,OAAO,KAAK,EAAE,aAAa,EAAsB,MAAM,aAAa,CAAA;AACpE,OAAO,EAEL,KAAK,eAAe,EAGrB,MAAM,aAAa,CAAA;AAYpB,+DAA+D;AAC/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAA;IACjC,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CACzB;AAED,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACtC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,kDAAkD;IAClD,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACzC,qEAAqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACzC,wFAAwF;IACxF,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACnE;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;CACxE;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB,CAAC,CAAC,EAAE,OAAO;IAC5C,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACnC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAA;IAChC;;;;OAIG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACxC;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAA;CACnC;AAmBD;;;;;GAKG;AACH,eAAO,MAAM,WAAW,GAAI,CAAC,EAAE,OAAO,EACpC,SAAS,kBAAkB,CAAC,CAAC,EAAE,OAAO,CAAC,KACtC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,CAiDhD,CAAA"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The driver's tuning: its defaults, and the checks that refuse a nonsensical
|
|
3
|
+
* value at construction rather than somewhere inside a running worker.
|
|
4
|
+
*
|
|
5
|
+
* The page size defaults to core's own catch-up page, and the supervision curve to
|
|
6
|
+
* the read side's — 100 ms doubling to 30 s — so a feed behaves like its
|
|
7
|
+
* neighbours unless told otherwise. The no-progress cap is the read side's too,
|
|
8
|
+
* and for the same reason: forty restarts on that curve is some twenty minutes of
|
|
9
|
+
* a sink refusing everything, which is long enough to ride out an outage and short
|
|
10
|
+
* enough that a broken wiring is reported the same day.
|
|
11
|
+
*
|
|
12
|
+
* Package-internal; the options a caller sets are `FeedSyncTuning`'s, on
|
|
13
|
+
* `runFeedSync`.
|
|
14
|
+
*/
|
|
15
|
+
import { ReadLimit } from '@kairos-es/core';
|
|
16
|
+
import { Duration } from 'effect';
|
|
17
|
+
/** Rows read, and changes sent, per page. */
|
|
18
|
+
export declare const DEFAULT_FEED_PAGE_SIZE: number;
|
|
19
|
+
/** The upper bound on an idle worker's sleep: the poll that is the guarantee. */
|
|
20
|
+
export declare const DEFAULT_FEED_POLL_INTERVAL: Duration.Duration;
|
|
21
|
+
/** The first restart back-off. */
|
|
22
|
+
export declare const DEFAULT_FEED_RESTART_MIN_DELAY: Duration.Duration;
|
|
23
|
+
/** The restart back-off's cap, before jitter. */
|
|
24
|
+
export declare const DEFAULT_FEED_RESTART_MAX_DELAY: Duration.Duration;
|
|
25
|
+
/** Consecutive restarts without a cursor advance before the worker gives up. */
|
|
26
|
+
export declare const DEFAULT_FEED_MAX_NO_PROGRESS_RESTARTS = 40;
|
|
27
|
+
/**
|
|
28
|
+
* How many poll intervals a rebuild held on its readiness waits between two
|
|
29
|
+
* warnings that it is still held. Not a bound: the wait goes on, since publishing
|
|
30
|
+
* early is the failure the wait exists to prevent. Counted in polls, not time, so
|
|
31
|
+
* it scales with `pollInterval`: sixty polls is long past any replay that is
|
|
32
|
+
* merely busy at any sensible interval, and rare enough not to flood a log while
|
|
33
|
+
* a long one runs.
|
|
34
|
+
*/
|
|
35
|
+
export declare const HELD_WAIT_WARN_POLLS = 60;
|
|
36
|
+
/** The page size as a `ReadLimit`, refusing one that is not a positive integer. */
|
|
37
|
+
export declare const readLimit: (pageSize: number) => ReadLimit;
|
|
38
|
+
/** A duration input decoded, refusing one that is not positive and finite. */
|
|
39
|
+
export declare const positiveDuration: (label: string, input: Duration.Input) => Duration.Duration;
|
|
40
|
+
//# sourceMappingURL=tuning.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tuning.d.ts","sourceRoot":"","sources":["../../src/tuning.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAA8B,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEjC,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB,EAAE,MAAmC,CAAA;AAExE,iFAAiF;AACjF,eAAO,MAAM,0BAA0B,EAAE,QAAQ,CAAC,QAA8B,CAAA;AAEhF,kCAAkC;AAClC,eAAO,MAAM,8BAA8B,EAAE,QAAQ,CAAC,QAChC,CAAA;AAEtB,iDAAiD;AACjD,eAAO,MAAM,8BAA8B,EAAE,QAAQ,CAAC,QAChC,CAAA;AAEtB,gFAAgF;AAChF,eAAO,MAAM,qCAAqC,KAAK,CAAA;AAEvD;;;;;;;GAOG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAA;AAEtC,mFAAmF;AACnF,eAAO,MAAM,SAAS,GAAI,UAAU,MAAM,KAAG,SAO5C,CAAA;AAED,8EAA8E;AAC9E,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,EACb,OAAO,QAAQ,CAAC,KAAK,KACpB,QAAQ,CAAC,QAMX,CAAA"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The feed's two faces: the WRITE side a read model's `apply` uses inside its
|
|
3
|
+
* commit, and the READ side a sync worker tails.
|
|
4
|
+
*
|
|
5
|
+
* ## Why the feed is written inside the materialisation's commit
|
|
6
|
+
*
|
|
7
|
+
* A feed row exists if and only if the view write and the checkpoint advance that
|
|
8
|
+
* produced it landed (ADR-0019 §2). That is the read side's existing exactly-once
|
|
9
|
+
* seam, reused rather than rebuilt: the `ProjectionStore` brackets a read model's
|
|
10
|
+
* view writes with its guarded checkpoint advance, so a feed written through the
|
|
11
|
+
* same bracket commits with the view, rolls back with it on a lost guard, and is
|
|
12
|
+
* never seen half-written. Nothing else about the feed's correctness would survive
|
|
13
|
+
* a feed written beside the commit instead of inside it.
|
|
14
|
+
*
|
|
15
|
+
* ## Why writes are batched before they land
|
|
16
|
+
*
|
|
17
|
+
* `BatchedFeedWriter.write` hands its body a `FeedWriter`, collects every upsert
|
|
18
|
+
* and remove the body makes, and lands them together when the body succeeds:
|
|
19
|
+
* the last write per key wins within one call. Two reasons, one per kind of store.
|
|
20
|
+
*
|
|
21
|
+
* - A transactional store would roll a part-written batch back on its own; there
|
|
22
|
+
* coalescing is purely an economy, one row write per key instead of one per
|
|
23
|
+
* event.
|
|
24
|
+
* - A non-transactional store cannot roll anything back, and the `ProjectionStore`
|
|
25
|
+
* port's contract therefore asks that such a store's per-batch view write be ONE
|
|
26
|
+
* atomic effect. Buffering is what makes the feed's half of the batch one atomic
|
|
27
|
+
* effect too, so a body that fails half-way leaves no feed row behind.
|
|
28
|
+
*
|
|
29
|
+
* ## What "coalesced, latest wins" means for the stored row
|
|
30
|
+
*
|
|
31
|
+
* One row per key. An upsert of a document EQUAL to the stored one writes nothing
|
|
32
|
+
* and takes no sequence value, so an event that changes nothing a sink indexes
|
|
33
|
+
* costs the sink nothing (ADR-0019 §4). Equality is of the ENCODED document, as a
|
|
34
|
+
* JSON value whose object keys are unordered, so what counts as a change is the
|
|
35
|
+
* same in every backend. A remove leaves a TOMBSTONE — the row, with no document —
|
|
36
|
+
* so every sink's cursor can pass it and send the delete, and a remove of a key
|
|
37
|
+
* that is absent or already a tombstone writes nothing.
|
|
38
|
+
*/
|
|
39
|
+
import type { Position, ReadLimit } from '@kairos-es/core';
|
|
40
|
+
import type { Effect, Option, Stream } from 'effect';
|
|
41
|
+
import type { FeedUnreadable, FeedWriteFailed } from './errors.js';
|
|
42
|
+
import type { FeedKey, FeedName, FeedSeq, Generation } from './identifiers.js';
|
|
43
|
+
/**
|
|
44
|
+
* One row of a feed: the newest state of one record. `doc` is `None` for a
|
|
45
|
+
* tombstone. `eventPosition` is the position of the event batch that last wrote
|
|
46
|
+
* the row, kept for "the sink is at least as fresh as P" and never used as a
|
|
47
|
+
* cursor; `seq` is the cursor.
|
|
48
|
+
*/
|
|
49
|
+
export interface FeedRow<D> {
|
|
50
|
+
readonly key: FeedKey;
|
|
51
|
+
readonly doc: Option.Option<D>;
|
|
52
|
+
readonly seq: FeedSeq;
|
|
53
|
+
readonly eventPosition: Position;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* What a read model's `apply` writes through, for each search record the batch
|
|
57
|
+
* touches. Used only inside {@link BatchedFeedWriter.write}'s body.
|
|
58
|
+
*/
|
|
59
|
+
export interface FeedWriter<D> {
|
|
60
|
+
/** The record's newest state. Writes nothing when the stored document is equal. */
|
|
61
|
+
readonly upsert: (key: FeedKey, doc: D) => Effect.Effect<void, FeedWriteFailed>;
|
|
62
|
+
/** A tombstone. Writes nothing when the key is absent or already a tombstone. */
|
|
63
|
+
readonly remove: (key: FeedKey) => Effect.Effect<void, FeedWriteFailed>;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* A feed's write side: run `body` with a {@link FeedWriter}, and land what it
|
|
67
|
+
* wrote, coalesced by key, when it succeeds. `at` becomes every written row's
|
|
68
|
+
* `eventPosition`; pass the position of the last event in the batch being
|
|
69
|
+
* applied. `RW` is what the backend itself needs from the context — a SQL backend
|
|
70
|
+
* writes through the read model's own client, so the client is in `RW` and the
|
|
71
|
+
* write joins whatever transaction the `ProjectionStore` opened around `apply`.
|
|
72
|
+
*
|
|
73
|
+
* Call it from inside the materialisation's `commit` and nowhere else: a feed
|
|
74
|
+
* written outside the commit is a feed that can disagree with its view.
|
|
75
|
+
*/
|
|
76
|
+
export interface BatchedFeedWriter<D, RW = never> {
|
|
77
|
+
readonly feed: FeedName;
|
|
78
|
+
readonly write: <A, E, R>(at: Position, body: (writer: FeedWriter<D>) => Effect.Effect<A, E, R>) => Effect.Effect<A, E | FeedWriteFailed, R | RW>;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* A feed's read side, as the sync worker sees it.
|
|
82
|
+
*
|
|
83
|
+
* `changesAfter` returns the rows whose `seq` is strictly greater than `after`, in
|
|
84
|
+
* ascending `seq` order, at most `limit` of them, tombstones included. Its one
|
|
85
|
+
* obligation is the subscribe machine's: a row committed and visible must never
|
|
86
|
+
* have a `seq` below one already returned. A feed bound to one materialisation
|
|
87
|
+
* keeps it, because that materialisation has one writer.
|
|
88
|
+
*
|
|
89
|
+
* `listen`, when present, is a WAKE and nothing more: each element says "there
|
|
90
|
+
* may be new rows", and a missed, duplicated or coalesced one makes a read early
|
|
91
|
+
* or late but never wrong, because the poll is the guarantee.
|
|
92
|
+
*/
|
|
93
|
+
export interface FeedSource<D> {
|
|
94
|
+
readonly feed: FeedName;
|
|
95
|
+
readonly generation: Effect.Effect<Generation, FeedUnreadable>;
|
|
96
|
+
readonly changesAfter: (after: FeedSeq, limit: ReadLimit) => Effect.Effect<ReadonlyArray<FeedRow<D>>, FeedUnreadable>;
|
|
97
|
+
readonly listen?: Stream.Stream<unknown, FeedUnreadable>;
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=Feed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Feed.d.ts","sourceRoot":"","sources":["../../src/Feed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AACpD,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,UAAU,CAAA;AAC/D,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAE3E;;;;;GAKG;AACH,MAAM,WAAW,OAAO,CAAC,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAA;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,mFAAmF;IACnF,QAAQ,CAAC,MAAM,EAAE,CACf,GAAG,EAAE,OAAO,EACZ,GAAG,EAAE,CAAC,KACH,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IACzC,iFAAiF;IACjF,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;CACxE;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC,EAAE,EAAE,GAAG,KAAK;IAC9C,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EACtB,EAAE,EAAE,QAAQ,EACZ,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KACpD,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,GAAG,EAAE,CAAC,CAAA;CACnD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC,CAAA;IAC9D,QAAQ,CAAC,YAAY,EAAE,CACrB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,SAAS,KACb,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;IAC7D,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;CACzD"}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The sink port: what a target a feed is copied to must offer the driver.
|
|
3
|
+
*
|
|
4
|
+
* ## Three parts, two of them optional
|
|
5
|
+
*
|
|
6
|
+
* - `apply` and `confirm` are the whole of the steady state. `apply` sends a
|
|
7
|
+
* batch and returns a `Receipt`; `confirm` resolves once that batch is applied
|
|
8
|
+
* AND VISIBLE in the target. The driver moves its cursor only after `confirm`
|
|
9
|
+
* (ADR-0019 §6), so a crash between the two resends the batch — which is why
|
|
10
|
+
* every sink must be IDEMPOTENT: the same batch applied twice leaves the target
|
|
11
|
+
* as once. Full-record replace satisfies that; a partial operation such as an
|
|
12
|
+
* increment does not, and a sink must never use one.
|
|
13
|
+
* - `resync` is how a sink rebuilds after the feed's generation moves: load every
|
|
14
|
+
* row into a fresh target that is not yet serving — tombstones too, as deletes,
|
|
15
|
+
* since the feed can delete a key the same resync has already loaded — then swap
|
|
16
|
+
* it over the live one atomically, so search keeps serving the old copy until the new one is
|
|
17
|
+
* complete and records deleted from the feed do not survive the rebuild.
|
|
18
|
+
* - `reconcile` lists the keys the target holds, so the driver can delete those
|
|
19
|
+
* the feed no longer has. Deletes go through the ordinary `apply`.
|
|
20
|
+
*
|
|
21
|
+
* A sink with no atomic swap leaves out `resync`, and the driver then resyncs IN
|
|
22
|
+
* PLACE: it sends every row to the live target and reconciles away what the feed
|
|
23
|
+
* no longer holds. That needs `reconcile`, so `runFeedSync` refuses at
|
|
24
|
+
* construction a sink that has neither part: such a sink could never be rebuilt.
|
|
25
|
+
*
|
|
26
|
+
* ## Why `FeedChange` is not `FeedRow`
|
|
27
|
+
*
|
|
28
|
+
* A sink is handed the change to make, not the row it came from: an `Upsert` with
|
|
29
|
+
* the document or a `Delete`. The sequence and the event position are the
|
|
30
|
+
* driver's business, and a sink that could see them could be tempted to keep a
|
|
31
|
+
* cursor of its own, which would be a second record of progress.
|
|
32
|
+
*/
|
|
33
|
+
import { Data, type Effect, type Stream } from 'effect';
|
|
34
|
+
import type { NonEmptyReadonlyArray } from 'effect/Array';
|
|
35
|
+
import type { SinkHalted, SinkRejected, SinkRetryable } from './errors.js';
|
|
36
|
+
import type { FeedRow } from './Feed.js';
|
|
37
|
+
import type { FeedKey, Generation, SinkName } from './identifiers.js';
|
|
38
|
+
/** One change for a sink to make: the record's newest state, or its removal. */
|
|
39
|
+
export type FeedChange<D> = Data.TaggedEnum<{
|
|
40
|
+
Upsert: {
|
|
41
|
+
readonly key: FeedKey;
|
|
42
|
+
readonly doc: D;
|
|
43
|
+
};
|
|
44
|
+
Delete: {
|
|
45
|
+
readonly key: FeedKey;
|
|
46
|
+
};
|
|
47
|
+
}>;
|
|
48
|
+
/** The generic definition behind {@link FeedChange}'s constructors. */
|
|
49
|
+
export interface FeedChangeDefinition extends Data.TaggedEnum.WithGenerics<1> {
|
|
50
|
+
readonly taggedEnum: FeedChange<this['A']>;
|
|
51
|
+
}
|
|
52
|
+
/** `FeedChange.Upsert({ key, doc })`, `FeedChange.Delete({ key })`, and matchers. */
|
|
53
|
+
export declare const FeedChange: {
|
|
54
|
+
readonly Upsert: <A>(args: {
|
|
55
|
+
readonly key: FeedKey;
|
|
56
|
+
readonly doc: A;
|
|
57
|
+
}) => {
|
|
58
|
+
readonly _tag: "Upsert";
|
|
59
|
+
readonly key: FeedKey;
|
|
60
|
+
readonly doc: A;
|
|
61
|
+
};
|
|
62
|
+
readonly Delete: <A>(args: {
|
|
63
|
+
readonly key: FeedKey;
|
|
64
|
+
}) => {
|
|
65
|
+
readonly _tag: "Delete";
|
|
66
|
+
readonly key: FeedKey;
|
|
67
|
+
};
|
|
68
|
+
readonly $is: <Tag extends "Upsert" | "Delete">(tag: Tag) => {
|
|
69
|
+
<T extends {
|
|
70
|
+
readonly _tag: "Upsert";
|
|
71
|
+
readonly key: FeedKey;
|
|
72
|
+
readonly doc: any;
|
|
73
|
+
} | {
|
|
74
|
+
readonly _tag: "Delete";
|
|
75
|
+
readonly key: FeedKey;
|
|
76
|
+
}>(u: T): u is T & {
|
|
77
|
+
readonly _tag: Tag;
|
|
78
|
+
};
|
|
79
|
+
(u: unknown): u is Extract<{
|
|
80
|
+
readonly _tag: "Upsert";
|
|
81
|
+
readonly key: FeedKey;
|
|
82
|
+
readonly doc: unknown;
|
|
83
|
+
}, {
|
|
84
|
+
readonly _tag: Tag;
|
|
85
|
+
}> | Extract<{
|
|
86
|
+
readonly _tag: "Delete";
|
|
87
|
+
readonly key: FeedKey;
|
|
88
|
+
}, {
|
|
89
|
+
readonly _tag: Tag;
|
|
90
|
+
}>;
|
|
91
|
+
};
|
|
92
|
+
readonly $match: {
|
|
93
|
+
<A, B, C, D, Cases extends {
|
|
94
|
+
readonly Upsert: (args: {
|
|
95
|
+
readonly _tag: "Upsert";
|
|
96
|
+
readonly key: FeedKey;
|
|
97
|
+
readonly doc: A;
|
|
98
|
+
}) => any;
|
|
99
|
+
readonly Delete: (args: {
|
|
100
|
+
readonly _tag: "Delete";
|
|
101
|
+
readonly key: FeedKey;
|
|
102
|
+
}) => any;
|
|
103
|
+
}>(cases: Cases): (self: {
|
|
104
|
+
readonly _tag: "Upsert";
|
|
105
|
+
readonly key: FeedKey;
|
|
106
|
+
readonly doc: A;
|
|
107
|
+
} | {
|
|
108
|
+
readonly _tag: "Delete";
|
|
109
|
+
readonly key: FeedKey;
|
|
110
|
+
}) => import("effect/Unify").Unify<ReturnType<Cases["Upsert" | "Delete"]>>;
|
|
111
|
+
<A, B, C, D, Cases extends {
|
|
112
|
+
readonly Upsert: (args: {
|
|
113
|
+
readonly _tag: "Upsert";
|
|
114
|
+
readonly key: FeedKey;
|
|
115
|
+
readonly doc: A;
|
|
116
|
+
}) => any;
|
|
117
|
+
readonly Delete: (args: {
|
|
118
|
+
readonly _tag: "Delete";
|
|
119
|
+
readonly key: FeedKey;
|
|
120
|
+
}) => any;
|
|
121
|
+
}>(self: {
|
|
122
|
+
readonly _tag: "Upsert";
|
|
123
|
+
readonly key: FeedKey;
|
|
124
|
+
readonly doc: A;
|
|
125
|
+
} | {
|
|
126
|
+
readonly _tag: "Delete";
|
|
127
|
+
readonly key: FeedKey;
|
|
128
|
+
}, cases: Cases): import("effect/Unify").Unify<ReturnType<Cases["Upsert" | "Delete"]>>;
|
|
129
|
+
};
|
|
130
|
+
};
|
|
131
|
+
/** The change a row asks of a sink: its document as an upsert, a tombstone as a delete. */
|
|
132
|
+
export declare const changeOf: <D>(row: FeedRow<D>) => FeedChange<D>;
|
|
133
|
+
/**
|
|
134
|
+
* Where ONE resync attempt is being built. `name` is the sink's own name for it (an
|
|
135
|
+
* index name, say), and it is the attempt's alone: no two `begin`s return the same
|
|
136
|
+
* one, even for one generation. So a worker that has lost its lease without yet
|
|
137
|
+
* knowing it can only ever publish its OWN target, never the half-built target of
|
|
138
|
+
* the worker that holds the lease now; `runFeedSync.ts` owns what that buys.
|
|
139
|
+
*/
|
|
140
|
+
export interface ResyncTarget {
|
|
141
|
+
readonly generation: Generation;
|
|
142
|
+
readonly name: string;
|
|
143
|
+
}
|
|
144
|
+
/** A sink's rebuild: a fresh target, loaded, then swapped over the live one. */
|
|
145
|
+
export interface FeedResync<D, Receipt> {
|
|
146
|
+
/**
|
|
147
|
+
* A fresh, empty target that is not live, under a name no earlier `begin`
|
|
148
|
+
* returned. Drops every other target an earlier attempt left — crashed,
|
|
149
|
+
* abandoned, or still being loaded by a worker that lost its lease — whose
|
|
150
|
+
* `publish` must then fail retryably rather than touch the live copy.
|
|
151
|
+
*/
|
|
152
|
+
readonly begin: (generation: Generation) => Effect.Effect<ResyncTarget, SinkRetryable | SinkHalted>;
|
|
153
|
+
/**
|
|
154
|
+
* `apply`, against the target rather than the live copy. Confirmed by `confirm`.
|
|
155
|
+
* A target a later `begin` dropped must NEVER BE PUBLISHED, since the worker
|
|
156
|
+
* loading into it can be one that lost its lease without knowing it, and a target
|
|
157
|
+
* rebuilt from its later pages alone must never be swapped over the live copy as
|
|
158
|
+
* a fragment. Failing `SinkRetryable` here is the plain way to keep that. A sink
|
|
159
|
+
* whose writes recreate a missing target (an index that springs into being on its
|
|
160
|
+
* first write) may let the load land and recreate it instead, provided `publish`
|
|
161
|
+
* then tells the recreated target from the one `begin` made and refuses it.
|
|
162
|
+
*/
|
|
163
|
+
readonly load: (target: ResyncTarget, batch: NonEmptyReadonlyArray<FeedChange<D>>) => Effect.Effect<Receipt, SinkRetryable | SinkRejected | SinkHalted>;
|
|
164
|
+
/**
|
|
165
|
+
* Swap the target over the live copy, atomically, and resolve once it serves. A
|
|
166
|
+
* target a later `begin` dropped — gone, or brought back by a load — fails
|
|
167
|
+
* `SinkRetryable` and leaves the live copy as it was. Once the swap has landed, a
|
|
168
|
+
* sink may tidy the live copy of its own bookkeeping (a marker it set to tell its
|
|
169
|
+
* target apart) on a best-effort basis: the swap is the publish, so a failure of
|
|
170
|
+
* that tidying must not fail it.
|
|
171
|
+
*/
|
|
172
|
+
readonly publish: (target: ResyncTarget) => Effect.Effect<void, SinkRetryable | SinkHalted>;
|
|
173
|
+
}
|
|
174
|
+
/** A sink's key listing, for deleting what the feed no longer holds. */
|
|
175
|
+
export interface FeedReconcile {
|
|
176
|
+
/** Every key the live target holds, in any order, each once. */
|
|
177
|
+
readonly listKeys: Stream.Stream<FeedKey, SinkRetryable | SinkHalted>;
|
|
178
|
+
}
|
|
179
|
+
/** A target a feed is copied to. See the module doc. */
|
|
180
|
+
export interface FeedSink<D, Receipt> {
|
|
181
|
+
readonly name: SinkName;
|
|
182
|
+
readonly apply: (batch: NonEmptyReadonlyArray<FeedChange<D>>) => Effect.Effect<Receipt, SinkRetryable | SinkRejected | SinkHalted>;
|
|
183
|
+
/** Resolves once the batch the receipt names is applied and visible. */
|
|
184
|
+
readonly confirm: (receipt: Receipt) => Effect.Effect<void, SinkRetryable | SinkHalted>;
|
|
185
|
+
/** Absent: the driver resyncs in place, which needs `reconcile`. */
|
|
186
|
+
readonly resync?: FeedResync<D, Receipt>;
|
|
187
|
+
readonly reconcile?: FeedReconcile;
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=FeedSink.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FeedSink.d.ts","sourceRoot":"","sources":["../../src/FeedSink.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,IAAI,EAAE,KAAK,MAAM,EAAU,KAAK,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/D,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACvE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAElE,gFAAgF;AAChF,MAAM,MAAM,UAAU,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC;IAC1C,MAAM,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAClD,MAAM,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;KAAE,CAAA;CAClC,CAAC,CAAA;AAEF,uEAAuE;AACvE,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;IAC3E,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;CAC3C;AAED,qFAAqF;AACrF,eAAO,MAAM,UAAU;;sBAVG,OAAO;;;;sBAAP,OAAO;;;;sBACP,OAAO;;;sBAAP,OAAO;;;;;0BADP,OAAO;;;;0BACP,OAAO;;;;;;0BADP,OAAO;;;;;;0BACP,OAAO;;;;;;;;;8BADP,OAAO;;;;;8BACP,OAAO;;;;0BADP,OAAO;;;;0BACP,OAAO;;;;;8BADP,OAAO;;;;;8BACP,OAAO;;;;0BADP,OAAO;;;;0BACP,OAAO;;;CASgC,CAAA;AAEjE,2FAA2F;AAC3F,eAAO,MAAM,QAAQ,GAAI,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,KAAG,UAAU,CAAC,CAAC,CAItD,CAAA;AAEJ;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED,gFAAgF;AAChF,MAAM,WAAW,UAAU,CAAC,CAAC,EAAE,OAAO;IACpC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,CACd,UAAU,EAAE,UAAU,KACnB,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;IAC5D;;;;;;;;;OASG;IACH,QAAQ,CAAC,IAAI,EAAE,CACb,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KACxC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,GAAG,UAAU,CAAC,CAAA;IACtE;;;;;;;OAOG;IACH,QAAQ,CAAC,OAAO,EAAE,CAChB,MAAM,EAAE,YAAY,KACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;CACrD;AAED,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;CACtE;AAED,wDAAwD;AACxD,MAAM,WAAW,QAAQ,CAAC,CAAC,EAAE,OAAO;IAClC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,CACd,KAAK,EAAE,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KACxC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,GAAG,UAAU,CAAC,CAAA;IACtE,wEAAwE;IACxE,QAAQ,CAAC,OAAO,EAAE,CAChB,OAAO,EAAE,OAAO,KACb,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;IACpD,oEAAoE;IACpE,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACxC,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAA;CACnC"}
|