@kairos-es/feed 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +109 -2
- package/dist/cjs/Feed.js +6 -0
- package/dist/cjs/Feed.js.map +1 -0
- package/dist/cjs/FeedSink.js +54 -0
- package/dist/cjs/FeedSink.js.map +1 -0
- package/dist/cjs/FeedSyncStore.js +10 -0
- package/dist/cjs/FeedSyncStore.js.map +1 -0
- package/dist/cjs/deliver.js +86 -0
- package/dist/cjs/deliver.js.map +1 -0
- package/dist/cjs/errors.js +114 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/identifiers.js +63 -0
- package/dist/cjs/identifiers.js.map +1 -0
- package/dist/cjs/inMemoryFeed.js +328 -0
- package/dist/cjs/inMemoryFeed.js.map +1 -0
- package/dist/cjs/inMemorySink.js +142 -0
- package/dist/cjs/inMemorySink.js.map +1 -0
- package/dist/cjs/index.js +184 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/inspect.js +68 -0
- package/dist/cjs/inspect.js.map +1 -0
- package/dist/cjs/readiness.js +100 -0
- package/dist/cjs/readiness.js.map +1 -0
- package/dist/cjs/reconcile.js +164 -0
- package/dist/cjs/reconcile.js.map +1 -0
- package/dist/cjs/runFeedSync.js +603 -0
- package/dist/cjs/runFeedSync.js.map +1 -0
- package/dist/cjs/tuning.js +60 -0
- package/dist/cjs/tuning.js.map +1 -0
- package/dist/dts-cjs/Feed.d.ts +99 -0
- package/dist/dts-cjs/Feed.d.ts.map +1 -0
- package/dist/dts-cjs/FeedSink.d.ts +189 -0
- package/dist/dts-cjs/FeedSink.d.ts.map +1 -0
- package/dist/dts-cjs/FeedSyncStore.d.ts +281 -0
- package/dist/dts-cjs/FeedSyncStore.d.ts.map +1 -0
- package/dist/dts-cjs/deliver.d.ts +16 -0
- package/dist/dts-cjs/deliver.d.ts.map +1 -0
- package/dist/dts-cjs/errors.d.ts +131 -0
- package/dist/dts-cjs/errors.d.ts.map +1 -0
- package/dist/dts-cjs/identifiers.d.ts +67 -0
- package/dist/dts-cjs/identifiers.d.ts.map +1 -0
- package/dist/dts-cjs/inMemoryFeed.d.ts +32 -0
- package/dist/dts-cjs/inMemoryFeed.d.ts.map +1 -0
- package/dist/dts-cjs/inMemorySink.d.ts +61 -0
- package/dist/dts-cjs/inMemorySink.d.ts.map +1 -0
- package/dist/dts-cjs/index.d.ts +63 -0
- package/dist/dts-cjs/index.d.ts.map +1 -0
- package/dist/dts-cjs/inspect.d.ts +71 -0
- package/dist/dts-cjs/inspect.d.ts.map +1 -0
- package/dist/dts-cjs/package.json +3 -0
- package/dist/dts-cjs/readiness.d.ts +84 -0
- package/dist/dts-cjs/readiness.d.ts.map +1 -0
- package/dist/dts-cjs/reconcile.d.ts +68 -0
- package/dist/dts-cjs/reconcile.d.ts.map +1 -0
- package/dist/dts-cjs/runFeedSync.d.ts +220 -0
- package/dist/dts-cjs/runFeedSync.d.ts.map +1 -0
- package/dist/dts-cjs/tuning.d.ts +40 -0
- package/dist/dts-cjs/tuning.d.ts.map +1 -0
- package/dist/dts-esm/Feed.d.ts +99 -0
- package/dist/dts-esm/Feed.d.ts.map +1 -0
- package/dist/dts-esm/FeedSink.d.ts +189 -0
- package/dist/dts-esm/FeedSink.d.ts.map +1 -0
- package/dist/dts-esm/FeedSyncStore.d.ts +281 -0
- package/dist/dts-esm/FeedSyncStore.d.ts.map +1 -0
- package/dist/dts-esm/deliver.d.ts +16 -0
- package/dist/dts-esm/deliver.d.ts.map +1 -0
- package/dist/dts-esm/errors.d.ts +131 -0
- package/dist/dts-esm/errors.d.ts.map +1 -0
- package/dist/dts-esm/identifiers.d.ts +67 -0
- package/dist/dts-esm/identifiers.d.ts.map +1 -0
- package/dist/dts-esm/inMemoryFeed.d.ts +32 -0
- package/dist/dts-esm/inMemoryFeed.d.ts.map +1 -0
- package/dist/dts-esm/inMemorySink.d.ts +61 -0
- package/dist/dts-esm/inMemorySink.d.ts.map +1 -0
- package/dist/dts-esm/index.d.ts +63 -0
- package/dist/dts-esm/index.d.ts.map +1 -0
- package/dist/dts-esm/inspect.d.ts +71 -0
- package/dist/dts-esm/inspect.d.ts.map +1 -0
- package/dist/dts-esm/package.json +3 -0
- package/dist/dts-esm/readiness.d.ts +84 -0
- package/dist/dts-esm/readiness.d.ts.map +1 -0
- package/dist/dts-esm/reconcile.d.ts +68 -0
- package/dist/dts-esm/reconcile.d.ts.map +1 -0
- package/dist/dts-esm/runFeedSync.d.ts +220 -0
- package/dist/dts-esm/runFeedSync.d.ts.map +1 -0
- package/dist/dts-esm/tuning.d.ts +40 -0
- package/dist/dts-esm/tuning.d.ts.map +1 -0
- package/dist/esm/Feed.js +2 -0
- package/dist/esm/Feed.js.map +1 -0
- package/dist/esm/FeedSink.js +46 -0
- package/dist/esm/FeedSink.js.map +1 -0
- package/dist/esm/FeedSyncStore.js +3 -0
- package/dist/esm/FeedSyncStore.js.map +1 -0
- package/dist/esm/deliver.js +78 -0
- package/dist/esm/deliver.js.map +1 -0
- package/dist/esm/errors.js +98 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/identifiers.js +56 -0
- package/dist/esm/identifiers.js.map +1 -0
- package/dist/esm/inMemoryFeed.js +320 -0
- package/dist/esm/inMemoryFeed.js.map +1 -0
- package/dist/esm/inMemorySink.js +134 -0
- package/dist/esm/inMemorySink.js.map +1 -0
- package/dist/esm/index.js +62 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/inspect.js +58 -0
- package/dist/esm/inspect.js.map +1 -0
- package/dist/esm/package.json +4 -0
- package/dist/esm/readiness.js +92 -0
- package/dist/esm/readiness.js.map +1 -0
- package/dist/esm/reconcile.js +153 -0
- package/dist/esm/reconcile.js.map +1 -0
- package/dist/esm/runFeedSync.js +595 -0
- package/dist/esm/runFeedSync.js.map +1 -0
- package/dist/esm/tuning.js +51 -0
- package/dist/esm/tuning.js.map +1 -0
- package/package.json +34 -3
- package/src/Feed.ts +111 -0
- package/src/FeedSink.ts +130 -0
- package/src/FeedSyncStore.ts +326 -0
- package/src/deliver.ts +108 -0
- package/src/errors.ts +134 -0
- package/src/identifiers.ts +92 -0
- package/src/inMemoryFeed.ts +582 -0
- package/src/inMemorySink.ts +215 -0
- package/src/index.ts +136 -0
- package/src/inspect.ts +114 -0
- package/src/readiness.ts +137 -0
- package/src/reconcile.ts +224 -0
- package/src/runFeedSync.ts +882 -0
- package/src/tuning.ts +65 -0
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `FeedSyncStore` — the port for a sync worker's durable state, kept in the SOURCE
|
|
3
|
+
* database beside the feed rows: one cursor per feed and sink, the parked list,
|
|
4
|
+
* and the worker lease (ADR-0019 §10).
|
|
5
|
+
*
|
|
6
|
+
* The in-memory feed implements it as the parity oracle, and `@kairos-es/feed-postgres`
|
|
7
|
+
* implements it over its cursor and parked tables. Everything below is the
|
|
8
|
+
* contract both must keep; the driver relies on no more than this.
|
|
9
|
+
*
|
|
10
|
+
* ## The lease, and why a feed takes one where a projection does not
|
|
11
|
+
*
|
|
12
|
+
* A projection needs no lease (ADR-0007): its view is transactional, so a rival
|
|
13
|
+
* runner's write is rolled back by the guarded checkpoint advance before anyone
|
|
14
|
+
* sees it. A sink is not transactional. Two workers on one sink can both send
|
|
15
|
+
* before either moves the cursor, and an older read sent late overwrites a newer
|
|
16
|
+
* record, which no compare-and-set run AFTER the send can undo. So one worker per
|
|
17
|
+
* feed and sink holds a lease, and sends nothing without it (ADR-0019 §5).
|
|
18
|
+
*
|
|
19
|
+
* `acquireLease` is SCOPED. It either takes the lease or fails `LeaseHeld` at
|
|
20
|
+
* once — it never waits — and the lease is released when the scope closes. What
|
|
21
|
+
* it returns is a {@link FeedLease}: the holder id the store issued, and
|
|
22
|
+
* `ensureHeld`, which the driver runs before EVERY call that changes the sink —
|
|
23
|
+
* `apply`, and a resync's `begin`, `load` and `publish` — and which fails
|
|
24
|
+
* `LeaseLost` the moment the store can tell the lease is no longer this worker's.
|
|
25
|
+
* How the store keeps the lease alive, and how it tells, is its own business, and
|
|
26
|
+
* there are two shapes it may take:
|
|
27
|
+
*
|
|
28
|
+
* - **A lease row with an expiry**, renewed by a heartbeat the store forks into the
|
|
29
|
+
* lease's scope. `ensureHeld` fails once a renewal has found the row taken, and
|
|
30
|
+
* once the lease is within a SAFETY MARGIN of the expiry it last stored — never
|
|
31
|
+
* only at the expiry itself. This works through a transaction pooler and needs no
|
|
32
|
+
* reserved connection. Its cost is a window, and the margin is what bounds it:
|
|
33
|
+
* `ensureHeld` judges the lease when a call STARTS, and nothing stops a call
|
|
34
|
+
* already in flight when the lease lapses. A call that ends within the margin of
|
|
35
|
+
* the check that let it through lands while the lease is still this worker's,
|
|
36
|
+
* and cannot be overtaken by a new holder's, renewals failing meanwhile or not.
|
|
37
|
+
* A call that takes longer than the margin — a slow sink call, one retrying its
|
|
38
|
+
* hosts, or a process pause between the check and the call — can land after the
|
|
39
|
+
* lease has lapsed and after the new holder's own calls. That is ONE stale call,
|
|
40
|
+
* since the worker's next `ensureHeld` fails, and it can land as late as the
|
|
41
|
+
* call's duration less the margin past the lapse; no pause longer than the lease
|
|
42
|
+
* is needed, only a call longer than the margin while renewals are failing. In
|
|
43
|
+
* the tail that is one stale batch. In a resync it
|
|
44
|
+
* can be a `publish`, which swaps in the worker's whole target, complete as of
|
|
45
|
+
* its own read of the feed: if the new holder has already published and moved its
|
|
46
|
+
* cursor on, every record that changed in between is served at its older state —
|
|
47
|
+
* a key deleted since comes back, one created since is missing — until it changes
|
|
48
|
+
* again or the next resync. A reconcile removes the keys that came back, and
|
|
49
|
+
* repairs nothing else. It can be a `begin`, which drops the new holder's target
|
|
50
|
+
* and costs that holder a restarted resync. It is never a `publish` of another
|
|
51
|
+
* worker's target, because each `begin` names a target of its own
|
|
52
|
+
* (`runFeedSync.ts` owns why).
|
|
53
|
+
* - **A session lock on a dedicated connection.** `ensureHeld` checks the
|
|
54
|
+
* connection is still alive. A process pause opens no window of its own, because
|
|
55
|
+
* the lock survives it and goes only when the session goes. The window is the
|
|
56
|
+
* session dying while a call is in flight, or between the check and the call:
|
|
57
|
+
* that call is let through, and the new holder can take the lock the moment the
|
|
58
|
+
* session is gone. It is the same ONE stale call, at the same costs as above,
|
|
59
|
+
* since the next `ensureHeld` asks a session that is gone, and it can land as
|
|
60
|
+
* late as that call's whole duration after the session died — which, for a sink
|
|
61
|
+
* call that retries its hosts, can be long. The price of this shape is the
|
|
62
|
+
* connection.
|
|
63
|
+
*
|
|
64
|
+
* Both shapes record the holder where the cursor compare-and-set can see it, and
|
|
65
|
+
* `casCursor` FENCES on it: a call whose `holder` is not the current holder fails
|
|
66
|
+
* `CursorSuperseded` and lands nothing. That is what stops a worker that lost its
|
|
67
|
+
* lease from moving the cursor, whichever shape the lease takes. The in-memory
|
|
68
|
+
* feed takes a third, trivial shape: the holder recorded in its one state, taken
|
|
69
|
+
* only when no holder is recorded, in the same atomic step as that check, and
|
|
70
|
+
* cleared by the finaliser of the lease's scope.
|
|
71
|
+
*
|
|
72
|
+
* ## The cursor
|
|
73
|
+
*
|
|
74
|
+
* A cursor is a `(generation, seq)` pair; a sink with no cursor yet has none, and
|
|
75
|
+
* resyncs. `casCursor` is guarded like a checkpoint: it lands only when the stored
|
|
76
|
+
* cursor EQUALS `expected` (both absent, or both present with an equal generation
|
|
77
|
+
* and an equal seq) AND the lease is held by `holder`. It moves the parked list in
|
|
78
|
+
* the SAME atomic step, so a park recorded for a batch can never outlive a cursor
|
|
79
|
+
* advance that was refused, nor be lost from one that landed.
|
|
80
|
+
*
|
|
81
|
+
* An advance that lands also records WHEN it landed, as the sink's `lastAdvance`,
|
|
82
|
+
* in that same step; a refused one records nothing. The time is read from the
|
|
83
|
+
* Effect `Clock` of the process making the call (`DateTime.now`), never from the
|
|
84
|
+
* database's own clock, so `TestClock` governs it exactly as it governs the
|
|
85
|
+
* lease's expiry, and every backend reports one clock. Taking a lease is not an
|
|
86
|
+
* advance: a sink whose cursor has never moved has no `lastAdvance`.
|
|
87
|
+
*
|
|
88
|
+
* ## The parked list
|
|
89
|
+
*
|
|
90
|
+
* Parks are written only through `casCursor`, as a {@link ParkedChange}:
|
|
91
|
+
*
|
|
92
|
+
* - `Amend` upserts each entry in `park` (a key already parked takes the new seq,
|
|
93
|
+
* reason and time) and deletes each key in `unpark` that is parked. A key never
|
|
94
|
+
* appears in both. The driver unparks every key it sent successfully, which is
|
|
95
|
+
* how a parked record leaves the list once a later change of it syncs.
|
|
96
|
+
* - `Replace` discards the whole list for this feed and sink, then parks `park`.
|
|
97
|
+
* A resync uses it, because a resync re-sends every record and the parks it
|
|
98
|
+
* meets are the only ones still true; `park` holds a key only when the last
|
|
99
|
+
* change of it the resync sent was refused.
|
|
100
|
+
*
|
|
101
|
+
* `retryParked` deletes one park AND re-stamps the row's `seq` with a fresh value,
|
|
102
|
+
* in one atomic step, so the next pass of every sink on the feed sends it again;
|
|
103
|
+
* the other sinks receive a record they already hold, which an idempotent sink
|
|
104
|
+
* absorbs. A key with no row left is simply unparked.
|
|
105
|
+
*
|
|
106
|
+
* ## Tombstone collection
|
|
107
|
+
*
|
|
108
|
+
* `collectTombstones(feed)` deletes the tombstone rows whose `seq` is at or below
|
|
109
|
+
* the LOWEST cursor `seq` over EVERY sink registered on the feed — every sink the
|
|
110
|
+
* store holds a cursor or a lease for — and reports how many it deleted. The
|
|
111
|
+
* minimum is taken over every registered sink's cursor WHATEVER generation it was
|
|
112
|
+
* saved at. A registered sink whose cursor is ABSENT blocks collection entirely: it
|
|
113
|
+
* counts as below every tombstone, and nothing is deleted until it has saved one.
|
|
114
|
+
* No sink registered, nothing deleted.
|
|
115
|
+
*
|
|
116
|
+
* A cursor at an older generation needs no rule of its own, because the minimum
|
|
117
|
+
* already holds it below every tombstone. A reset empties the feed in the same
|
|
118
|
+
* atomic step as it moves the generation, and `seq` never goes back, so every row
|
|
119
|
+
* the feed holds after a reset carries a `seq` above every value the feed reached
|
|
120
|
+
* before it. And a cursor is saved at a generation only with a `seq` read while the
|
|
121
|
+
* feed was at that generation: the driver reads the generation AFTER reading the
|
|
122
|
+
* rows and before saving, so rows read after a reset are never saved under the
|
|
123
|
+
* generation the reset left. So a cursor at an older generation is below every row
|
|
124
|
+
* the feed now holds, tombstones included, and holds collection back until its
|
|
125
|
+
* sink saves a cursor at the current generation. An arm of its own could only
|
|
126
|
+
* restate that, and could never be seen to differ from it.
|
|
127
|
+
*
|
|
128
|
+
* Such a sink is resyncing, as is one with no cursor, and the obvious reading —
|
|
129
|
+
* that a resync starts from the origin and so needs no tombstone — is wrong. A
|
|
130
|
+
* resync reads the feed page by page while the feed keeps moving, and it saves its
|
|
131
|
+
* cursor at the last `seq` it READ, not at the head. A key it loaded on an early
|
|
132
|
+
* page can be deleted after its last page was read; another sink passes that
|
|
133
|
+
* tombstone; and were it collected then, the resync would publish the key and
|
|
134
|
+
* start its tail at a cursor below a delete that no longer exists, holding the
|
|
135
|
+
* record for good. So the rule counts every registered sink, and a resync holds
|
|
136
|
+
* collection back until its cursor is saved. The cost is that tombstones
|
|
137
|
+
* accumulate while any sink resyncs — or is registered and never run again, which
|
|
138
|
+
* holds collection back until the sink is removed (below). `lastAdvance` in a
|
|
139
|
+
* {@link FeedSyncDescription} is how such a sink shows.
|
|
140
|
+
*
|
|
141
|
+
* The store computes the lowest cursor itself, in the same step as the delete, so
|
|
142
|
+
* a cursor moving between the two cannot let a tombstone go that some sink still
|
|
143
|
+
* needs.
|
|
144
|
+
*
|
|
145
|
+
* ## Removing a sink
|
|
146
|
+
*
|
|
147
|
+
* `removeSink(id)` deregisters a sink the application has retired. In ONE atomic
|
|
148
|
+
* step it deletes everything the store holds for that feed and sink — its cursor,
|
|
149
|
+
* its `lastAdvance`, its lease record and every parked record — and succeeds with
|
|
150
|
+
* whether there was anything to delete. A registered sink holds tombstone
|
|
151
|
+
* collection back for as long as it is registered, so a retired one left
|
|
152
|
+
* registered holds it back for good; this is the way out, and the next
|
|
153
|
+
* `collectTombstones` takes its minimum over the sinks that remain.
|
|
154
|
+
*
|
|
155
|
+
* It is REFUSED with `LeaseHeld`, deleting nothing, while a LIVE worker holds the
|
|
156
|
+
* sink's lease. Deleting the cursor under a running worker would only make it
|
|
157
|
+
* resync from the origin and register the sink again: stop the worker, then
|
|
158
|
+
* remove the sink. Whether the lease is live is the lease's own test, made in the
|
|
159
|
+
* same atomic step as the delete, so no worker can take the lease between the
|
|
160
|
+
* test and the delete:
|
|
161
|
+
*
|
|
162
|
+
* - **A lease row with an expiry** is live while its expiry is later than the
|
|
163
|
+
* caller's `Clock` now. A lapsed row is not live, and is deleted with the rest.
|
|
164
|
+
* - **A session lock** is live while any session holds the lock, which the row
|
|
165
|
+
* cannot say. So the store tries the lock ITSELF, on the same key, for the length
|
|
166
|
+
* of the removing transaction (a transaction-scoped try-lock), and refuses when
|
|
167
|
+
* it cannot have it. A holder recorded on the row while the store CAN have the
|
|
168
|
+
* lock belongs to a session that has gone, and is deleted with the rest.
|
|
169
|
+
* - **The in-memory feed** refuses while a holder is recorded.
|
|
170
|
+
*
|
|
171
|
+
* Each test belongs to the kind of lease the HOLDER took, not to the kind the
|
|
172
|
+
* store was built to take: where one sink can be leased either way, by workers
|
|
173
|
+
* built differently, the store makes every test that applies, since a row alone
|
|
174
|
+
* cannot say which kind is holding it.
|
|
175
|
+
*
|
|
176
|
+
* Nothing about the fence changes. A worker that lost its lease without knowing it
|
|
177
|
+
* finds no holder recorded after a removal, so its `casCursor` fails
|
|
178
|
+
* `CursorSuperseded`, as it would against any other holder. And a worker started
|
|
179
|
+
* for the sink again simply registers it anew, with no cursor, and resyncs: a
|
|
180
|
+
* removal leaves no mark that would refuse it.
|
|
181
|
+
*
|
|
182
|
+
* ## Description
|
|
183
|
+
*
|
|
184
|
+
* `describe(feed)` is everything `inspectFeed` reports that lives in the store, in
|
|
185
|
+
* one read: the feed's head (its highest stored `seq`, `FEED_ORIGIN` when it holds
|
|
186
|
+
* no row) and, per sink with a cursor or a lease, the cursor, when it last moved
|
|
187
|
+
* (`lastAdvance`), the lease, the parked list and the number of rows still pending
|
|
188
|
+
* for it.
|
|
189
|
+
*
|
|
190
|
+
* ## Errors
|
|
191
|
+
*
|
|
192
|
+
* An infrastructure fault is `FeedUnreadable` on a read and `FeedWriteFailed` on a
|
|
193
|
+
* write, never a defect, so the driver's supervision can back off and try again.
|
|
194
|
+
* A `FeedSinkId` naming a feed the store does not serve is a wiring defect, and a
|
|
195
|
+
* store may die on it.
|
|
196
|
+
*/
|
|
197
|
+
import type { DateTime, Effect, Option, Scope } from 'effect';
|
|
198
|
+
import type { CursorSuperseded, FeedUnreadable, FeedWriteFailed, LeaseHeld, LeaseLost } from './errors.js';
|
|
199
|
+
import type { FeedKey, FeedName, FeedSeq, FeedSinkId, Generation, LeaseHolder, SinkName } from './identifiers.js';
|
|
200
|
+
/** How far one sink has been synced: the generation, and the last confirmed seq. */
|
|
201
|
+
export interface FeedCursor {
|
|
202
|
+
readonly generation: Generation;
|
|
203
|
+
readonly seq: FeedSeq;
|
|
204
|
+
}
|
|
205
|
+
/** Whether two optional cursors are the same cursor, as `casCursor` compares them. */
|
|
206
|
+
export declare const sameCursor: (left: Option.Option<FeedCursor>, right: Option.Option<FeedCursor>) => boolean;
|
|
207
|
+
/** A held lease, as the worker that holds it sees it. */
|
|
208
|
+
export interface FeedLease {
|
|
209
|
+
readonly holder: LeaseHolder;
|
|
210
|
+
/** Fails `LeaseLost` once the lease is no longer this holder's. Run before every send. */
|
|
211
|
+
readonly ensureHeld: Effect.Effect<void, LeaseLost | FeedUnreadable>;
|
|
212
|
+
}
|
|
213
|
+
/** One parked record: the key, the `seq` of the row the sink refused, and why. */
|
|
214
|
+
export interface ParkedRecord {
|
|
215
|
+
readonly key: FeedKey;
|
|
216
|
+
readonly seq: FeedSeq;
|
|
217
|
+
readonly reason: string;
|
|
218
|
+
readonly parkedAt: DateTime.Utc;
|
|
219
|
+
}
|
|
220
|
+
/** How a cursor advance moves the parked list; see the module doc. */
|
|
221
|
+
export type ParkedChange = {
|
|
222
|
+
readonly _tag: 'Amend';
|
|
223
|
+
readonly park: ReadonlyArray<ParkedRecord>;
|
|
224
|
+
readonly unpark: ReadonlyArray<FeedKey>;
|
|
225
|
+
} | {
|
|
226
|
+
readonly _tag: 'Replace';
|
|
227
|
+
readonly park: ReadonlyArray<ParkedRecord>;
|
|
228
|
+
};
|
|
229
|
+
/** The lease as a description reports it. `expiresAt` is `None` for a lease with no expiry. */
|
|
230
|
+
export interface LeaseDescription {
|
|
231
|
+
readonly holder: LeaseHolder;
|
|
232
|
+
readonly expiresAt: Option.Option<DateTime.Utc>;
|
|
233
|
+
}
|
|
234
|
+
/** One sink's state, as the store holds it. */
|
|
235
|
+
export interface SinkSyncState {
|
|
236
|
+
readonly sink: SinkName;
|
|
237
|
+
readonly cursor: Option.Option<FeedCursor>;
|
|
238
|
+
/**
|
|
239
|
+
* When the cursor last moved, by the `Clock` of the worker that moved it; `None`
|
|
240
|
+
* if it never has. A sink whose cursor stays still while the feed's head moves is
|
|
241
|
+
* a sink nothing runs, and it holds tombstone collection back until removed.
|
|
242
|
+
*/
|
|
243
|
+
readonly lastAdvance: Option.Option<DateTime.Utc>;
|
|
244
|
+
readonly lease: Option.Option<LeaseDescription>;
|
|
245
|
+
readonly parked: ReadonlyArray<ParkedRecord>;
|
|
246
|
+
/**
|
|
247
|
+
* The rows this sink has still to receive: those above its cursor's `seq`, or
|
|
248
|
+
* every live row when it has no cursor. Counted against the stored rows, so a
|
|
249
|
+
* cursor at an older generation counts as if it were current; `inspectFeed` is
|
|
250
|
+
* what says a resync is due.
|
|
251
|
+
*/
|
|
252
|
+
readonly pending: number;
|
|
253
|
+
}
|
|
254
|
+
/** Everything `describe` reports; see the module doc. */
|
|
255
|
+
export interface FeedSyncDescription {
|
|
256
|
+
readonly head: FeedSeq;
|
|
257
|
+
readonly sinks: ReadonlyArray<SinkSyncState>;
|
|
258
|
+
}
|
|
259
|
+
/** The port for a sync worker's durable state. See the module doc for every rule. */
|
|
260
|
+
export interface FeedSyncStore {
|
|
261
|
+
readonly acquireLease: (id: FeedSinkId) => Effect.Effect<FeedLease, LeaseHeld | FeedUnreadable | FeedWriteFailed, Scope.Scope>;
|
|
262
|
+
readonly readCursor: (id: FeedSinkId) => Effect.Effect<Option.Option<FeedCursor>, FeedUnreadable>;
|
|
263
|
+
readonly casCursor: (args: {
|
|
264
|
+
readonly feed: FeedName;
|
|
265
|
+
readonly sink: SinkName;
|
|
266
|
+
readonly holder: LeaseHolder;
|
|
267
|
+
readonly expected: Option.Option<FeedCursor>;
|
|
268
|
+
readonly next: FeedCursor;
|
|
269
|
+
readonly parked: ParkedChange;
|
|
270
|
+
}) => Effect.Effect<void, CursorSuperseded | FeedWriteFailed>;
|
|
271
|
+
readonly readParked: (id: FeedSinkId) => Effect.Effect<ReadonlyArray<ParkedRecord>, FeedUnreadable>;
|
|
272
|
+
readonly retryParked: (args: {
|
|
273
|
+
readonly feed: FeedName;
|
|
274
|
+
readonly sink: SinkName;
|
|
275
|
+
readonly key: FeedKey;
|
|
276
|
+
}) => Effect.Effect<boolean, FeedWriteFailed>;
|
|
277
|
+
readonly collectTombstones: (feed: FeedName) => Effect.Effect<number, FeedWriteFailed>;
|
|
278
|
+
readonly removeSink: (id: FeedSinkId) => Effect.Effect<boolean, LeaseHeld | FeedWriteFailed>;
|
|
279
|
+
readonly describe: (feed: FeedName) => Effect.Effect<FeedSyncDescription, FeedUnreadable>;
|
|
280
|
+
}
|
|
281
|
+
//# sourceMappingURL=FeedSyncStore.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FeedSyncStore.d.ts","sourceRoot":"","sources":["../../src/FeedSyncStore.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmMG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAA;AAC7D,OAAO,KAAK,EACV,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,SAAS,EACT,SAAS,EACV,MAAM,UAAU,CAAA;AACjB,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EACR,OAAO,EACP,UAAU,EACV,UAAU,EACV,WAAW,EACX,QAAQ,EACT,MAAM,eAAe,CAAA;AAEtB,oFAAoF;AACpF,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;CACtB;AAED,sFAAsF;AACtF,eAAO,MAAM,UAAU,GACrB,MAAM,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAC/B,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,KAC/B,OAKqC,CAAA;AAExC,yDAAyD;AACzD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,0FAA0F;IAC1F,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,GAAG,cAAc,CAAC,CAAA;CACrE;AAED,kFAAkF;AAClF,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAA;CAChC;AAED,sEAAsE;AACtE,MAAM,MAAM,YAAY,GACpB;IACE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,YAAY,CAAC,CAAA;IAC1C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;CACxC,GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,YAAY,CAAC,CAAA;CAAE,CAAA;AAE5E,+FAA+F;AAC/F,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;CAChD;AAED,+CAA+C;AAC/C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC1C;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAA;IAC/C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,YAAY,CAAC,CAAA;IAC5C;;;;;OAKG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CACzB;AAED,yDAAyD;AACzD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,aAAa,CAAC,CAAA;CAC7C;AAED,qFAAqF;AACrF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,YAAY,EAAE,CACrB,EAAE,EAAE,UAAU,KACX,MAAM,CAAC,MAAM,CAChB,SAAS,EACT,SAAS,GAAG,cAAc,GAAG,eAAe,EAC5C,KAAK,CAAC,KAAK,CACZ,CAAA;IACD,QAAQ,CAAC,UAAU,EAAE,CACnB,EAAE,EAAE,UAAU,KACX,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,CAAC,CAAA;IAC7D,QAAQ,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE;QACzB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;QACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;QACvB,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAA;QAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;QAC5C,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;QACzB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;KAC9B,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,GAAG,eAAe,CAAC,CAAA;IAC7D,QAAQ,CAAC,UAAU,EAAE,CACnB,EAAE,EAAE,UAAU,KACX,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE,cAAc,CAAC,CAAA;IAC/D,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE;QAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;QACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;QACvB,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;KACtB,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAC7C,QAAQ,CAAC,iBAAiB,EAAE,CAC1B,IAAI,EAAE,QAAQ,KACX,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAC3C,QAAQ,CAAC,UAAU,EAAE,CACnB,EAAE,EAAE,UAAU,KACX,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,GAAG,eAAe,CAAC,CAAA;IACxD,QAAQ,CAAC,QAAQ,EAAE,CACjB,IAAI,EAAE,QAAQ,KACX,MAAM,CAAC,MAAM,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAA;CACxD"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { FeedChange } from './FeedSink.js';
|
|
2
|
+
import type { ParkedRecord } from './FeedSyncStore.js';
|
|
3
|
+
import type { FeedKey, FeedSeq } from './identifiers.js';
|
|
4
|
+
/** One change to send, and the `seq` of the row it came from (for its park). */
|
|
5
|
+
export interface Pending<D> {
|
|
6
|
+
readonly change: FeedChange<D>;
|
|
7
|
+
readonly seq: FeedSeq;
|
|
8
|
+
}
|
|
9
|
+
/** What a delivered batch came to: the keys the sink took, and the parks. */
|
|
10
|
+
export interface Delivered {
|
|
11
|
+
readonly sent: ReadonlyArray<FeedKey>;
|
|
12
|
+
readonly parked: ReadonlyArray<ParkedRecord>;
|
|
13
|
+
}
|
|
14
|
+
/** The reason recorded for a refused key the sink gave no reason for. */
|
|
15
|
+
export declare const UNEXPLAINED_REFUSAL = "refused by the sink, with no reason given";
|
|
16
|
+
//# sourceMappingURL=deliver.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deliver.d.ts","sourceRoot":"","sources":["../../src/deliver.ts"],"names":[],"mappings":"AA+BA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,KAAK,EAAa,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAErD,gFAAgF;AAChF,MAAM,WAAW,OAAO,CAAC,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;CACtB;AAED,6EAA6E;AAC7E,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;IACrC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,YAAY,CAAC,CAAA;CAC7C;AAED,yEAAyE;AACzE,eAAO,MAAM,mBAAmB,8CAA8C,CAAA"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The change feed's error vocabulary.
|
|
3
|
+
*
|
|
4
|
+
* ## The sink's three answers
|
|
5
|
+
*
|
|
6
|
+
* A sink classifies every failure into one of three, and the driver's whole
|
|
7
|
+
* behaviour on failure follows from which. The names borrow the meaning of
|
|
8
|
+
* `@kairos-es/automation`'s classification without depending on that package: a
|
|
9
|
+
* feed is not an automation (ADR-0019), and a dependency for three class names
|
|
10
|
+
* would make it look like one.
|
|
11
|
+
*
|
|
12
|
+
* - `SinkRetryable` — the sink could not take the batch NOW (a rate limit, a 5xx, a
|
|
13
|
+
* timeout). The cursor stays where it is and the batch is sent again after a
|
|
14
|
+
* back-off, never sooner than `retryAfter` when the sink names one.
|
|
15
|
+
* - `SinkRejected` — the sink will never take THESE records as they stand (too
|
|
16
|
+
* large, invalid). The named keys are parked with the sink's reasons, the rest of
|
|
17
|
+
* the batch is sent again without them, and the cursor moves past the whole page.
|
|
18
|
+
* A parked record is retried when its row next changes.
|
|
19
|
+
* - `SinkHalted` — something is wrong that retrying will not mend (bad
|
|
20
|
+
* credentials, a missing index). The worker stops, and supervision decides.
|
|
21
|
+
*
|
|
22
|
+
* ## The feed's own tables
|
|
23
|
+
*
|
|
24
|
+
* `FeedWriteFailed` and `FeedUnreadable` are infrastructure faults on the feed's
|
|
25
|
+
* own durable state, split by direction: a write that did not land, a read that
|
|
26
|
+
* could not be made. They are tagged errors rather than defects for the reason the
|
|
27
|
+
* `ProjectionStore` port gives for its own: the driver's supervision has to tell
|
|
28
|
+
* "back off and try again" from "give up", and only a value can carry that.
|
|
29
|
+
*
|
|
30
|
+
* ## The worker's lease and cursor
|
|
31
|
+
*
|
|
32
|
+
* `LeaseHeld`, `LeaseLost` and `CursorSuperseded` are the three ways the
|
|
33
|
+
* single-worker guarantee shows itself: another worker holds the lease, this
|
|
34
|
+
* worker's lease went while it ran, or the cursor it expected to advance is not
|
|
35
|
+
* the one stored (a rival moved it, or the lease changed hands).
|
|
36
|
+
*/
|
|
37
|
+
import { Duration } from 'effect';
|
|
38
|
+
import type { NonEmptyReadonlyArray } from 'effect/Array';
|
|
39
|
+
import type { FeedKey, FeedName, FeedSinkId, SinkName } from './identifiers.js';
|
|
40
|
+
declare const SinkRetryable_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
41
|
+
readonly _tag: "SinkRetryable";
|
|
42
|
+
} & Readonly<A>;
|
|
43
|
+
/** The sink cannot take the batch now: keep the cursor and back off. */
|
|
44
|
+
export declare class SinkRetryable extends SinkRetryable_base<{
|
|
45
|
+
readonly reason: string;
|
|
46
|
+
/** The least the driver waits before sending again, when the sink knows it. */
|
|
47
|
+
readonly retryAfter?: Duration.Duration;
|
|
48
|
+
}> {
|
|
49
|
+
get message(): string;
|
|
50
|
+
}
|
|
51
|
+
declare const SinkRejected_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
52
|
+
readonly _tag: "SinkRejected";
|
|
53
|
+
} & Readonly<A>;
|
|
54
|
+
/** The sink refuses these records as they stand: park them, send the rest. */
|
|
55
|
+
export declare class SinkRejected extends SinkRejected_base<{
|
|
56
|
+
readonly keys: NonEmptyReadonlyArray<FeedKey>;
|
|
57
|
+
/** The sink's reason per refused key; a key absent here is parked unexplained. */
|
|
58
|
+
readonly reasons: ReadonlyMap<FeedKey, string>;
|
|
59
|
+
}> {
|
|
60
|
+
get message(): string;
|
|
61
|
+
}
|
|
62
|
+
declare const SinkHalted_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
63
|
+
readonly _tag: "SinkHalted";
|
|
64
|
+
} & Readonly<A>;
|
|
65
|
+
/** Something retrying will not mend: stop the worker. */
|
|
66
|
+
export declare class SinkHalted extends SinkHalted_base<{
|
|
67
|
+
readonly reason: string;
|
|
68
|
+
}> {
|
|
69
|
+
get message(): string;
|
|
70
|
+
}
|
|
71
|
+
declare const FeedWriteFailed_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
72
|
+
readonly _tag: "FeedWriteFailed";
|
|
73
|
+
} & Readonly<A>;
|
|
74
|
+
/** A write to the feed's own tables did not land. */
|
|
75
|
+
export declare class FeedWriteFailed extends FeedWriteFailed_base<{
|
|
76
|
+
readonly reason: string;
|
|
77
|
+
readonly cause?: unknown;
|
|
78
|
+
}> {
|
|
79
|
+
get message(): string;
|
|
80
|
+
}
|
|
81
|
+
declare const FeedUnreadable_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
82
|
+
readonly _tag: "FeedUnreadable";
|
|
83
|
+
} & Readonly<A>;
|
|
84
|
+
/** A read of the feed's own tables could not be made. */
|
|
85
|
+
export declare class FeedUnreadable extends FeedUnreadable_base<{
|
|
86
|
+
readonly reason: string;
|
|
87
|
+
readonly cause?: unknown;
|
|
88
|
+
}> {
|
|
89
|
+
get message(): string;
|
|
90
|
+
}
|
|
91
|
+
declare const LeaseHeld_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
92
|
+
readonly _tag: "LeaseHeld";
|
|
93
|
+
} & Readonly<A>;
|
|
94
|
+
/** Another worker holds the lease for this feed and sink. */
|
|
95
|
+
export declare class LeaseHeld extends LeaseHeld_base<FeedSinkId> {
|
|
96
|
+
get message(): string;
|
|
97
|
+
}
|
|
98
|
+
declare const LeaseLost_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
99
|
+
readonly _tag: "LeaseLost";
|
|
100
|
+
} & Readonly<A>;
|
|
101
|
+
/** This worker's lease is no longer its own; it must stop sending at once. */
|
|
102
|
+
export declare class LeaseLost extends LeaseLost_base<FeedSinkId> {
|
|
103
|
+
get message(): string;
|
|
104
|
+
}
|
|
105
|
+
declare const CursorSuperseded_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
106
|
+
readonly _tag: "CursorSuperseded";
|
|
107
|
+
} & Readonly<A>;
|
|
108
|
+
/**
|
|
109
|
+
* The cursor compare-and-set lost: the stored cursor is not the one expected, or
|
|
110
|
+
* the lease is not held by the worker that asked. Nothing the call carried landed.
|
|
111
|
+
*/
|
|
112
|
+
export declare class CursorSuperseded extends CursorSuperseded_base<FeedSinkId> {
|
|
113
|
+
get message(): string;
|
|
114
|
+
}
|
|
115
|
+
declare const FeedSyncStalled_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
|
|
116
|
+
readonly _tag: "FeedSyncStalled";
|
|
117
|
+
} & Readonly<A>;
|
|
118
|
+
/**
|
|
119
|
+
* The sync worker gave up: it restarted `restarts` times in a row without moving
|
|
120
|
+
* its cursor once. `reason` is the failure that ended the last attempt.
|
|
121
|
+
*/
|
|
122
|
+
export declare class FeedSyncStalled extends FeedSyncStalled_base<{
|
|
123
|
+
readonly feed: FeedName;
|
|
124
|
+
readonly sink: SinkName;
|
|
125
|
+
readonly restarts: number;
|
|
126
|
+
readonly reason: unknown;
|
|
127
|
+
}> {
|
|
128
|
+
get message(): string;
|
|
129
|
+
}
|
|
130
|
+
export {};
|
|
131
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAQ,QAAQ,EAAE,MAAM,QAAQ,CAAA;AACvC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;;;;AAE5E,wEAAwE;AACxE,qBAAa,aAAc,SAAQ,mBAAkC;IACnE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,+EAA+E;IAC/E,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAA;CACxC,CAAC;IACA,IAAa,OAAO,IAAI,MAAM,CAI7B;CACF;;;;AAED,8EAA8E;AAC9E,qBAAa,YAAa,SAAQ,kBAAiC;IACjE,QAAQ,CAAC,IAAI,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAA;IAC7C,kFAAkF;IAClF,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;CAC/C,CAAC;IACA,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF;;;;AAED,yDAAyD;AACzD,qBAAa,UAAW,SAAQ,gBAA+B;IAC7D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CACxB,CAAC;IACA,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF;;;;AAED,qDAAqD;AACrD,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CACzB,CAAC;IACA,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF;;;;AAED,yDAAyD;AACzD,qBAAa,cAAe,SAAQ,oBAAmC;IACrE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CACzB,CAAC;IACA,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF;;;;AAED,6DAA6D;AAC7D,qBAAa,SAAU,SAAQ,eAA8B,UAAU,CAAC;IACtE,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF;;;;AAED,8EAA8E;AAC9E,qBAAa,SAAU,SAAQ,eAA8B,UAAU,CAAC;IACtE,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF;;;;AAED;;;GAGG;AACH,qBAAa,gBAAiB,SAAQ,sBAEpC,UAAU,CAAC;IACX,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF;;;;AAED;;;GAGG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CACzB,CAAC;IACA,IAAa,OAAO,IAAI,MAAM,CAE7B;CACF"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The change feed's branded identifiers.
|
|
3
|
+
*
|
|
4
|
+
* Every one of them is a branded `Schema`, as every domain identifier in this
|
|
5
|
+
* repository is, so a raw string or `bigint` cannot stand in for one by accident.
|
|
6
|
+
* Two of them are worth a word, because they look like things the read side
|
|
7
|
+
* already has and are deliberately not those things.
|
|
8
|
+
*
|
|
9
|
+
* - `FeedSeq` is NOT a `Position`. A `Position` is a place in the global event log,
|
|
10
|
+
* and one event may fan out into many feed rows (a teacher renamed, fifty courses
|
|
11
|
+
* touched). Give those fifty rows the event's position as their cursor and they
|
|
12
|
+
* all share one value, so a page boundary that falls among them loses the rest of
|
|
13
|
+
* them: the next page asks for rows strictly after a value the lost ones also
|
|
14
|
+
* carry. A feed-local sequence gives every row a cursor value of its own. The
|
|
15
|
+
* event position is still kept on each row, as `eventPosition`, for the one
|
|
16
|
+
* question it answers — "is the sink at least as fresh as P?" — and is never a
|
|
17
|
+
* cursor.
|
|
18
|
+
* - `Generation` is NOT a counter a caller reads meaning into. It is compared for
|
|
19
|
+
* EQUALITY only: a sink whose cursor was saved at a generation other than the
|
|
20
|
+
* feed's current one resyncs. That is what lets the in-memory feed draw a fresh
|
|
21
|
+
* generation from `Random` on every boot while a durable feed counts upwards, and
|
|
22
|
+
* both satisfy the one rule the driver needs.
|
|
23
|
+
*/
|
|
24
|
+
import { Schema } from 'effect';
|
|
25
|
+
/**
|
|
26
|
+
* The name of one change feed. A feed is bound to ONE materialisation, because a
|
|
27
|
+
* feed sequence taken inside a commit is ascending in commit order only while one
|
|
28
|
+
* transaction at a time writes that feed — which the read side gives per
|
|
29
|
+
* materialisation and no wider.
|
|
30
|
+
*/
|
|
31
|
+
export declare const FeedName: Schema.brand<Schema.String, "FeedName">;
|
|
32
|
+
export type FeedName = typeof FeedName.Type;
|
|
33
|
+
/** A record's key within a feed, and the sink's object id for that record. */
|
|
34
|
+
export declare const FeedKey: Schema.brand<Schema.String, "FeedKey">;
|
|
35
|
+
export type FeedKey = typeof FeedKey.Type;
|
|
36
|
+
/**
|
|
37
|
+
* The feed-local cursor: unique, and ascending in commit order. Compared, never
|
|
38
|
+
* arithmetic'd, and never assumed contiguous — a sequence shared by several feeds
|
|
39
|
+
* leaves gaps in each, and a coalesced row gives up its old value when it moves.
|
|
40
|
+
*/
|
|
41
|
+
export declare const FeedSeq: Schema.brand<Schema.BigInt, "FeedSeq">;
|
|
42
|
+
export type FeedSeq = typeof FeedSeq.Type;
|
|
43
|
+
/** The sentinel feed sequence meaning "before the first row". */
|
|
44
|
+
export declare const FEED_ORIGIN: FeedSeq;
|
|
45
|
+
/**
|
|
46
|
+
* The feed's generation: it moves on every reset of the materialisation the feed
|
|
47
|
+
* is bound to, and on every boot of an ephemeral feed. Compared for equality only;
|
|
48
|
+
* see the module doc.
|
|
49
|
+
*/
|
|
50
|
+
export declare const Generation: Schema.brand<Schema.BigInt, "Generation">;
|
|
51
|
+
export type Generation = typeof Generation.Type;
|
|
52
|
+
/** The name of one sink. A feed keeps one cursor per sink. */
|
|
53
|
+
export declare const SinkName: Schema.brand<Schema.String, "SinkName">;
|
|
54
|
+
export type SinkName = typeof SinkName.Type;
|
|
55
|
+
/**
|
|
56
|
+
* Who holds a sync worker's lease. Issued by the `FeedSyncStore` when the lease is
|
|
57
|
+
* taken, and checked again by every cursor compare-and-set, so a worker that has
|
|
58
|
+
* lost its lease cannot move the cursor even if it is still running.
|
|
59
|
+
*/
|
|
60
|
+
export declare const LeaseHolder: Schema.brand<Schema.String, "LeaseHolder">;
|
|
61
|
+
export type LeaseHolder = typeof LeaseHolder.Type;
|
|
62
|
+
/** One sync worker's identity: the feed it reads and the sink it writes. */
|
|
63
|
+
export interface FeedSinkId {
|
|
64
|
+
readonly feed: FeedName;
|
|
65
|
+
readonly sink: SinkName;
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=identifiers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"identifiers.d.ts","sourceRoot":"","sources":["../../src/identifiers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAE/B;;;;;GAKG;AACH,eAAO,MAAM,QAAQ,yCAGpB,CAAA;AACD,MAAM,MAAM,QAAQ,GAAG,OAAO,QAAQ,CAAC,IAAI,CAAA;AAE3C,8EAA8E;AAC9E,eAAO,MAAM,OAAO,wCAGnB,CAAA;AACD,MAAM,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC,IAAI,CAAA;AAEzC;;;;GAIG;AACH,eAAO,MAAM,OAAO,wCAGnB,CAAA;AACD,MAAM,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC,IAAI,CAAA;AAEzC,iEAAiE;AACjE,eAAO,MAAM,WAAW,EAAE,OAA0B,CAAA;AAEpD;;;;GAIG;AACH,eAAO,MAAM,UAAU,2CAGtB,CAAA;AACD,MAAM,MAAM,UAAU,GAAG,OAAO,UAAU,CAAC,IAAI,CAAA;AAE/C,8DAA8D;AAC9D,eAAO,MAAM,QAAQ,yCAGpB,CAAA;AACD,MAAM,MAAM,QAAQ,GAAG,OAAO,QAAQ,CAAC,IAAI,CAAA;AAE3C;;;;GAIG;AACH,eAAO,MAAM,WAAW,4CAGvB,CAAA;AACD,MAAM,MAAM,WAAW,GAAG,OAAO,WAAW,CAAC,IAAI,CAAA;AAEjD,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;CACxB"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { CheckpointKey, ProjectionStore } from '@kairos-es/read';
|
|
2
|
+
import { Effect, Schema } from 'effect';
|
|
3
|
+
import type { BatchedFeedWriter, FeedSource } from './Feed.js';
|
|
4
|
+
import { type FeedSyncStore } from './FeedSyncStore.js';
|
|
5
|
+
import { type FeedName } from './identifiers.js';
|
|
6
|
+
/** The in-memory feed; see the module doc. */
|
|
7
|
+
export interface InMemoryFeed<D> extends BatchedFeedWriter<D> {
|
|
8
|
+
readonly source: FeedSource<D>;
|
|
9
|
+
readonly syncStore: FeedSyncStore;
|
|
10
|
+
}
|
|
11
|
+
/** Options for {@link makeInMemoryFeed}. */
|
|
12
|
+
export interface InMemoryFeedOptions<D, I> {
|
|
13
|
+
readonly feed: FeedName;
|
|
14
|
+
/** The document's schema: what an upsert encodes, and compares, through. */
|
|
15
|
+
readonly doc: Schema.Codec<D, I>;
|
|
16
|
+
}
|
|
17
|
+
/** Build an in-memory feed at a fresh generation; see the module doc. */
|
|
18
|
+
export declare const makeInMemoryFeed: <D, I>(options: InMemoryFeedOptions<D, I>) => Effect.Effect<InMemoryFeed<D>>;
|
|
19
|
+
/**
|
|
20
|
+
* `store`, with `resetCheckpoint` for `key` replaced by one that also empties
|
|
21
|
+
* `feed` and moves its generation on, so every sink resyncs once the read model is
|
|
22
|
+
* replayed. Every other key passes through untouched, and `readCheckpoint` and
|
|
23
|
+
* `commit` are the store's own, so the result works through `forKey` exactly as
|
|
24
|
+
* the store does.
|
|
25
|
+
*
|
|
26
|
+
* Binding is checked here, at construction: a feed serves ONE materialisation,
|
|
27
|
+
* because one writer is what keeps its sequence ascending in commit order, so a
|
|
28
|
+
* feed already bound to another checkpoint key is refused, as is a durable `store`
|
|
29
|
+
* (see the module doc). Both refusals are defects.
|
|
30
|
+
*/
|
|
31
|
+
export declare const withInMemoryFeed: <D>(store: ProjectionStore, feed: InMemoryFeed<D>, key: CheckpointKey) => Effect.Effect<ProjectionStore>;
|
|
32
|
+
//# sourceMappingURL=inMemoryFeed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inMemoryFeed.d.ts","sourceRoot":"","sources":["../../src/inMemoryFeed.ts"],"names":[],"mappings":"AA8CA,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACrE,OAAO,EAEL,MAAM,EAKN,MAAM,EAEP,MAAM,QAAQ,CAAA;AAEf,OAAO,KAAK,EAAE,iBAAiB,EAAW,UAAU,EAAc,MAAM,QAAQ,CAAA;AAChF,OAAO,EAIL,KAAK,aAAa,EAKnB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAGL,KAAK,QAAQ,EAMd,MAAM,eAAe,CAAA;AAEtB,8CAA8C;AAC9C,MAAM,WAAW,YAAY,CAAC,CAAC,CAAE,SAAQ,iBAAiB,CAAC,CAAC,CAAC;IAC3D,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAA;CAClC;AAsDD,4CAA4C;AAC5C,MAAM,WAAW,mBAAmB,CAAC,CAAC,EAAE,CAAC;IACvC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,4EAA4E;IAC5E,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;CACjC;AAED,yEAAyE;AACzE,eAAO,MAAM,gBAAgB,GAAI,CAAC,EAAE,CAAC,EACnC,SAAS,mBAAmB,CAAC,CAAC,EAAE,CAAC,CAAC,KACjC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CA0W5B,CAAA;AAwBJ;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,gBAAgB,GAAI,CAAC,EAChC,OAAO,eAAe,EACtB,MAAM,YAAY,CAAC,CAAC,CAAC,EACrB,KAAK,aAAa,KACjB,MAAM,CAAC,MAAM,CAAC,eAAe,CA+B5B,CAAA"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The in-memory sink: a `FeedSink` over a `Ref` of documents, with an optional
|
|
3
|
+
* `FeedResync` and `FeedReconcile`, and the ORACLE the sink law suite holds every
|
|
4
|
+
* other sink against.
|
|
5
|
+
*
|
|
6
|
+
* It keeps the laws by construction: an upsert replaces the whole document, a
|
|
7
|
+
* delete of an absent key is a no-op, so the same batch applied twice leaves it as
|
|
8
|
+
* once; a batch is visible the moment `apply` returns, so `confirm` has nothing to
|
|
9
|
+
* wait for; a resync target is a separate map that `publish` swaps in whole, so a
|
|
10
|
+
* published resync holds exactly what was loaded into it, and a load into a target
|
|
11
|
+
* a later `begin` dropped fails rather than recreating it. Each `begin` names its
|
|
12
|
+
* target by a counter the sink's own state holds, so two workers sharing one sink
|
|
13
|
+
* value — the way two processes share one real index — never share a target, and
|
|
14
|
+
* the name is the same on every run of a test.
|
|
15
|
+
*
|
|
16
|
+
* Its two optional parts are switched on by option rather than always present,
|
|
17
|
+
* because which parts a sink has CHANGES what the driver does — a sink with no
|
|
18
|
+
* `resync` is resynced in place — and a test of either path needs a sink that
|
|
19
|
+
* lacks the other. It also records every batch it is handed, live or into a
|
|
20
|
+
* target, so a case can say how many sends a change cost.
|
|
21
|
+
*
|
|
22
|
+
* It lives for as long as the value is held, independently of any feed, which is
|
|
23
|
+
* what lets a test keep one sink across two in-memory feeds and watch the second
|
|
24
|
+
* feed's fresh generation force a resync.
|
|
25
|
+
*/
|
|
26
|
+
import { Effect } from 'effect';
|
|
27
|
+
import type { NonEmptyReadonlyArray } from 'effect/Array';
|
|
28
|
+
import type { FeedChange, FeedSink } from './FeedSink.js';
|
|
29
|
+
import type { FeedKey, Generation, SinkName } from './identifiers.js';
|
|
30
|
+
/** What the in-memory sink's `apply` and `load` return: where, and which batch. */
|
|
31
|
+
export interface InMemoryReceipt {
|
|
32
|
+
/** `'live'`, or the name of the resync target the batch was loaded into. */
|
|
33
|
+
readonly target: string;
|
|
34
|
+
readonly batch: number;
|
|
35
|
+
}
|
|
36
|
+
/** One batch the sink was handed, and where it went. */
|
|
37
|
+
export interface InMemorySentBatch<D> {
|
|
38
|
+
readonly target: string;
|
|
39
|
+
readonly changes: NonEmptyReadonlyArray<FeedChange<D>>;
|
|
40
|
+
}
|
|
41
|
+
/** Options for {@link makeInMemorySink}. */
|
|
42
|
+
export interface InMemorySinkOptions {
|
|
43
|
+
readonly name: SinkName;
|
|
44
|
+
/** Give the sink a `FeedResync` (an atomic swap). */
|
|
45
|
+
readonly resync?: boolean;
|
|
46
|
+
/** Give the sink a `FeedReconcile` (a key listing). */
|
|
47
|
+
readonly reconcile?: boolean;
|
|
48
|
+
}
|
|
49
|
+
/** The in-memory sink and what a test reads back from it. */
|
|
50
|
+
export interface InMemorySink<D> {
|
|
51
|
+
readonly sink: FeedSink<D, InMemoryReceipt>;
|
|
52
|
+
/** The live documents, by key. */
|
|
53
|
+
readonly documents: Effect.Effect<ReadonlyMap<FeedKey, D>>;
|
|
54
|
+
/** Every batch handed to `apply` or `load`, in order. */
|
|
55
|
+
readonly batches: Effect.Effect<ReadonlyArray<InMemorySentBatch<D>>>;
|
|
56
|
+
/** The generation of every resync published, in order. */
|
|
57
|
+
readonly published: Effect.Effect<ReadonlyArray<Generation>>;
|
|
58
|
+
}
|
|
59
|
+
/** Build an in-memory sink; see the module doc. */
|
|
60
|
+
export declare const makeInMemorySink: <D>(options: InMemorySinkOptions) => Effect.Effect<InMemorySink<D>>;
|
|
61
|
+
//# sourceMappingURL=inMemorySink.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inMemorySink.d.ts","sourceRoot":"","sources":["../../src/inMemorySink.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,MAAM,EAAe,MAAM,QAAQ,CAAA;AAC5C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AAEzD,OAAO,KAAK,EAAE,UAAU,EAAc,QAAQ,EAAgB,MAAM,YAAY,CAAA;AAChF,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAElE,mFAAmF;AACnF,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CACvB;AAED,wDAAwD;AACxD,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,OAAO,EAAE,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;CACvD;AAED,4CAA4C;AAC5C,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,qDAAqD;IACrD,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;IACzB,uDAAuD;IACvD,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAC7B;AAED,6DAA6D;AAC7D,MAAM,WAAW,YAAY,CAAC,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,eAAe,CAAC,CAAA;IAC3C,kCAAkC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAA;IAC1D,yDAAyD;IACzD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACpE,0DAA0D;IAC1D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAA;CAC7D;AAsCD,mDAAmD;AACnD,eAAO,MAAM,gBAAgB,GAAI,CAAC,EAChC,SAAS,mBAAmB,KAC3B,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CA+G5B,CAAA"}
|