@kairos-es/feed 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +28 -0
  2. package/README.md +109 -2
  3. package/dist/cjs/Feed.js +6 -0
  4. package/dist/cjs/Feed.js.map +1 -0
  5. package/dist/cjs/FeedSink.js +54 -0
  6. package/dist/cjs/FeedSink.js.map +1 -0
  7. package/dist/cjs/FeedSyncStore.js +10 -0
  8. package/dist/cjs/FeedSyncStore.js.map +1 -0
  9. package/dist/cjs/deliver.js +86 -0
  10. package/dist/cjs/deliver.js.map +1 -0
  11. package/dist/cjs/errors.js +114 -0
  12. package/dist/cjs/errors.js.map +1 -0
  13. package/dist/cjs/identifiers.js +63 -0
  14. package/dist/cjs/identifiers.js.map +1 -0
  15. package/dist/cjs/inMemoryFeed.js +328 -0
  16. package/dist/cjs/inMemoryFeed.js.map +1 -0
  17. package/dist/cjs/inMemorySink.js +142 -0
  18. package/dist/cjs/inMemorySink.js.map +1 -0
  19. package/dist/cjs/index.js +184 -0
  20. package/dist/cjs/index.js.map +1 -0
  21. package/dist/cjs/inspect.js +68 -0
  22. package/dist/cjs/inspect.js.map +1 -0
  23. package/dist/cjs/readiness.js +100 -0
  24. package/dist/cjs/readiness.js.map +1 -0
  25. package/dist/cjs/reconcile.js +164 -0
  26. package/dist/cjs/reconcile.js.map +1 -0
  27. package/dist/cjs/runFeedSync.js +603 -0
  28. package/dist/cjs/runFeedSync.js.map +1 -0
  29. package/dist/cjs/tuning.js +60 -0
  30. package/dist/cjs/tuning.js.map +1 -0
  31. package/dist/dts-cjs/Feed.d.ts +99 -0
  32. package/dist/dts-cjs/Feed.d.ts.map +1 -0
  33. package/dist/dts-cjs/FeedSink.d.ts +189 -0
  34. package/dist/dts-cjs/FeedSink.d.ts.map +1 -0
  35. package/dist/dts-cjs/FeedSyncStore.d.ts +281 -0
  36. package/dist/dts-cjs/FeedSyncStore.d.ts.map +1 -0
  37. package/dist/dts-cjs/deliver.d.ts +16 -0
  38. package/dist/dts-cjs/deliver.d.ts.map +1 -0
  39. package/dist/dts-cjs/errors.d.ts +131 -0
  40. package/dist/dts-cjs/errors.d.ts.map +1 -0
  41. package/dist/dts-cjs/identifiers.d.ts +67 -0
  42. package/dist/dts-cjs/identifiers.d.ts.map +1 -0
  43. package/dist/dts-cjs/inMemoryFeed.d.ts +32 -0
  44. package/dist/dts-cjs/inMemoryFeed.d.ts.map +1 -0
  45. package/dist/dts-cjs/inMemorySink.d.ts +61 -0
  46. package/dist/dts-cjs/inMemorySink.d.ts.map +1 -0
  47. package/dist/dts-cjs/index.d.ts +63 -0
  48. package/dist/dts-cjs/index.d.ts.map +1 -0
  49. package/dist/dts-cjs/inspect.d.ts +71 -0
  50. package/dist/dts-cjs/inspect.d.ts.map +1 -0
  51. package/dist/dts-cjs/package.json +3 -0
  52. package/dist/dts-cjs/readiness.d.ts +84 -0
  53. package/dist/dts-cjs/readiness.d.ts.map +1 -0
  54. package/dist/dts-cjs/reconcile.d.ts +68 -0
  55. package/dist/dts-cjs/reconcile.d.ts.map +1 -0
  56. package/dist/dts-cjs/runFeedSync.d.ts +220 -0
  57. package/dist/dts-cjs/runFeedSync.d.ts.map +1 -0
  58. package/dist/dts-cjs/tuning.d.ts +40 -0
  59. package/dist/dts-cjs/tuning.d.ts.map +1 -0
  60. package/dist/dts-esm/Feed.d.ts +99 -0
  61. package/dist/dts-esm/Feed.d.ts.map +1 -0
  62. package/dist/dts-esm/FeedSink.d.ts +189 -0
  63. package/dist/dts-esm/FeedSink.d.ts.map +1 -0
  64. package/dist/dts-esm/FeedSyncStore.d.ts +281 -0
  65. package/dist/dts-esm/FeedSyncStore.d.ts.map +1 -0
  66. package/dist/dts-esm/deliver.d.ts +16 -0
  67. package/dist/dts-esm/deliver.d.ts.map +1 -0
  68. package/dist/dts-esm/errors.d.ts +131 -0
  69. package/dist/dts-esm/errors.d.ts.map +1 -0
  70. package/dist/dts-esm/identifiers.d.ts +67 -0
  71. package/dist/dts-esm/identifiers.d.ts.map +1 -0
  72. package/dist/dts-esm/inMemoryFeed.d.ts +32 -0
  73. package/dist/dts-esm/inMemoryFeed.d.ts.map +1 -0
  74. package/dist/dts-esm/inMemorySink.d.ts +61 -0
  75. package/dist/dts-esm/inMemorySink.d.ts.map +1 -0
  76. package/dist/dts-esm/index.d.ts +63 -0
  77. package/dist/dts-esm/index.d.ts.map +1 -0
  78. package/dist/dts-esm/inspect.d.ts +71 -0
  79. package/dist/dts-esm/inspect.d.ts.map +1 -0
  80. package/dist/dts-esm/package.json +3 -0
  81. package/dist/dts-esm/readiness.d.ts +84 -0
  82. package/dist/dts-esm/readiness.d.ts.map +1 -0
  83. package/dist/dts-esm/reconcile.d.ts +68 -0
  84. package/dist/dts-esm/reconcile.d.ts.map +1 -0
  85. package/dist/dts-esm/runFeedSync.d.ts +220 -0
  86. package/dist/dts-esm/runFeedSync.d.ts.map +1 -0
  87. package/dist/dts-esm/tuning.d.ts +40 -0
  88. package/dist/dts-esm/tuning.d.ts.map +1 -0
  89. package/dist/esm/Feed.js +2 -0
  90. package/dist/esm/Feed.js.map +1 -0
  91. package/dist/esm/FeedSink.js +46 -0
  92. package/dist/esm/FeedSink.js.map +1 -0
  93. package/dist/esm/FeedSyncStore.js +3 -0
  94. package/dist/esm/FeedSyncStore.js.map +1 -0
  95. package/dist/esm/deliver.js +78 -0
  96. package/dist/esm/deliver.js.map +1 -0
  97. package/dist/esm/errors.js +98 -0
  98. package/dist/esm/errors.js.map +1 -0
  99. package/dist/esm/identifiers.js +56 -0
  100. package/dist/esm/identifiers.js.map +1 -0
  101. package/dist/esm/inMemoryFeed.js +320 -0
  102. package/dist/esm/inMemoryFeed.js.map +1 -0
  103. package/dist/esm/inMemorySink.js +134 -0
  104. package/dist/esm/inMemorySink.js.map +1 -0
  105. package/dist/esm/index.js +62 -0
  106. package/dist/esm/index.js.map +1 -0
  107. package/dist/esm/inspect.js +58 -0
  108. package/dist/esm/inspect.js.map +1 -0
  109. package/dist/esm/package.json +4 -0
  110. package/dist/esm/readiness.js +92 -0
  111. package/dist/esm/readiness.js.map +1 -0
  112. package/dist/esm/reconcile.js +153 -0
  113. package/dist/esm/reconcile.js.map +1 -0
  114. package/dist/esm/runFeedSync.js +595 -0
  115. package/dist/esm/runFeedSync.js.map +1 -0
  116. package/dist/esm/tuning.js +51 -0
  117. package/dist/esm/tuning.js.map +1 -0
  118. package/package.json +34 -3
  119. package/src/Feed.ts +111 -0
  120. package/src/FeedSink.ts +130 -0
  121. package/src/FeedSyncStore.ts +326 -0
  122. package/src/deliver.ts +108 -0
  123. package/src/errors.ts +134 -0
  124. package/src/identifiers.ts +92 -0
  125. package/src/inMemoryFeed.ts +582 -0
  126. package/src/inMemorySink.ts +215 -0
  127. package/src/index.ts +136 -0
  128. package/src/inspect.ts +114 -0
  129. package/src/readiness.ts +137 -0
  130. package/src/reconcile.ts +224 -0
  131. package/src/runFeedSync.ts +882 -0
  132. package/src/tuning.ts +65 -0
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Reconcile: delete what a sink holds that the feed no longer has.
3
+ *
4
+ * ## Why a feed needs it at all
5
+ *
6
+ * The steady state deletes through tombstones, and a tombstone lives only until
7
+ * every sink registered on the feed has passed it. A record can still be left
8
+ * behind in a sink: a sink restored from a backup, a key written into the index by
9
+ * hand, or a record deleted before a reset, whose tombstone the reset cleared with
10
+ * every other row, followed by an in-place resync that, having no atomic swap,
11
+ * sends only the rows the new generation holds and so never deletes it. Reconcile is the one operation that compares the sink with the feed
12
+ * rather than replaying the feed into the sink, and so the only one that finds
13
+ * those orphans.
14
+ *
15
+ * ## The order of the two reads, and why it is safe
16
+ *
17
+ * The sink's keys are listed FIRST and the feed's live keys read SECOND. A key the
18
+ * sink lists that has a live row by the time the feed is read is kept, so a record
19
+ * written while reconcile runs is never deleted by it. A key whose row turns into
20
+ * a tombstone after the feed is read is kept too, and its tombstone deletes it on
21
+ * the next pass. And a key whose row comes back to life after the feed is read is
22
+ * deleted here and re-sent by the next pass, because its new `seq` is above the
23
+ * cursor. Reconcile runs inside the worker that holds the lease, between batches,
24
+ * so nothing else is sending to the sink while it does (ADR-0019 §8).
25
+ *
26
+ * ## What it costs
27
+ *
28
+ * It holds every key the sink lists and every live key of the feed in memory at
29
+ * once, and reads every live document to find the keys. It is a periodic repair,
30
+ * not a steady-state path: `reconcileEvery` should be measured in hours.
31
+ */
32
+ import { Effect } from 'effect';
33
+ import type { FeedUnreadable, FeedWriteFailed, LeaseHeld, LeaseLost, SinkHalted, SinkRetryable } from './errors.js';
34
+ import type { FeedSource } from './Feed.js';
35
+ import { type FeedReconcile, type FeedSink } from './FeedSink.js';
36
+ import type { FeedSyncStore } from './FeedSyncStore.js';
37
+ import { type FeedKey } from './identifiers.js';
38
+ /** What one reconcile did. */
39
+ export interface ReconcileReport {
40
+ /** How many keys the sink listed. */
41
+ readonly listed: number;
42
+ /** The orphans deleted: listed by the sink, with no live row in the feed. */
43
+ readonly deleted: ReadonlyArray<FeedKey>;
44
+ /** Orphans whose delete the sink refused, with its reason. They stay in the sink. */
45
+ readonly refused: ReadonlyArray<{
46
+ readonly key: FeedKey;
47
+ readonly reason: string;
48
+ }>;
49
+ }
50
+ /** Every key of `source` whose row is live, read page by page from the origin. */
51
+ export declare const liveKeys: <D>(source: FeedSource<D>, pageSize: number) => Effect.Effect<ReadonlySet<FeedKey>, FeedUnreadable>;
52
+ /** Options for {@link reconcileFeed}. */
53
+ export interface ReconcileFeedOptions<D, Receipt> {
54
+ readonly source: FeedSource<D>;
55
+ readonly sink: FeedSink<D, Receipt> & {
56
+ readonly reconcile: FeedReconcile;
57
+ };
58
+ readonly store: FeedSyncStore;
59
+ /** How many rows to read, and deletes to send, at a time. */
60
+ readonly pageSize?: number;
61
+ }
62
+ /**
63
+ * Reconcile `sink` against `source` once, now. It takes the worker lease itself,
64
+ * so it fails `LeaseHeld` while a sync worker for the same feed and sink runs —
65
+ * reconcile through that worker's `reconcileEvery` instead, or stop it first.
66
+ */
67
+ export declare const reconcileFeed: <D, Receipt>(options: ReconcileFeedOptions<D, Receipt>) => Effect.Effect<ReconcileReport, LeaseHeld | LeaseLost | FeedUnreadable | FeedWriteFailed | SinkRetryable | SinkHalted>;
68
+ //# sourceMappingURL=reconcile.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../../src/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,OAAO,EAAgB,MAAM,EAAkB,MAAM,QAAQ,CAAA;AAG7D,OAAO,KAAK,EACV,cAAc,EACd,eAAe,EACf,SAAS,EACT,SAAS,EACT,UAAU,EACV,aAAa,EACd,MAAM,UAAU,CAAA;AACjB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,EAAc,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,CAAA;AAC1E,OAAO,KAAK,EAAa,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/D,OAAO,EAAe,KAAK,OAAO,EAAmB,MAAM,eAAe,CAAA;AAG1E,8BAA8B;AAC9B,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;IACxC,qFAAqF;IACrF,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;QACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KACxB,CAAC,CAAA;CACH;AAED,kFAAkF;AAClF,eAAO,MAAM,QAAQ,GAAI,CAAC,EACxB,QAAQ,UAAU,CAAC,CAAC,CAAC,EACrB,UAAU,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,cAAc,CAoBjD,CAAA;AAmGJ,yCAAyC;AACzC,MAAM,WAAW,oBAAoB,CAAC,CAAC,EAAE,OAAO;IAC9C,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG;QAAE,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAA;KAAE,CAAA;IAC3E,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,GAAI,CAAC,EAAE,OAAO,EACtC,SAAS,oBAAoB,CAAC,CAAC,EAAE,OAAO,CAAC,KACxC,MAAM,CAAC,MAAM,CACd,eAAe,EACb,SAAS,GACT,SAAS,GACT,cAAc,GACd,eAAe,GACf,aAAa,GACb,UAAU,CAgBX,CAAA"}
@@ -0,0 +1,220 @@
1
+ /**
2
+ * `runFeedSync` — the sync worker: one per feed and sink, holding the lease,
3
+ * copying the feed's rows to the sink and moving its cursor only once the sink
4
+ * confirms them (ADR-0019).
5
+ *
6
+ * ## The lifecycle
7
+ *
8
+ * A worker runs in SESSIONS. A session takes the lease, then loops:
9
+ *
10
+ * 1. Read the cursor and the feed's generation. No cursor, or a cursor at another
11
+ * generation, means the sink holds a copy of something the feed no longer is,
12
+ * so the session RESYNCS (below) and loops.
13
+ * 2. Otherwise it TAILS: the rows after the cursor, through core's generic
14
+ * subscribe machine with `emit: 'page'`, so each page is sent as it is read and
15
+ * the worker holds one page in memory, never the backlog. Each page is checked
16
+ * against the generation first, and so is every poll interval on its own, since
17
+ * a reset that writes nothing yields no page; a generation that moved since the
18
+ * cursor was saved ends the tail, and step 1 sees the mismatch.
19
+ *
20
+ * Every page goes through one delivery: send, park the keys the sink refuses with
21
+ * its reasons, send the rest again, confirm. Then ONE compare-and-set moves the
22
+ * cursor past the whole page and moves the parked list with it — the refused keys
23
+ * parked, every key that was taken unparked — and fenced on the lease holder, so a
24
+ * worker that lost its lease moves nothing. Then tombstones every sink has passed
25
+ * are collected.
26
+ *
27
+ * The rules that sequence keeps, each a consequence of a sink not being
28
+ * transactional:
29
+ *
30
+ * - **The cursor moves only after `confirm`.** A crash between `apply` and
31
+ * `confirm` resends the batch; a sink is idempotent, so that costs a resend and
32
+ * nothing else.
33
+ * - **A refused key does not hold the feed up.** It is parked and the cursor moves
34
+ * past it. It is retried when its row next changes, because a change gives the
35
+ * row a `seq` above the cursor, and it leaves the parked list once it is sent
36
+ * successfully — a tombstone for a parked key included, which is how a refused
37
+ * record that is then deleted stops being parked without anyone acting.
38
+ * - **A retryable failure keeps the cursor.** The session ends, and supervision
39
+ * (below) starts a new one after a back-off no shorter than the sink's
40
+ * `retryAfter`; the new session sends the same page again.
41
+ *
42
+ * ## Resync
43
+ *
44
+ * Every row, page by page from the origin, goes into `resync.begin`'s fresh
45
+ * target; then the generation is checked once more, the target is published over
46
+ * the live copy, and the cursor is saved at the new generation with the parked list
47
+ * REPLACED by the refusals the resync met. Resync progress is held in memory only,
48
+ * so a crash restarts it from the origin, with a new `begin` that drops the
49
+ * half-built target the crash left.
50
+ *
51
+ * ### Waiting for the materialisation
52
+ *
53
+ * The feed's end is not the rebuild's end. After a reset the materialisation
54
+ * replays the log and writes the feed as it goes, so a resync that reaches the
55
+ * feed's last row has reached wherever the replay had got to, and publishing there
56
+ * swaps a complete live copy for a partial one. `readiness.ts` owns that argument.
57
+ * So `runFeedSync` takes an optional `readiness`: asked once as each attempt
58
+ * starts, it hands back a probe, and the attempt does not publish until the probe
59
+ * says the materialisation has caught up. Until then the attempt polls: once per
60
+ * poll interval it checks the generation, abandoning the attempt as stale if a
61
+ * reset moved it again, asks the probe, and reads the feed on from where it left
62
+ * off, loading what the replay wrote meanwhile. The probe is asked BEFORE that
63
+ * read, so the read that follows a `true` sees every row the materialisation had
64
+ * committed by the time it answered, since a feed row commits with the checkpoint
65
+ * that produced it. The lease is still checked immediately before `publish`, and
66
+ * the generation once more just before that, however long the wait was.
67
+ *
68
+ * With no `readiness` the attempt publishes at the feed's end, which is exactly the
69
+ * behaviour above, and search can serve a PARTIAL copy for as long as the replay
70
+ * takes. An in-place rebuild has no publish; there the wait holds the closing
71
+ * reconcile and the cursor save instead, so the reconcile never deletes a record
72
+ * the replay has not reached yet. Without a `readiness` an in-place rebuild
73
+ * reconciles at the feed's end too, and so deletes from the live copy every record
74
+ * the replay had not yet written again; the tail sends each back as the replay
75
+ * reaches it.
76
+ *
77
+ * The wait is UNBOUNDED. A probe that never says yes holds the attempt for good,
78
+ * with no error and no restart, because publishing early is the very failure the
79
+ * wait exists to prevent and no timeout could tell a stuck replay from a long one.
80
+ * It is made visible instead: `inspectFeed` reports the sink `resyncDue` for as long
81
+ * as it lasts, since the cursor stays at the old generation, and every
82
+ * `HELD_WAIT_WARN_POLLS` poll intervals (`tuning.ts`) the attempt logs a warning,
83
+ * "feed sync rebuild held on readiness", with the feed, the sink, the generation,
84
+ * the polls so far and how long it has been held.
85
+ *
86
+ * Each `begin` names a target of its own, never one an earlier attempt used, and
87
+ * that is what bounds a lease lapse here. A worker that has lost its lease can
88
+ * still land one call after the new holder's (`FeedSyncStore.ts` says when). Were
89
+ * the target named after its generation alone, that call could be a `publish` of
90
+ * the NEW holder's half-built target, which then goes on loading into a fresh one
91
+ * holding only the later pages, publishes it, and saves its cursor at the head:
92
+ * whole pages lost, and nothing to bring them back. With a target per attempt, a
93
+ * stale `publish` moves only the stale worker's own target — or fails, retryably,
94
+ * when a newer `begin` has dropped it — and a stale `begin` drops the new holder's
95
+ * target, whose `publish` then fails retryably and whose resync starts again.
96
+ * What a stale `publish` of its own target still costs is `FeedSyncStore.ts`'s to
97
+ * say, with the rest of the lease's window.
98
+ *
99
+ * `begin`, `load`, `apply` and `publish` each run behind `ensureHeld`; `confirm`
100
+ * and a reconcile's key listing change nothing, and do not.
101
+ *
102
+ * Tombstones are sent as deletes, although the target starts empty. The feed does
103
+ * not stand still while a resync pages through it: a key loaded on an early page
104
+ * can be deleted by the read model before the resync is done, and its tombstone
105
+ * then sits at a `seq` a LATER page reads. Dropping it would load the key, save
106
+ * the cursor past its delete, and publish a target that holds a record the feed no
107
+ * longer has, with nothing ever coming to remove it. A tombstone for a key the
108
+ * target never held costs a send and changes nothing, since a delete of an absent
109
+ * key is a no-op by the sink laws.
110
+ *
111
+ * The same movement decides the parked list. A key can be met more than once in
112
+ * one resync — refused on an early page, then changed and met again later — so
113
+ * the list saved is what each key's LAST change in the resync came to: parked if
114
+ * that change was refused, absent if it was taken. Keeping every refusal the
115
+ * resync met would leave parked a key the target holds, or one the feed deleted.
116
+ *
117
+ * A sink with no `resync` is resynced IN PLACE: every row is sent to the live copy,
118
+ * live rows as upserts and tombstones as deletes, and then a reconcile deletes what
119
+ * the feed no longer holds. The live copy serves a mixture of old and new from the
120
+ * first page on, which is the price of a target with no atomic swap. Its refusals
121
+ * land in the parked list it saves like any resync's, with two consequences of
122
+ * writing over the live copy: a refused upsert leaves the PREVIOUS generation's
123
+ * document of that key live, and an orphan whose delete the closing reconcile was
124
+ * refused stays live too. Both are parked with the sink's reason, the orphan at the
125
+ * origin `seq` since it has no row, and the closing reconcile's report goes to
126
+ * `onReconcile` like a scheduled one's. A sink with neither part could never be
127
+ * rebuilt, and is refused at construction.
128
+ *
129
+ * ## Supervision
130
+ *
131
+ * Every failure a session meets ends it, and the worker's own loop decides what
132
+ * next. It follows `@kairos-es/automation`'s daemon rather than generalising the
133
+ * read side's supervisor, whose own doc says a progress signal the caller owns does
134
+ * not qualify — and the signal here, a cursor advance, is the worker's own.
135
+ *
136
+ * - `LeaseHeld`: another worker is running. This one waits a poll interval and
137
+ * tries again, and the wait is not a failure: a standby worker is a healthy one,
138
+ * and it takes over within a poll interval of the first worker's lease going.
139
+ * - Anything else: a back-off of `restartMinDelay` doubling to `restartMaxDelay`,
140
+ * scaled by a jitter drawn from `Random` so a fleet of workers does not retry in
141
+ * lockstep, and never shorter than a `SinkRetryable`'s `retryAfter`. A restart
142
+ * that follows no cursor advance since the previous restart counts towards
143
+ * `maxNoProgressRestarts`; reaching it ends the worker with `FeedSyncStalled`,
144
+ * for the caller to report. Any advance resets the count and the curve.
145
+ *
146
+ * Defects are not caught: they are bugs, in the driver or in a sink or store that
147
+ * should have classified its fault, and supervision must not retry a bug.
148
+ *
149
+ * ## The scope
150
+ *
151
+ * The worker never returns. Run it in a fibre of its own beside the projections,
152
+ * and close the scope to stop it: each session runs in a child scope of the
153
+ * caller's, so closing the caller's scope releases the lease.
154
+ */
155
+ import { Duration, Effect, Scope } from 'effect';
156
+ import { FeedSyncStalled } from './errors.js';
157
+ import type { FeedSource } from './Feed.js';
158
+ import { type FeedSink } from './FeedSink.js';
159
+ import type { FeedSyncStore } from './FeedSyncStore.js';
160
+ import { type FeedName, type SinkName } from './identifiers.js';
161
+ import type { FeedReadiness } from './readiness.js';
162
+ import { type ReconcileReport } from './reconcile.js';
163
+ /** What `onRestart` is told before each restart's back-off. */
164
+ export interface FeedSyncRestart {
165
+ readonly feed: FeedName;
166
+ readonly sink: SinkName;
167
+ /** Consecutive restarts with no cursor advance, this one included. */
168
+ readonly restarts: number;
169
+ readonly delay: Duration.Duration;
170
+ /** The failure that ended the session. */
171
+ readonly reason: unknown;
172
+ }
173
+ /** The worker's knobs. Every one has a default; see `tuning.ts`'s module doc. */
174
+ export interface FeedSyncTuning {
175
+ /** The upper bound on an idle sleep. Default 1 second. */
176
+ readonly pollInterval?: Duration.Input;
177
+ /** Rows read, and changes sent, per page. Default core's catch-up page, 512. */
178
+ readonly pageSize?: number;
179
+ /** The first restart back-off. Default 100 ms. */
180
+ readonly restartMinDelay?: Duration.Input;
181
+ /** The restart back-off's cap, before jitter. Default 30 seconds. */
182
+ readonly restartMaxDelay?: Duration.Input;
183
+ /** Consecutive restarts with no cursor advance before `FeedSyncStalled`. Default 40. */
184
+ readonly maxNoProgressRestarts?: number;
185
+ readonly onRestart?: (info: FeedSyncRestart) => Effect.Effect<void>;
186
+ /**
187
+ * Told what each reconcile the worker runs did: each scheduled one, and the one
188
+ * that closes an in-place rebuild.
189
+ */
190
+ readonly onReconcile?: (report: ReconcileReport) => Effect.Effect<void>;
191
+ }
192
+ /** Options for {@link runFeedSync}. */
193
+ export interface RunFeedSyncOptions<D, Receipt> {
194
+ readonly source: FeedSource<D>;
195
+ readonly sink: FeedSink<D, Receipt>;
196
+ readonly store: FeedSyncStore;
197
+ readonly tuning?: FeedSyncTuning;
198
+ /**
199
+ * Reconcile the sink against the feed this often, between batches, under the
200
+ * worker's own lease. Needs the sink's `reconcile`; absent, the worker
201
+ * reconciles only as part of an in-place resync.
202
+ */
203
+ readonly reconcileEvery?: Duration.Input;
204
+ /**
205
+ * Hold each resync's `publish` — or, in place, its closing reconcile — until the
206
+ * materialisation writing the feed has caught up with the log, for as long as
207
+ * that takes; see the module doc's Resync section. Absent, a resync publishes at
208
+ * the feed's end, and search can be PARTIAL while a rebuild's replay runs.
209
+ * `materialisationCaughtUp` builds the common case from the read model.
210
+ */
211
+ readonly readiness?: FeedReadiness;
212
+ }
213
+ /**
214
+ * Run the sync worker for `options.source` and `options.sink`; see the module doc.
215
+ * Dies at once, before taking the lease, on a sink that has neither `resync` nor
216
+ * `reconcile`, on a `reconcileEvery` for a sink with no `reconcile`, and on tuning
217
+ * that is out of range.
218
+ */
219
+ export declare const runFeedSync: <D, Receipt>(options: RunFeedSyncOptions<D, Receipt>) => Effect.Effect<never, FeedSyncStalled, Scope.Scope>;
220
+ //# sourceMappingURL=runFeedSync.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runFeedSync.d.ts","sourceRoot":"","sources":["../../src/runFeedSync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyJG;AAGH,OAAO,EAKL,QAAQ,EACR,MAAM,EAIN,KAAK,EAEN,MAAM,QAAQ,CAAA;AAGf,OAAO,EAEL,eAAe,EAQhB,MAAM,UAAU,CAAA;AACjB,OAAO,KAAK,EAAW,UAAU,EAAE,MAAM,QAAQ,CAAA;AACjD,OAAO,EAKL,KAAK,QAAQ,EACd,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAGV,aAAa,EAGd,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAGL,KAAK,QAAQ,EAGb,KAAK,QAAQ,EACd,MAAM,eAAe,CAAA;AACtB,OAAO,KAAK,EAAE,aAAa,EAAsB,MAAM,aAAa,CAAA;AACpE,OAAO,EAEL,KAAK,eAAe,EAGrB,MAAM,aAAa,CAAA;AAYpB,+DAA+D;AAC/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAA;IACjC,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CACzB;AAED,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACtC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,kDAAkD;IAClD,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACzC,qEAAqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACzC,wFAAwF;IACxF,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACnE;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;CACxE;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB,CAAC,CAAC,EAAE,OAAO;IAC5C,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACnC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAA;IAChC;;;;OAIG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACxC;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAA;CACnC;AAmBD;;;;;GAKG;AACH,eAAO,MAAM,WAAW,GAAI,CAAC,EAAE,OAAO,EACpC,SAAS,kBAAkB,CAAC,CAAC,EAAE,OAAO,CAAC,KACtC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,CAiDhD,CAAA"}
@@ -0,0 +1,40 @@
1
+ /**
2
+ * The driver's tuning: its defaults, and the checks that refuse a nonsensical
3
+ * value at construction rather than somewhere inside a running worker.
4
+ *
5
+ * The page size defaults to core's own catch-up page, and the supervision curve to
6
+ * the read side's — 100 ms doubling to 30 s — so a feed behaves like its
7
+ * neighbours unless told otherwise. The no-progress cap is the read side's too,
8
+ * and for the same reason: forty restarts on that curve is some twenty minutes of
9
+ * a sink refusing everything, which is long enough to ride out an outage and short
10
+ * enough that a broken wiring is reported the same day.
11
+ *
12
+ * Package-internal; the options a caller sets are `FeedSyncTuning`'s, on
13
+ * `runFeedSync`.
14
+ */
15
+ import { ReadLimit } from '@kairos-es/core';
16
+ import { Duration } from 'effect';
17
+ /** Rows read, and changes sent, per page. */
18
+ export declare const DEFAULT_FEED_PAGE_SIZE: number;
19
+ /** The upper bound on an idle worker's sleep: the poll that is the guarantee. */
20
+ export declare const DEFAULT_FEED_POLL_INTERVAL: Duration.Duration;
21
+ /** The first restart back-off. */
22
+ export declare const DEFAULT_FEED_RESTART_MIN_DELAY: Duration.Duration;
23
+ /** The restart back-off's cap, before jitter. */
24
+ export declare const DEFAULT_FEED_RESTART_MAX_DELAY: Duration.Duration;
25
+ /** Consecutive restarts without a cursor advance before the worker gives up. */
26
+ export declare const DEFAULT_FEED_MAX_NO_PROGRESS_RESTARTS = 40;
27
+ /**
28
+ * How many poll intervals a rebuild held on its readiness waits between two
29
+ * warnings that it is still held. Not a bound: the wait goes on, since publishing
30
+ * early is the failure the wait exists to prevent. Counted in polls, not time, so
31
+ * it scales with `pollInterval`: sixty polls is long past any replay that is
32
+ * merely busy at any sensible interval, and rare enough not to flood a log while
33
+ * a long one runs.
34
+ */
35
+ export declare const HELD_WAIT_WARN_POLLS = 60;
36
+ /** The page size as a `ReadLimit`, refusing one that is not a positive integer. */
37
+ export declare const readLimit: (pageSize: number) => ReadLimit;
38
+ /** A duration input decoded, refusing one that is not positive and finite. */
39
+ export declare const positiveDuration: (label: string, input: Duration.Input) => Duration.Duration;
40
+ //# sourceMappingURL=tuning.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tuning.d.ts","sourceRoot":"","sources":["../../src/tuning.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAA8B,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEjC,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB,EAAE,MAAmC,CAAA;AAExE,iFAAiF;AACjF,eAAO,MAAM,0BAA0B,EAAE,QAAQ,CAAC,QAA8B,CAAA;AAEhF,kCAAkC;AAClC,eAAO,MAAM,8BAA8B,EAAE,QAAQ,CAAC,QAChC,CAAA;AAEtB,iDAAiD;AACjD,eAAO,MAAM,8BAA8B,EAAE,QAAQ,CAAC,QAChC,CAAA;AAEtB,gFAAgF;AAChF,eAAO,MAAM,qCAAqC,KAAK,CAAA;AAEvD;;;;;;;GAOG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAA;AAEtC,mFAAmF;AACnF,eAAO,MAAM,SAAS,GAAI,UAAU,MAAM,KAAG,SAO5C,CAAA;AAED,8EAA8E;AAC9E,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,EACb,OAAO,QAAQ,CAAC,KAAK,KACpB,QAAQ,CAAC,QAMX,CAAA"}
@@ -0,0 +1,99 @@
1
+ /**
2
+ * The feed's two faces: the WRITE side a read model's `apply` uses inside its
3
+ * commit, and the READ side a sync worker tails.
4
+ *
5
+ * ## Why the feed is written inside the materialisation's commit
6
+ *
7
+ * A feed row exists if and only if the view write and the checkpoint advance that
8
+ * produced it landed (ADR-0019 §2). That is the read side's existing exactly-once
9
+ * seam, reused rather than rebuilt: the `ProjectionStore` brackets a read model's
10
+ * view writes with its guarded checkpoint advance, so a feed written through the
11
+ * same bracket commits with the view, rolls back with it on a lost guard, and is
12
+ * never seen half-written. Nothing else about the feed's correctness would survive
13
+ * a feed written beside the commit instead of inside it.
14
+ *
15
+ * ## Why writes are batched before they land
16
+ *
17
+ * `BatchedFeedWriter.write` hands its body a `FeedWriter`, collects every upsert
18
+ * and remove the body makes, and lands them together when the body succeeds:
19
+ * the last write per key wins within one call. Two reasons, one per kind of store.
20
+ *
21
+ * - A transactional store would roll a part-written batch back on its own; there
22
+ * coalescing is purely an economy, one row write per key instead of one per
23
+ * event.
24
+ * - A non-transactional store cannot roll anything back, and the `ProjectionStore`
25
+ * port's contract therefore asks that such a store's per-batch view write be ONE
26
+ * atomic effect. Buffering is what makes the feed's half of the batch one atomic
27
+ * effect too, so a body that fails half-way leaves no feed row behind.
28
+ *
29
+ * ## What "coalesced, latest wins" means for the stored row
30
+ *
31
+ * One row per key. An upsert of a document EQUAL to the stored one writes nothing
32
+ * and takes no sequence value, so an event that changes nothing a sink indexes
33
+ * costs the sink nothing (ADR-0019 §4). Equality is of the ENCODED document, as a
34
+ * JSON value whose object keys are unordered, so what counts as a change is the
35
+ * same in every backend. A remove leaves a TOMBSTONE — the row, with no document —
36
+ * so every sink's cursor can pass it and send the delete, and a remove of a key
37
+ * that is absent or already a tombstone writes nothing.
38
+ */
39
+ import type { Position, ReadLimit } from '@kairos-es/core';
40
+ import type { Effect, Option, Stream } from 'effect';
41
+ import type { FeedUnreadable, FeedWriteFailed } from './errors.js';
42
+ import type { FeedKey, FeedName, FeedSeq, Generation } from './identifiers.js';
43
+ /**
44
+ * One row of a feed: the newest state of one record. `doc` is `None` for a
45
+ * tombstone. `eventPosition` is the position of the event batch that last wrote
46
+ * the row, kept for "the sink is at least as fresh as P" and never used as a
47
+ * cursor; `seq` is the cursor.
48
+ */
49
+ export interface FeedRow<D> {
50
+ readonly key: FeedKey;
51
+ readonly doc: Option.Option<D>;
52
+ readonly seq: FeedSeq;
53
+ readonly eventPosition: Position;
54
+ }
55
+ /**
56
+ * What a read model's `apply` writes through, for each search record the batch
57
+ * touches. Used only inside {@link BatchedFeedWriter.write}'s body.
58
+ */
59
+ export interface FeedWriter<D> {
60
+ /** The record's newest state. Writes nothing when the stored document is equal. */
61
+ readonly upsert: (key: FeedKey, doc: D) => Effect.Effect<void, FeedWriteFailed>;
62
+ /** A tombstone. Writes nothing when the key is absent or already a tombstone. */
63
+ readonly remove: (key: FeedKey) => Effect.Effect<void, FeedWriteFailed>;
64
+ }
65
+ /**
66
+ * A feed's write side: run `body` with a {@link FeedWriter}, and land what it
67
+ * wrote, coalesced by key, when it succeeds. `at` becomes every written row's
68
+ * `eventPosition`; pass the position of the last event in the batch being
69
+ * applied. `RW` is what the backend itself needs from the context — a SQL backend
70
+ * writes through the read model's own client, so the client is in `RW` and the
71
+ * write joins whatever transaction the `ProjectionStore` opened around `apply`.
72
+ *
73
+ * Call it from inside the materialisation's `commit` and nowhere else: a feed
74
+ * written outside the commit is a feed that can disagree with its view.
75
+ */
76
+ export interface BatchedFeedWriter<D, RW = never> {
77
+ readonly feed: FeedName;
78
+ readonly write: <A, E, R>(at: Position, body: (writer: FeedWriter<D>) => Effect.Effect<A, E, R>) => Effect.Effect<A, E | FeedWriteFailed, R | RW>;
79
+ }
80
+ /**
81
+ * A feed's read side, as the sync worker sees it.
82
+ *
83
+ * `changesAfter` returns the rows whose `seq` is strictly greater than `after`, in
84
+ * ascending `seq` order, at most `limit` of them, tombstones included. Its one
85
+ * obligation is the subscribe machine's: a row committed and visible must never
86
+ * have a `seq` below one already returned. A feed bound to one materialisation
87
+ * keeps it, because that materialisation has one writer.
88
+ *
89
+ * `listen`, when present, is a WAKE and nothing more: each element says "there
90
+ * may be new rows", and a missed, duplicated or coalesced one makes a read early
91
+ * or late but never wrong, because the poll is the guarantee.
92
+ */
93
+ export interface FeedSource<D> {
94
+ readonly feed: FeedName;
95
+ readonly generation: Effect.Effect<Generation, FeedUnreadable>;
96
+ readonly changesAfter: (after: FeedSeq, limit: ReadLimit) => Effect.Effect<ReadonlyArray<FeedRow<D>>, FeedUnreadable>;
97
+ readonly listen?: Stream.Stream<unknown, FeedUnreadable>;
98
+ }
99
+ //# sourceMappingURL=Feed.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Feed.d.ts","sourceRoot":"","sources":["../../src/Feed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AACpD,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,UAAU,CAAA;AAC/D,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAE3E;;;;;GAKG;AACH,MAAM,WAAW,OAAO,CAAC,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;IACrB,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAA;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,mFAAmF;IACnF,QAAQ,CAAC,MAAM,EAAE,CACf,GAAG,EAAE,OAAO,EACZ,GAAG,EAAE,CAAC,KACH,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IACzC,iFAAiF;IACjF,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;CACxE;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,iBAAiB,CAAC,CAAC,EAAE,EAAE,GAAG,KAAK;IAC9C,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EACtB,EAAE,EAAE,QAAQ,EACZ,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KACpD,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,GAAG,EAAE,CAAC,CAAA;CACnD;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,cAAc,CAAC,CAAA;IAC9D,QAAQ,CAAC,YAAY,EAAE,CACrB,KAAK,EAAE,OAAO,EACd,KAAK,EAAE,SAAS,KACb,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAA;IAC7D,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;CACzD"}
@@ -0,0 +1,189 @@
1
+ /**
2
+ * The sink port: what a target a feed is copied to must offer the driver.
3
+ *
4
+ * ## Three parts, two of them optional
5
+ *
6
+ * - `apply` and `confirm` are the whole of the steady state. `apply` sends a
7
+ * batch and returns a `Receipt`; `confirm` resolves once that batch is applied
8
+ * AND VISIBLE in the target. The driver moves its cursor only after `confirm`
9
+ * (ADR-0019 §6), so a crash between the two resends the batch — which is why
10
+ * every sink must be IDEMPOTENT: the same batch applied twice leaves the target
11
+ * as once. Full-record replace satisfies that; a partial operation such as an
12
+ * increment does not, and a sink must never use one.
13
+ * - `resync` is how a sink rebuilds after the feed's generation moves: load every
14
+ * row into a fresh target that is not yet serving — tombstones too, as deletes,
15
+ * since the feed can delete a key the same resync has already loaded — then swap
16
+ * it over the live one atomically, so search keeps serving the old copy until the new one is
17
+ * complete and records deleted from the feed do not survive the rebuild.
18
+ * - `reconcile` lists the keys the target holds, so the driver can delete those
19
+ * the feed no longer has. Deletes go through the ordinary `apply`.
20
+ *
21
+ * A sink with no atomic swap leaves out `resync`, and the driver then resyncs IN
22
+ * PLACE: it sends every row to the live target and reconciles away what the feed
23
+ * no longer holds. That needs `reconcile`, so `runFeedSync` refuses at
24
+ * construction a sink that has neither part: such a sink could never be rebuilt.
25
+ *
26
+ * ## Why `FeedChange` is not `FeedRow`
27
+ *
28
+ * A sink is handed the change to make, not the row it came from: an `Upsert` with
29
+ * the document or a `Delete`. The sequence and the event position are the
30
+ * driver's business, and a sink that could see them could be tempted to keep a
31
+ * cursor of its own, which would be a second record of progress.
32
+ */
33
+ import { Data, type Effect, type Stream } from 'effect';
34
+ import type { NonEmptyReadonlyArray } from 'effect/Array';
35
+ import type { SinkHalted, SinkRejected, SinkRetryable } from './errors.js';
36
+ import type { FeedRow } from './Feed.js';
37
+ import type { FeedKey, Generation, SinkName } from './identifiers.js';
38
+ /** One change for a sink to make: the record's newest state, or its removal. */
39
+ export type FeedChange<D> = Data.TaggedEnum<{
40
+ Upsert: {
41
+ readonly key: FeedKey;
42
+ readonly doc: D;
43
+ };
44
+ Delete: {
45
+ readonly key: FeedKey;
46
+ };
47
+ }>;
48
+ /** The generic definition behind {@link FeedChange}'s constructors. */
49
+ export interface FeedChangeDefinition extends Data.TaggedEnum.WithGenerics<1> {
50
+ readonly taggedEnum: FeedChange<this['A']>;
51
+ }
52
+ /** `FeedChange.Upsert({ key, doc })`, `FeedChange.Delete({ key })`, and matchers. */
53
+ export declare const FeedChange: {
54
+ readonly Upsert: <A>(args: {
55
+ readonly key: FeedKey;
56
+ readonly doc: A;
57
+ }) => {
58
+ readonly _tag: "Upsert";
59
+ readonly key: FeedKey;
60
+ readonly doc: A;
61
+ };
62
+ readonly Delete: <A>(args: {
63
+ readonly key: FeedKey;
64
+ }) => {
65
+ readonly _tag: "Delete";
66
+ readonly key: FeedKey;
67
+ };
68
+ readonly $is: <Tag extends "Upsert" | "Delete">(tag: Tag) => {
69
+ <T extends {
70
+ readonly _tag: "Upsert";
71
+ readonly key: FeedKey;
72
+ readonly doc: any;
73
+ } | {
74
+ readonly _tag: "Delete";
75
+ readonly key: FeedKey;
76
+ }>(u: T): u is T & {
77
+ readonly _tag: Tag;
78
+ };
79
+ (u: unknown): u is Extract<{
80
+ readonly _tag: "Upsert";
81
+ readonly key: FeedKey;
82
+ readonly doc: unknown;
83
+ }, {
84
+ readonly _tag: Tag;
85
+ }> | Extract<{
86
+ readonly _tag: "Delete";
87
+ readonly key: FeedKey;
88
+ }, {
89
+ readonly _tag: Tag;
90
+ }>;
91
+ };
92
+ readonly $match: {
93
+ <A, B, C, D, Cases extends {
94
+ readonly Upsert: (args: {
95
+ readonly _tag: "Upsert";
96
+ readonly key: FeedKey;
97
+ readonly doc: A;
98
+ }) => any;
99
+ readonly Delete: (args: {
100
+ readonly _tag: "Delete";
101
+ readonly key: FeedKey;
102
+ }) => any;
103
+ }>(cases: Cases): (self: {
104
+ readonly _tag: "Upsert";
105
+ readonly key: FeedKey;
106
+ readonly doc: A;
107
+ } | {
108
+ readonly _tag: "Delete";
109
+ readonly key: FeedKey;
110
+ }) => import("effect/Unify").Unify<ReturnType<Cases["Upsert" | "Delete"]>>;
111
+ <A, B, C, D, Cases extends {
112
+ readonly Upsert: (args: {
113
+ readonly _tag: "Upsert";
114
+ readonly key: FeedKey;
115
+ readonly doc: A;
116
+ }) => any;
117
+ readonly Delete: (args: {
118
+ readonly _tag: "Delete";
119
+ readonly key: FeedKey;
120
+ }) => any;
121
+ }>(self: {
122
+ readonly _tag: "Upsert";
123
+ readonly key: FeedKey;
124
+ readonly doc: A;
125
+ } | {
126
+ readonly _tag: "Delete";
127
+ readonly key: FeedKey;
128
+ }, cases: Cases): import("effect/Unify").Unify<ReturnType<Cases["Upsert" | "Delete"]>>;
129
+ };
130
+ };
131
+ /** The change a row asks of a sink: its document as an upsert, a tombstone as a delete. */
132
+ export declare const changeOf: <D>(row: FeedRow<D>) => FeedChange<D>;
133
+ /**
134
+ * Where ONE resync attempt is being built. `name` is the sink's own name for it (an
135
+ * index name, say), and it is the attempt's alone: no two `begin`s return the same
136
+ * one, even for one generation. So a worker that has lost its lease without yet
137
+ * knowing it can only ever publish its OWN target, never the half-built target of
138
+ * the worker that holds the lease now; `runFeedSync.ts` owns what that buys.
139
+ */
140
+ export interface ResyncTarget {
141
+ readonly generation: Generation;
142
+ readonly name: string;
143
+ }
144
+ /** A sink's rebuild: a fresh target, loaded, then swapped over the live one. */
145
+ export interface FeedResync<D, Receipt> {
146
+ /**
147
+ * A fresh, empty target that is not live, under a name no earlier `begin`
148
+ * returned. Drops every other target an earlier attempt left — crashed,
149
+ * abandoned, or still being loaded by a worker that lost its lease — whose
150
+ * `publish` must then fail retryably rather than touch the live copy.
151
+ */
152
+ readonly begin: (generation: Generation) => Effect.Effect<ResyncTarget, SinkRetryable | SinkHalted>;
153
+ /**
154
+ * `apply`, against the target rather than the live copy. Confirmed by `confirm`.
155
+ * A target a later `begin` dropped must NEVER BE PUBLISHED, since the worker
156
+ * loading into it can be one that lost its lease without knowing it, and a target
157
+ * rebuilt from its later pages alone must never be swapped over the live copy as
158
+ * a fragment. Failing `SinkRetryable` here is the plain way to keep that. A sink
159
+ * whose writes recreate a missing target (an index that springs into being on its
160
+ * first write) may let the load land and recreate it instead, provided `publish`
161
+ * then tells the recreated target from the one `begin` made and refuses it.
162
+ */
163
+ readonly load: (target: ResyncTarget, batch: NonEmptyReadonlyArray<FeedChange<D>>) => Effect.Effect<Receipt, SinkRetryable | SinkRejected | SinkHalted>;
164
+ /**
165
+ * Swap the target over the live copy, atomically, and resolve once it serves. A
166
+ * target a later `begin` dropped — gone, or brought back by a load — fails
167
+ * `SinkRetryable` and leaves the live copy as it was. Once the swap has landed, a
168
+ * sink may tidy the live copy of its own bookkeeping (a marker it set to tell its
169
+ * target apart) on a best-effort basis: the swap is the publish, so a failure of
170
+ * that tidying must not fail it.
171
+ */
172
+ readonly publish: (target: ResyncTarget) => Effect.Effect<void, SinkRetryable | SinkHalted>;
173
+ }
174
+ /** A sink's key listing, for deleting what the feed no longer holds. */
175
+ export interface FeedReconcile {
176
+ /** Every key the live target holds, in any order, each once. */
177
+ readonly listKeys: Stream.Stream<FeedKey, SinkRetryable | SinkHalted>;
178
+ }
179
+ /** A target a feed is copied to. See the module doc. */
180
+ export interface FeedSink<D, Receipt> {
181
+ readonly name: SinkName;
182
+ readonly apply: (batch: NonEmptyReadonlyArray<FeedChange<D>>) => Effect.Effect<Receipt, SinkRetryable | SinkRejected | SinkHalted>;
183
+ /** Resolves once the batch the receipt names is applied and visible. */
184
+ readonly confirm: (receipt: Receipt) => Effect.Effect<void, SinkRetryable | SinkHalted>;
185
+ /** Absent: the driver resyncs in place, which needs `reconcile`. */
186
+ readonly resync?: FeedResync<D, Receipt>;
187
+ readonly reconcile?: FeedReconcile;
188
+ }
189
+ //# sourceMappingURL=FeedSink.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"FeedSink.d.ts","sourceRoot":"","sources":["../../src/FeedSink.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,IAAI,EAAE,KAAK,MAAM,EAAU,KAAK,MAAM,EAAE,MAAM,QAAQ,CAAA;AAC/D,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACzD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACvE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAElE,gFAAgF;AAChF,MAAM,MAAM,UAAU,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC;IAC1C,MAAM,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAA;KAAE,CAAA;IAClD,MAAM,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;KAAE,CAAA;CAClC,CAAC,CAAA;AAEF,uEAAuE;AACvE,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;IAC3E,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;CAC3C;AAED,qFAAqF;AACrF,eAAO,MAAM,UAAU;;sBAVG,OAAO;;;;sBAAP,OAAO;;;;sBACP,OAAO;;;sBAAP,OAAO;;;;;0BADP,OAAO;;;;0BACP,OAAO;;;;;;0BADP,OAAO;;;;;;0BACP,OAAO;;;;;;;;;8BADP,OAAO;;;;;8BACP,OAAO;;;;0BADP,OAAO;;;;0BACP,OAAO;;;;;8BADP,OAAO;;;;;8BACP,OAAO;;;;0BADP,OAAO;;;;0BACP,OAAO;;;CASgC,CAAA;AAEjE,2FAA2F;AAC3F,eAAO,MAAM,QAAQ,GAAI,CAAC,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,KAAG,UAAU,CAAC,CAAC,CAItD,CAAA;AAEJ;;;;;;GAMG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CACtB;AAED,gFAAgF;AAChF,MAAM,WAAW,UAAU,CAAC,CAAC,EAAE,OAAO;IACpC;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,EAAE,CACd,UAAU,EAAE,UAAU,KACnB,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;IAC5D;;;;;;;;;OASG;IACH,QAAQ,CAAC,IAAI,EAAE,CACb,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KACxC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,GAAG,UAAU,CAAC,CAAA;IACtE;;;;;;;OAOG;IACH,QAAQ,CAAC,OAAO,EAAE,CAChB,MAAM,EAAE,YAAY,KACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;CACrD;AAED,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;CACtE;AAED,wDAAwD;AACxD,MAAM,WAAW,QAAQ,CAAC,CAAC,EAAE,OAAO;IAClC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,CACd,KAAK,EAAE,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KACxC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,GAAG,UAAU,CAAC,CAAA;IACtE,wEAAwE;IACxE,QAAQ,CAAC,OAAO,EAAE,CAChB,OAAO,EAAE,OAAO,KACb,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,GAAG,UAAU,CAAC,CAAA;IACpD,oEAAoE;IACpE,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACxC,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAA;CACnC"}