@kairos-es/feed 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +28 -0
  2. package/README.md +109 -2
  3. package/dist/cjs/Feed.js +6 -0
  4. package/dist/cjs/Feed.js.map +1 -0
  5. package/dist/cjs/FeedSink.js +54 -0
  6. package/dist/cjs/FeedSink.js.map +1 -0
  7. package/dist/cjs/FeedSyncStore.js +10 -0
  8. package/dist/cjs/FeedSyncStore.js.map +1 -0
  9. package/dist/cjs/deliver.js +86 -0
  10. package/dist/cjs/deliver.js.map +1 -0
  11. package/dist/cjs/errors.js +114 -0
  12. package/dist/cjs/errors.js.map +1 -0
  13. package/dist/cjs/identifiers.js +63 -0
  14. package/dist/cjs/identifiers.js.map +1 -0
  15. package/dist/cjs/inMemoryFeed.js +328 -0
  16. package/dist/cjs/inMemoryFeed.js.map +1 -0
  17. package/dist/cjs/inMemorySink.js +142 -0
  18. package/dist/cjs/inMemorySink.js.map +1 -0
  19. package/dist/cjs/index.js +184 -0
  20. package/dist/cjs/index.js.map +1 -0
  21. package/dist/cjs/inspect.js +68 -0
  22. package/dist/cjs/inspect.js.map +1 -0
  23. package/dist/cjs/readiness.js +100 -0
  24. package/dist/cjs/readiness.js.map +1 -0
  25. package/dist/cjs/reconcile.js +164 -0
  26. package/dist/cjs/reconcile.js.map +1 -0
  27. package/dist/cjs/runFeedSync.js +603 -0
  28. package/dist/cjs/runFeedSync.js.map +1 -0
  29. package/dist/cjs/tuning.js +60 -0
  30. package/dist/cjs/tuning.js.map +1 -0
  31. package/dist/dts-cjs/Feed.d.ts +99 -0
  32. package/dist/dts-cjs/Feed.d.ts.map +1 -0
  33. package/dist/dts-cjs/FeedSink.d.ts +189 -0
  34. package/dist/dts-cjs/FeedSink.d.ts.map +1 -0
  35. package/dist/dts-cjs/FeedSyncStore.d.ts +281 -0
  36. package/dist/dts-cjs/FeedSyncStore.d.ts.map +1 -0
  37. package/dist/dts-cjs/deliver.d.ts +16 -0
  38. package/dist/dts-cjs/deliver.d.ts.map +1 -0
  39. package/dist/dts-cjs/errors.d.ts +131 -0
  40. package/dist/dts-cjs/errors.d.ts.map +1 -0
  41. package/dist/dts-cjs/identifiers.d.ts +67 -0
  42. package/dist/dts-cjs/identifiers.d.ts.map +1 -0
  43. package/dist/dts-cjs/inMemoryFeed.d.ts +32 -0
  44. package/dist/dts-cjs/inMemoryFeed.d.ts.map +1 -0
  45. package/dist/dts-cjs/inMemorySink.d.ts +61 -0
  46. package/dist/dts-cjs/inMemorySink.d.ts.map +1 -0
  47. package/dist/dts-cjs/index.d.ts +63 -0
  48. package/dist/dts-cjs/index.d.ts.map +1 -0
  49. package/dist/dts-cjs/inspect.d.ts +71 -0
  50. package/dist/dts-cjs/inspect.d.ts.map +1 -0
  51. package/dist/dts-cjs/package.json +3 -0
  52. package/dist/dts-cjs/readiness.d.ts +84 -0
  53. package/dist/dts-cjs/readiness.d.ts.map +1 -0
  54. package/dist/dts-cjs/reconcile.d.ts +68 -0
  55. package/dist/dts-cjs/reconcile.d.ts.map +1 -0
  56. package/dist/dts-cjs/runFeedSync.d.ts +220 -0
  57. package/dist/dts-cjs/runFeedSync.d.ts.map +1 -0
  58. package/dist/dts-cjs/tuning.d.ts +40 -0
  59. package/dist/dts-cjs/tuning.d.ts.map +1 -0
  60. package/dist/dts-esm/Feed.d.ts +99 -0
  61. package/dist/dts-esm/Feed.d.ts.map +1 -0
  62. package/dist/dts-esm/FeedSink.d.ts +189 -0
  63. package/dist/dts-esm/FeedSink.d.ts.map +1 -0
  64. package/dist/dts-esm/FeedSyncStore.d.ts +281 -0
  65. package/dist/dts-esm/FeedSyncStore.d.ts.map +1 -0
  66. package/dist/dts-esm/deliver.d.ts +16 -0
  67. package/dist/dts-esm/deliver.d.ts.map +1 -0
  68. package/dist/dts-esm/errors.d.ts +131 -0
  69. package/dist/dts-esm/errors.d.ts.map +1 -0
  70. package/dist/dts-esm/identifiers.d.ts +67 -0
  71. package/dist/dts-esm/identifiers.d.ts.map +1 -0
  72. package/dist/dts-esm/inMemoryFeed.d.ts +32 -0
  73. package/dist/dts-esm/inMemoryFeed.d.ts.map +1 -0
  74. package/dist/dts-esm/inMemorySink.d.ts +61 -0
  75. package/dist/dts-esm/inMemorySink.d.ts.map +1 -0
  76. package/dist/dts-esm/index.d.ts +63 -0
  77. package/dist/dts-esm/index.d.ts.map +1 -0
  78. package/dist/dts-esm/inspect.d.ts +71 -0
  79. package/dist/dts-esm/inspect.d.ts.map +1 -0
  80. package/dist/dts-esm/package.json +3 -0
  81. package/dist/dts-esm/readiness.d.ts +84 -0
  82. package/dist/dts-esm/readiness.d.ts.map +1 -0
  83. package/dist/dts-esm/reconcile.d.ts +68 -0
  84. package/dist/dts-esm/reconcile.d.ts.map +1 -0
  85. package/dist/dts-esm/runFeedSync.d.ts +220 -0
  86. package/dist/dts-esm/runFeedSync.d.ts.map +1 -0
  87. package/dist/dts-esm/tuning.d.ts +40 -0
  88. package/dist/dts-esm/tuning.d.ts.map +1 -0
  89. package/dist/esm/Feed.js +2 -0
  90. package/dist/esm/Feed.js.map +1 -0
  91. package/dist/esm/FeedSink.js +46 -0
  92. package/dist/esm/FeedSink.js.map +1 -0
  93. package/dist/esm/FeedSyncStore.js +3 -0
  94. package/dist/esm/FeedSyncStore.js.map +1 -0
  95. package/dist/esm/deliver.js +78 -0
  96. package/dist/esm/deliver.js.map +1 -0
  97. package/dist/esm/errors.js +98 -0
  98. package/dist/esm/errors.js.map +1 -0
  99. package/dist/esm/identifiers.js +56 -0
  100. package/dist/esm/identifiers.js.map +1 -0
  101. package/dist/esm/inMemoryFeed.js +320 -0
  102. package/dist/esm/inMemoryFeed.js.map +1 -0
  103. package/dist/esm/inMemorySink.js +134 -0
  104. package/dist/esm/inMemorySink.js.map +1 -0
  105. package/dist/esm/index.js +62 -0
  106. package/dist/esm/index.js.map +1 -0
  107. package/dist/esm/inspect.js +58 -0
  108. package/dist/esm/inspect.js.map +1 -0
  109. package/dist/esm/package.json +4 -0
  110. package/dist/esm/readiness.js +92 -0
  111. package/dist/esm/readiness.js.map +1 -0
  112. package/dist/esm/reconcile.js +153 -0
  113. package/dist/esm/reconcile.js.map +1 -0
  114. package/dist/esm/runFeedSync.js +595 -0
  115. package/dist/esm/runFeedSync.js.map +1 -0
  116. package/dist/esm/tuning.js +51 -0
  117. package/dist/esm/tuning.js.map +1 -0
  118. package/package.json +34 -3
  119. package/src/Feed.ts +111 -0
  120. package/src/FeedSink.ts +130 -0
  121. package/src/FeedSyncStore.ts +326 -0
  122. package/src/deliver.ts +108 -0
  123. package/src/errors.ts +134 -0
  124. package/src/identifiers.ts +92 -0
  125. package/src/inMemoryFeed.ts +582 -0
  126. package/src/inMemorySink.ts +215 -0
  127. package/src/index.ts +136 -0
  128. package/src/inspect.ts +114 -0
  129. package/src/readiness.ts +137 -0
  130. package/src/reconcile.ts +224 -0
  131. package/src/runFeedSync.ts +882 -0
  132. package/src/tuning.ts +65 -0
@@ -0,0 +1,63 @@
1
+ /**
2
+ * @kairos-es/feed — a state-only change feed: the newest state of each record a
3
+ * read model touches, written inside that read model's own commit, and copied by
4
+ * a sync worker to a sink that has no transaction of its own — a search index, a
5
+ * cache, a mirror (ADR-0019).
6
+ *
7
+ * The feed copies STATE. It never tells anyone each thing that happened; that is
8
+ * `@kairos-es/automation`'s job, and the two are deliberately different shapes: an
9
+ * automation's only progress record is the log, while a feed keeps a cursor, a
10
+ * lease and a parked list beside its rows, because what it mirrors is state and
11
+ * its progress is legitimately a cursor.
12
+ *
13
+ * What is here, and where each part's argument lives:
14
+ *
15
+ * - The identifiers (`identifiers.ts`) and the error vocabulary (`errors.ts`).
16
+ * - The feed's write and read sides, `BatchedFeedWriter` / `FeedWriter` and
17
+ * `FeedSource` (`Feed.ts`): why the feed is written inside the commit, and what
18
+ * "coalesced, latest wins" means for a stored row.
19
+ * - The sink port, `FeedSink` with its optional `FeedResync` and `FeedReconcile`
20
+ * (`FeedSink.ts`): what a sink must keep, and why a sink with neither optional
21
+ * part is refused.
22
+ * - The sync store port, `FeedSyncStore` (`FeedSyncStore.ts`): the cursor, the
23
+ * parked list and the worker lease, and the exact contract a backend implements.
24
+ * - The driver, `runFeedSync` (`runFeedSync.ts`): the lifecycle, the rules it
25
+ * keeps, resync, and its own supervision.
26
+ * - `FeedReadiness` and `materialisationCaughtUp` (`readiness.ts`): why a
27
+ * rebuild's publish can wait for the materialisation, and what it waits on.
28
+ * - `reconcileFeed` (`reconcile.ts`), and `inspectFeed` / `retryParked` /
29
+ * `removeSink` (`inspect.ts`).
30
+ * - The in-memory feed (`inMemoryFeed.ts`) and sink (`inMemorySink.ts`): the parity
31
+ * oracles, and what the in-memory feed refuses.
32
+ *
33
+ * Wiring, end to end, in memory:
34
+ *
35
+ * ```ts
36
+ * const feed = yield* makeInMemoryFeed({ feed: FeedName.make('course-search'), doc: CourseDoc })
37
+ * const store = yield* withInMemoryFeed(yield* makeInMemoryProjectionStore, feed, key)
38
+ *
39
+ * // In the read model's `apply`, as the LAST step of the batch:
40
+ * const apply = (batch) =>
41
+ * feed.write(Arr.lastNonEmpty(batch).sequenced.position, (writer) =>
42
+ * Effect.forEach(touchedCourses(batch), (course) =>
43
+ * writer.upsert(FeedKey.make(course.id), toDoc(course)),
44
+ * ),
45
+ * )
46
+ *
47
+ * // Beside the projections, in a fibre of its own:
48
+ * const { sink } = yield* makeInMemorySink<CourseDoc>({ name: SinkName.make('search'), resync: true })
49
+ * yield* Effect.forkScoped(runFeedSync({ source: feed.source, sink, store: feed.syncStore }))
50
+ * ```
51
+ */
52
+ export { CursorSuperseded, FeedSyncStalled, FeedUnreadable, FeedWriteFailed, LeaseHeld, LeaseLost, SinkHalted, SinkRejected, SinkRetryable, } from './errors.js';
53
+ export type { BatchedFeedWriter, FeedRow, FeedSource, FeedWriter, } from './Feed.js';
54
+ export { changeOf, FeedChange, type FeedChangeDefinition, type FeedReconcile, type FeedResync, type FeedSink, type ResyncTarget, } from './FeedSink.js';
55
+ export { type FeedCursor, type FeedLease, type FeedSyncDescription, type FeedSyncStore, type LeaseDescription, type ParkedChange, type ParkedRecord, type SinkSyncState, sameCursor, } from './FeedSyncStore.js';
56
+ export { FEED_ORIGIN, FeedKey, FeedName, FeedSeq, type FeedSinkId, Generation, LeaseHolder, SinkName, } from './identifiers.js';
57
+ export { type InMemoryFeed, type InMemoryFeedOptions, makeInMemoryFeed, withInMemoryFeed, } from './inMemoryFeed.js';
58
+ export { type InMemoryReceipt, type InMemorySentBatch, type InMemorySink, type InMemorySinkOptions, makeInMemorySink, } from './inMemorySink.js';
59
+ export { type FeedInspection, inspectFeed, removeSink, retryParked, type SinkInspection, } from './inspect.js';
60
+ export { type CaughtUpReadModel, type FeedReadiness, type FeedReadinessProbe, materialisationCaughtUp, } from './readiness.js';
61
+ export { type ReconcileFeedOptions, type ReconcileReport, reconcileFeed, } from './reconcile.js';
62
+ export { type FeedSyncRestart, type FeedSyncTuning, type RunFeedSyncOptions, runFeedSync, } from './runFeedSync.js';
63
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AAEH,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,cAAc,EACd,eAAe,EACf,SAAS,EACT,SAAS,EACT,UAAU,EACV,YAAY,EACZ,aAAa,GACd,MAAM,UAAU,CAAA;AACjB,YAAY,EACV,iBAAiB,EACjB,OAAO,EACP,UAAU,EACV,UAAU,GACX,MAAM,QAAQ,CAAA;AACf,OAAO,EACL,QAAQ,EACR,UAAU,EACV,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAClB,KAAK,UAAU,EACf,KAAK,QAAQ,EACb,KAAK,YAAY,GAClB,MAAM,YAAY,CAAA;AACnB,OAAO,EACL,KAAK,UAAU,EACf,KAAK,SAAS,EACd,KAAK,mBAAmB,EACxB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,aAAa,EAClB,UAAU,GACX,MAAM,iBAAiB,CAAA;AACxB,OAAO,EACL,WAAW,EACX,OAAO,EACP,QAAQ,EACR,OAAO,EACP,KAAK,UAAU,EACf,UAAU,EACV,WAAW,EACX,QAAQ,GACT,MAAM,eAAe,CAAA;AACtB,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,gBAAgB,GACjB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACL,KAAK,cAAc,EACnB,WAAW,EACX,UAAU,EACV,WAAW,EACX,KAAK,cAAc,GACpB,MAAM,WAAW,CAAA;AAClB,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,uBAAuB,GACxB,MAAM,aAAa,CAAA;AACpB,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,aAAa,GACd,MAAM,aAAa,CAAA;AACpB,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,WAAW,GACZ,MAAM,eAAe,CAAA"}
@@ -0,0 +1,71 @@
1
+ /**
2
+ * `inspectFeed`, `retryParked` and `removeSink`: an operator's view of a feed's
3
+ * sinks, and the two repairs an operator makes by hand.
4
+ *
5
+ * None of them needs the sink's client, because everything they read or change
6
+ * lives in the source database beside the feed: an operator can see a sink's lag,
7
+ * its lease, its parked records and when it last advanced, send a parked record
8
+ * again, and retire a sink, without the credentials of the index concerned. The
9
+ * same choice `@kairos-es/automation` made for its own `inspect`. There is no CLI
10
+ * or UI over any of them; they are effects an application wires into whatever it
11
+ * already has.
12
+ *
13
+ * `lastAdvance` and `removeSink` are a pair. Every sink registered on a feed holds
14
+ * its tombstone collection back, so a sink the application stopped running and
15
+ * never removed lets tombstones pile up for good; `lastAdvance` is how such a sink
16
+ * shows — still, while the head moves — and `removeSink` is what lets collection
17
+ * proceed without it. `FeedSyncStore.ts` owns both rules.
18
+ */
19
+ import { type DateTime, Effect, Option } from 'effect';
20
+ import type { FeedUnreadable, FeedWriteFailed, LeaseHeld } from './errors.js';
21
+ import type { FeedSource } from './Feed.js';
22
+ import type { FeedCursor, FeedSyncStore, LeaseDescription, ParkedRecord } from './FeedSyncStore.js';
23
+ import type { FeedKey, FeedName, FeedSeq, Generation, SinkName } from './identifiers.js';
24
+ /** One sink of a feed, as `inspectFeed` reports it. */
25
+ export interface SinkInspection {
26
+ readonly sink: SinkName;
27
+ /** The generation the sink's cursor was saved at; `None` before its first sync. */
28
+ readonly generation: Option.Option<Generation>;
29
+ readonly cursor: Option.Option<FeedCursor>;
30
+ /** When the cursor last moved; `None` if it never has. See `SinkSyncState.lastAdvance`. */
31
+ readonly lastAdvance: Option.Option<DateTime.Utc>;
32
+ /** True when the next pass will resync: no cursor, or one at another generation. */
33
+ readonly resyncDue: boolean;
34
+ /** Rows the sink has still to receive; see `SinkSyncState.pending`. */
35
+ readonly lag: number;
36
+ readonly lease: Option.Option<LeaseDescription>;
37
+ readonly parked: ReadonlyArray<ParkedRecord>;
38
+ }
39
+ /** A feed and its sinks. */
40
+ export interface FeedInspection {
41
+ readonly feed: FeedName;
42
+ readonly generation: Generation;
43
+ /** The highest `seq` the feed holds. */
44
+ readonly head: FeedSeq;
45
+ readonly sinks: ReadonlyArray<SinkInspection>;
46
+ }
47
+ /** Every sink of `source`'s feed: its cursor, lag, lease and parked records. */
48
+ export declare const inspectFeed: <D>(source: FeedSource<D>, store: FeedSyncStore) => Effect.Effect<FeedInspection, FeedUnreadable>;
49
+ /**
50
+ * Send one parked record again: delete its park and re-stamp its row's `seq`, so
51
+ * the next pass of every sink on the feed sends it. Succeeds with whether the key
52
+ * was parked at all. Every sink on the feed receives the record again, not only
53
+ * `sink`; an idempotent sink absorbs that.
54
+ */
55
+ export declare const retryParked: (store: FeedSyncStore, args: {
56
+ readonly feed: FeedName;
57
+ readonly sink: SinkName;
58
+ readonly key: FeedKey;
59
+ }) => Effect.Effect<boolean, FeedWriteFailed>;
60
+ /**
61
+ * Retire `sink` from `feed`: delete its cursor, its lease record and its parked
62
+ * records, so it no longer holds the feed's tombstone collection back. Succeeds
63
+ * with whether the store held anything for it. Fails `LeaseHeld`, removing
64
+ * nothing, while a worker for the sink is running: stop the worker first. A worker
65
+ * started for the sink later registers it again from scratch, and resyncs.
66
+ */
67
+ export declare const removeSink: (store: FeedSyncStore, args: {
68
+ readonly feed: FeedName;
69
+ readonly sink: SinkName;
70
+ }) => Effect.Effect<boolean, LeaseHeld | FeedWriteFailed>;
71
+ //# sourceMappingURL=inspect.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"inspect.d.ts","sourceRoot":"","sources":["../../src/inspect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AAC1E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,KAAK,EACV,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,YAAY,EACb,MAAM,iBAAiB,CAAA;AACxB,OAAO,KAAK,EACV,OAAO,EACP,QAAQ,EACR,OAAO,EACP,UAAU,EACV,QAAQ,EACT,MAAM,eAAe,CAAA;AAEtB,uDAAuD;AACvD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,mFAAmF;IACnF,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC9C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAC1C,2FAA2F;IAC3F,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACjD,oFAAoF;IACpF,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;IAC3B,uEAAuE;IACvE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAA;IAC/C,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,YAAY,CAAC,CAAA;CAC7C;AAED,4BAA4B;AAC5B,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAA;IAC/B,wCAAwC;IACxC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;CAC9C;AAED,gFAAgF;AAChF,eAAO,MAAM,WAAW,GAAI,CAAC,EAC3B,QAAQ,UAAU,CAAC,CAAC,CAAC,EACrB,OAAO,aAAa,KACnB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,cAAc,CAsB3C,CAAA;AAEJ;;;;;GAKG;AACH,eAAO,MAAM,WAAW,GACtB,OAAO,aAAa,EACpB,MAAM;IACJ,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;CACtB,KACA,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAA4B,CAAA;AAErE;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GACrB,OAAO,aAAa,EACpB,MAAM;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,KACzD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,GAAG,eAAe,CAA2B,CAAA"}
@@ -0,0 +1,3 @@
1
+ {
2
+ "type": "module"
3
+ }
@@ -0,0 +1,84 @@
1
+ /**
2
+ * `FeedReadiness` — when a rebuilt copy may go live — and
3
+ * `materialisationCaughtUp`, the readiness of the common case.
4
+ *
5
+ * ## Why a rebuild has anything to wait for
6
+ *
7
+ * A reset empties the feed and moves its generation on, and the materialisation
8
+ * then replays the log, writing the feed afresh as it goes. The sync worker sees
9
+ * the new generation within a poll interval and starts its resync at once, but the
10
+ * feed it pages through is only as complete as the replay is far along. Reaching
11
+ * the feed's last row says nothing about the replay: it is wherever the replay had
12
+ * got to. So a resync that publishes at the feed's end swaps a complete live copy
13
+ * for whatever fraction of the rebuild the replay had written, and search serves
14
+ * that partial copy until the tail fills the rest in.
15
+ *
16
+ * Only the materialisation knows how far its replay has got, and the feed's ports
17
+ * deliberately name no materialisation, so the driver is TOLD, through an optional
18
+ * readiness signal on `runFeedSync`, and holds the step that makes the rebuild
19
+ * final until the signal says yes. With no signal the resync publishes at the
20
+ * feed's end, as it always did, and a rebuild can serve a partial copy for as long
21
+ * as the replay takes.
22
+ *
23
+ * ## Why it is a function returning an effect
24
+ *
25
+ * "Caught up" has to be relative to a fixed point: the log as it stood when the
26
+ * rebuild began. Judged against the log as it stands at each ask, a read model
27
+ * under steady writes is always a batch or two behind, and the publish would wait
28
+ * for a quiet moment that may never come. So the signal is asked ONCE per resync
29
+ * attempt, as the attempt starts, for the generation that attempt rebuilds, and
30
+ * returns the PROBE the worker then polls — by which time whatever point the probe
31
+ * judges by is fixed. An attempt starts after the reset that moved the generation,
32
+ * so a point taken then is at or after the log as it stood at the reset: the error
33
+ * is towards waiting slightly longer, never towards publishing early.
34
+ *
35
+ * Both halves fail `FeedUnreadable`, the feed's own "a read could not be made":
36
+ * the worker cannot tell a probe that could not read the materialisation's
37
+ * progress from a feed it could not read, and needs to do the same thing about
38
+ * either — end the session and try again after a back-off.
39
+ */
40
+ import { type AnyProjection, DcbEventStore } from '@kairos-es/core';
41
+ import type { KeyedProjectionStore } from '@kairos-es/read';
42
+ import { Effect } from 'effect';
43
+ import { FeedUnreadable } from './errors.js';
44
+ import type { Generation } from './identifiers.js';
45
+ /**
46
+ * Polled by the worker, between pages, until it answers `true`: the rebuild this
47
+ * probe was made for may now go live. See the module doc.
48
+ */
49
+ export type FeedReadinessProbe = Effect.Effect<boolean, FeedUnreadable>;
50
+ /**
51
+ * Asked once per resync attempt, as it starts, for the generation it rebuilds,
52
+ * and answering with the {@link FeedReadinessProbe} that attempt polls. See the
53
+ * module doc for why it is two steps.
54
+ */
55
+ export type FeedReadiness = (generation: Generation) => Effect.Effect<FeedReadinessProbe, FeedUnreadable>;
56
+ /**
57
+ * What {@link materialisationCaughtUp} needs of a read model: its bound view store
58
+ * and its slices. Structural, so any `ReadModel` from `@kairos-es/read` is one.
59
+ */
60
+ export interface CaughtUpReadModel {
61
+ readonly store: KeyedProjectionStore;
62
+ readonly slices: Readonly<Record<string, AnyProjection>>;
63
+ }
64
+ /**
65
+ * The readiness of the common case: the materialisation's checkpoint has reached
66
+ * the last event its query selects, as the log stood when the resync attempt
67
+ * started. Pass the very read model whose `apply` writes the feed, so its query
68
+ * and its checkpoint are the ones the replay moves.
69
+ *
70
+ * The point is the last event the read model's QUERY selects, and not the log's
71
+ * head, because the runner advances its checkpoint to the positions it is
72
+ * delivered and is never delivered an event its query does not select: a
73
+ * checkpoint judged against a head an unrelated event moved would never arrive.
74
+ * The store hands no cheaper way to that position than reading up to it, so each
75
+ * attempt starts with one paged read of the query's events past the checkpoint —
76
+ * the backlog the replay itself is about to read, a page at a time and never held
77
+ * whole — and each probe after it is one checkpoint read.
78
+ *
79
+ * Generation-blind, deliberately: the read model has one checkpoint whatever the
80
+ * feed's generation, and an attempt whose generation moves on while it waits is
81
+ * abandoned by the worker's own generation check, so the probe never has to know.
82
+ */
83
+ export declare const materialisationCaughtUp: (readModel: CaughtUpReadModel) => Effect.Effect<FeedReadiness, never, DcbEventStore>;
84
+ //# sourceMappingURL=readiness.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"readiness.d.ts","sourceRoot":"","sources":["../../src/readiness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,OAAO,EACL,KAAK,aAAa,EAElB,aAAa,EAId,MAAM,iBAAiB,CAAA;AACxB,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AAC3D,OAAO,EAAgB,MAAM,EAAkB,MAAM,QAAQ,CAAA;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAE/C;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;AAEvE;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,CAC1B,UAAU,EAAE,UAAU,KACnB,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAA;AAEtD;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAA;IACpC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAA;CACzD;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,uBAAuB,GAClC,WAAW,iBAAiB,KAC3B,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,aAAa,CAuChD,CAAA"}
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Reconcile: delete what a sink holds that the feed no longer has.
3
+ *
4
+ * ## Why a feed needs it at all
5
+ *
6
+ * The steady state deletes through tombstones, and a tombstone lives only until
7
+ * every sink registered on the feed has passed it. A record can still be left
8
+ * behind in a sink: a sink restored from a backup, a key written into the index by
9
+ * hand, or a record deleted before a reset, whose tombstone the reset cleared with
10
+ * every other row, followed by an in-place resync that, having no atomic swap,
11
+ * sends only the rows the new generation holds and so never deletes it. Reconcile is the one operation that compares the sink with the feed
12
+ * rather than replaying the feed into the sink, and so the only one that finds
13
+ * those orphans.
14
+ *
15
+ * ## The order of the two reads, and why it is safe
16
+ *
17
+ * The sink's keys are listed FIRST and the feed's live keys read SECOND. A key the
18
+ * sink lists that has a live row by the time the feed is read is kept, so a record
19
+ * written while reconcile runs is never deleted by it. A key whose row turns into
20
+ * a tombstone after the feed is read is kept too, and its tombstone deletes it on
21
+ * the next pass. And a key whose row comes back to life after the feed is read is
22
+ * deleted here and re-sent by the next pass, because its new `seq` is above the
23
+ * cursor. Reconcile runs inside the worker that holds the lease, between batches,
24
+ * so nothing else is sending to the sink while it does (ADR-0019 §8).
25
+ *
26
+ * ## What it costs
27
+ *
28
+ * It holds every key the sink lists and every live key of the feed in memory at
29
+ * once, and reads every live document to find the keys. It is a periodic repair,
30
+ * not a steady-state path: `reconcileEvery` should be measured in hours.
31
+ */
32
+ import { Effect } from 'effect';
33
+ import type { FeedUnreadable, FeedWriteFailed, LeaseHeld, LeaseLost, SinkHalted, SinkRetryable } from './errors.js';
34
+ import type { FeedSource } from './Feed.js';
35
+ import { type FeedReconcile, type FeedSink } from './FeedSink.js';
36
+ import type { FeedSyncStore } from './FeedSyncStore.js';
37
+ import { type FeedKey } from './identifiers.js';
38
+ /** What one reconcile did. */
39
+ export interface ReconcileReport {
40
+ /** How many keys the sink listed. */
41
+ readonly listed: number;
42
+ /** The orphans deleted: listed by the sink, with no live row in the feed. */
43
+ readonly deleted: ReadonlyArray<FeedKey>;
44
+ /** Orphans whose delete the sink refused, with its reason. They stay in the sink. */
45
+ readonly refused: ReadonlyArray<{
46
+ readonly key: FeedKey;
47
+ readonly reason: string;
48
+ }>;
49
+ }
50
+ /** Every key of `source` whose row is live, read page by page from the origin. */
51
+ export declare const liveKeys: <D>(source: FeedSource<D>, pageSize: number) => Effect.Effect<ReadonlySet<FeedKey>, FeedUnreadable>;
52
+ /** Options for {@link reconcileFeed}. */
53
+ export interface ReconcileFeedOptions<D, Receipt> {
54
+ readonly source: FeedSource<D>;
55
+ readonly sink: FeedSink<D, Receipt> & {
56
+ readonly reconcile: FeedReconcile;
57
+ };
58
+ readonly store: FeedSyncStore;
59
+ /** How many rows to read, and deletes to send, at a time. */
60
+ readonly pageSize?: number;
61
+ }
62
+ /**
63
+ * Reconcile `sink` against `source` once, now. It takes the worker lease itself,
64
+ * so it fails `LeaseHeld` while a sync worker for the same feed and sink runs —
65
+ * reconcile through that worker's `reconcileEvery` instead, or stop it first.
66
+ */
67
+ export declare const reconcileFeed: <D, Receipt>(options: ReconcileFeedOptions<D, Receipt>) => Effect.Effect<ReconcileReport, LeaseHeld | LeaseLost | FeedUnreadable | FeedWriteFailed | SinkRetryable | SinkHalted>;
68
+ //# sourceMappingURL=reconcile.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reconcile.d.ts","sourceRoot":"","sources":["../../src/reconcile.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,OAAO,EAAgB,MAAM,EAAkB,MAAM,QAAQ,CAAA;AAG7D,OAAO,KAAK,EACV,cAAc,EACd,eAAe,EACf,SAAS,EACT,SAAS,EACT,UAAU,EACV,aAAa,EACd,MAAM,UAAU,CAAA;AACjB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,EAAc,KAAK,aAAa,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,CAAA;AAC1E,OAAO,KAAK,EAAa,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/D,OAAO,EAAe,KAAK,OAAO,EAAmB,MAAM,eAAe,CAAA;AAG1E,8BAA8B;AAC9B,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;IACxC,qFAAqF;IACrF,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;QAC9B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAA;QACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KACxB,CAAC,CAAA;CACH;AAED,kFAAkF;AAClF,eAAO,MAAM,QAAQ,GAAI,CAAC,EACxB,QAAQ,UAAU,CAAC,CAAC,CAAC,EACrB,UAAU,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,cAAc,CAoBjD,CAAA;AAmGJ,yCAAyC;AACzC,MAAM,WAAW,oBAAoB,CAAC,CAAC,EAAE,OAAO;IAC9C,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG;QAAE,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAA;KAAE,CAAA;IAC3E,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,GAAI,CAAC,EAAE,OAAO,EACtC,SAAS,oBAAoB,CAAC,CAAC,EAAE,OAAO,CAAC,KACxC,MAAM,CAAC,MAAM,CACd,eAAe,EACb,SAAS,GACT,SAAS,GACT,cAAc,GACd,eAAe,GACf,aAAa,GACb,UAAU,CAgBX,CAAA"}
@@ -0,0 +1,220 @@
1
+ /**
2
+ * `runFeedSync` — the sync worker: one per feed and sink, holding the lease,
3
+ * copying the feed's rows to the sink and moving its cursor only once the sink
4
+ * confirms them (ADR-0019).
5
+ *
6
+ * ## The lifecycle
7
+ *
8
+ * A worker runs in SESSIONS. A session takes the lease, then loops:
9
+ *
10
+ * 1. Read the cursor and the feed's generation. No cursor, or a cursor at another
11
+ * generation, means the sink holds a copy of something the feed no longer is,
12
+ * so the session RESYNCS (below) and loops.
13
+ * 2. Otherwise it TAILS: the rows after the cursor, through core's generic
14
+ * subscribe machine with `emit: 'page'`, so each page is sent as it is read and
15
+ * the worker holds one page in memory, never the backlog. Each page is checked
16
+ * against the generation first, and so is every poll interval on its own, since
17
+ * a reset that writes nothing yields no page; a generation that moved since the
18
+ * cursor was saved ends the tail, and step 1 sees the mismatch.
19
+ *
20
+ * Every page goes through one delivery: send, park the keys the sink refuses with
21
+ * its reasons, send the rest again, confirm. Then ONE compare-and-set moves the
22
+ * cursor past the whole page and moves the parked list with it — the refused keys
23
+ * parked, every key that was taken unparked — and fenced on the lease holder, so a
24
+ * worker that lost its lease moves nothing. Then tombstones every sink has passed
25
+ * are collected.
26
+ *
27
+ * The rules that sequence keeps, each a consequence of a sink not being
28
+ * transactional:
29
+ *
30
+ * - **The cursor moves only after `confirm`.** A crash between `apply` and
31
+ * `confirm` resends the batch; a sink is idempotent, so that costs a resend and
32
+ * nothing else.
33
+ * - **A refused key does not hold the feed up.** It is parked and the cursor moves
34
+ * past it. It is retried when its row next changes, because a change gives the
35
+ * row a `seq` above the cursor, and it leaves the parked list once it is sent
36
+ * successfully — a tombstone for a parked key included, which is how a refused
37
+ * record that is then deleted stops being parked without anyone acting.
38
+ * - **A retryable failure keeps the cursor.** The session ends, and supervision
39
+ * (below) starts a new one after a back-off no shorter than the sink's
40
+ * `retryAfter`; the new session sends the same page again.
41
+ *
42
+ * ## Resync
43
+ *
44
+ * Every row, page by page from the origin, goes into `resync.begin`'s fresh
45
+ * target; then the generation is checked once more, the target is published over
46
+ * the live copy, and the cursor is saved at the new generation with the parked list
47
+ * REPLACED by the refusals the resync met. Resync progress is held in memory only,
48
+ * so a crash restarts it from the origin, with a new `begin` that drops the
49
+ * half-built target the crash left.
50
+ *
51
+ * ### Waiting for the materialisation
52
+ *
53
+ * The feed's end is not the rebuild's end. After a reset the materialisation
54
+ * replays the log and writes the feed as it goes, so a resync that reaches the
55
+ * feed's last row has reached wherever the replay had got to, and publishing there
56
+ * swaps a complete live copy for a partial one. `readiness.ts` owns that argument.
57
+ * So `runFeedSync` takes an optional `readiness`: asked once as each attempt
58
+ * starts, it hands back a probe, and the attempt does not publish until the probe
59
+ * says the materialisation has caught up. Until then the attempt polls: once per
60
+ * poll interval it checks the generation, abandoning the attempt as stale if a
61
+ * reset moved it again, asks the probe, and reads the feed on from where it left
62
+ * off, loading what the replay wrote meanwhile. The probe is asked BEFORE that
63
+ * read, so the read that follows a `true` sees every row the materialisation had
64
+ * committed by the time it answered, since a feed row commits with the checkpoint
65
+ * that produced it. The lease is still checked immediately before `publish`, and
66
+ * the generation once more just before that, however long the wait was.
67
+ *
68
+ * With no `readiness` the attempt publishes at the feed's end, which is exactly the
69
+ * behaviour above, and search can serve a PARTIAL copy for as long as the replay
70
+ * takes. An in-place rebuild has no publish; there the wait holds the closing
71
+ * reconcile and the cursor save instead, so the reconcile never deletes a record
72
+ * the replay has not reached yet. Without a `readiness` an in-place rebuild
73
+ * reconciles at the feed's end too, and so deletes from the live copy every record
74
+ * the replay had not yet written again; the tail sends each back as the replay
75
+ * reaches it.
76
+ *
77
+ * The wait is UNBOUNDED. A probe that never says yes holds the attempt for good,
78
+ * with no error and no restart, because publishing early is the very failure the
79
+ * wait exists to prevent and no timeout could tell a stuck replay from a long one.
80
+ * It is made visible instead: `inspectFeed` reports the sink `resyncDue` for as long
81
+ * as it lasts, since the cursor stays at the old generation, and every
82
+ * `HELD_WAIT_WARN_POLLS` poll intervals (`tuning.ts`) the attempt logs a warning,
83
+ * "feed sync rebuild held on readiness", with the feed, the sink, the generation,
84
+ * the polls so far and how long it has been held.
85
+ *
86
+ * Each `begin` names a target of its own, never one an earlier attempt used, and
87
+ * that is what bounds a lease lapse here. A worker that has lost its lease can
88
+ * still land one call after the new holder's (`FeedSyncStore.ts` says when). Were
89
+ * the target named after its generation alone, that call could be a `publish` of
90
+ * the NEW holder's half-built target, which then goes on loading into a fresh one
91
+ * holding only the later pages, publishes it, and saves its cursor at the head:
92
+ * whole pages lost, and nothing to bring them back. With a target per attempt, a
93
+ * stale `publish` moves only the stale worker's own target — or fails, retryably,
94
+ * when a newer `begin` has dropped it — and a stale `begin` drops the new holder's
95
+ * target, whose `publish` then fails retryably and whose resync starts again.
96
+ * What a stale `publish` of its own target still costs is `FeedSyncStore.ts`'s to
97
+ * say, with the rest of the lease's window.
98
+ *
99
+ * `begin`, `load`, `apply` and `publish` each run behind `ensureHeld`; `confirm`
100
+ * and a reconcile's key listing change nothing, and do not.
101
+ *
102
+ * Tombstones are sent as deletes, although the target starts empty. The feed does
103
+ * not stand still while a resync pages through it: a key loaded on an early page
104
+ * can be deleted by the read model before the resync is done, and its tombstone
105
+ * then sits at a `seq` a LATER page reads. Dropping it would load the key, save
106
+ * the cursor past its delete, and publish a target that holds a record the feed no
107
+ * longer has, with nothing ever coming to remove it. A tombstone for a key the
108
+ * target never held costs a send and changes nothing, since a delete of an absent
109
+ * key is a no-op by the sink laws.
110
+ *
111
+ * The same movement decides the parked list. A key can be met more than once in
112
+ * one resync — refused on an early page, then changed and met again later — so
113
+ * the list saved is what each key's LAST change in the resync came to: parked if
114
+ * that change was refused, absent if it was taken. Keeping every refusal the
115
+ * resync met would leave parked a key the target holds, or one the feed deleted.
116
+ *
117
+ * A sink with no `resync` is resynced IN PLACE: every row is sent to the live copy,
118
+ * live rows as upserts and tombstones as deletes, and then a reconcile deletes what
119
+ * the feed no longer holds. The live copy serves a mixture of old and new from the
120
+ * first page on, which is the price of a target with no atomic swap. Its refusals
121
+ * land in the parked list it saves like any resync's, with two consequences of
122
+ * writing over the live copy: a refused upsert leaves the PREVIOUS generation's
123
+ * document of that key live, and an orphan whose delete the closing reconcile was
124
+ * refused stays live too. Both are parked with the sink's reason, the orphan at the
125
+ * origin `seq` since it has no row, and the closing reconcile's report goes to
126
+ * `onReconcile` like a scheduled one's. A sink with neither part could never be
127
+ * rebuilt, and is refused at construction.
128
+ *
129
+ * ## Supervision
130
+ *
131
+ * Every failure a session meets ends it, and the worker's own loop decides what
132
+ * next. It follows `@kairos-es/automation`'s daemon rather than generalising the
133
+ * read side's supervisor, whose own doc says a progress signal the caller owns does
134
+ * not qualify — and the signal here, a cursor advance, is the worker's own.
135
+ *
136
+ * - `LeaseHeld`: another worker is running. This one waits a poll interval and
137
+ * tries again, and the wait is not a failure: a standby worker is a healthy one,
138
+ * and it takes over within a poll interval of the first worker's lease going.
139
+ * - Anything else: a back-off of `restartMinDelay` doubling to `restartMaxDelay`,
140
+ * scaled by a jitter drawn from `Random` so a fleet of workers does not retry in
141
+ * lockstep, and never shorter than a `SinkRetryable`'s `retryAfter`. A restart
142
+ * that follows no cursor advance since the previous restart counts towards
143
+ * `maxNoProgressRestarts`; reaching it ends the worker with `FeedSyncStalled`,
144
+ * for the caller to report. Any advance resets the count and the curve.
145
+ *
146
+ * Defects are not caught: they are bugs, in the driver or in a sink or store that
147
+ * should have classified its fault, and supervision must not retry a bug.
148
+ *
149
+ * ## The scope
150
+ *
151
+ * The worker never returns. Run it in a fibre of its own beside the projections,
152
+ * and close the scope to stop it: each session runs in a child scope of the
153
+ * caller's, so closing the caller's scope releases the lease.
154
+ */
155
+ import { Duration, Effect, Scope } from 'effect';
156
+ import { FeedSyncStalled } from './errors.js';
157
+ import type { FeedSource } from './Feed.js';
158
+ import { type FeedSink } from './FeedSink.js';
159
+ import type { FeedSyncStore } from './FeedSyncStore.js';
160
+ import { type FeedName, type SinkName } from './identifiers.js';
161
+ import type { FeedReadiness } from './readiness.js';
162
+ import { type ReconcileReport } from './reconcile.js';
163
+ /** What `onRestart` is told before each restart's back-off. */
164
+ export interface FeedSyncRestart {
165
+ readonly feed: FeedName;
166
+ readonly sink: SinkName;
167
+ /** Consecutive restarts with no cursor advance, this one included. */
168
+ readonly restarts: number;
169
+ readonly delay: Duration.Duration;
170
+ /** The failure that ended the session. */
171
+ readonly reason: unknown;
172
+ }
173
+ /** The worker's knobs. Every one has a default; see `tuning.ts`'s module doc. */
174
+ export interface FeedSyncTuning {
175
+ /** The upper bound on an idle sleep. Default 1 second. */
176
+ readonly pollInterval?: Duration.Input;
177
+ /** Rows read, and changes sent, per page. Default core's catch-up page, 512. */
178
+ readonly pageSize?: number;
179
+ /** The first restart back-off. Default 100 ms. */
180
+ readonly restartMinDelay?: Duration.Input;
181
+ /** The restart back-off's cap, before jitter. Default 30 seconds. */
182
+ readonly restartMaxDelay?: Duration.Input;
183
+ /** Consecutive restarts with no cursor advance before `FeedSyncStalled`. Default 40. */
184
+ readonly maxNoProgressRestarts?: number;
185
+ readonly onRestart?: (info: FeedSyncRestart) => Effect.Effect<void>;
186
+ /**
187
+ * Told what each reconcile the worker runs did: each scheduled one, and the one
188
+ * that closes an in-place rebuild.
189
+ */
190
+ readonly onReconcile?: (report: ReconcileReport) => Effect.Effect<void>;
191
+ }
192
+ /** Options for {@link runFeedSync}. */
193
+ export interface RunFeedSyncOptions<D, Receipt> {
194
+ readonly source: FeedSource<D>;
195
+ readonly sink: FeedSink<D, Receipt>;
196
+ readonly store: FeedSyncStore;
197
+ readonly tuning?: FeedSyncTuning;
198
+ /**
199
+ * Reconcile the sink against the feed this often, between batches, under the
200
+ * worker's own lease. Needs the sink's `reconcile`; absent, the worker
201
+ * reconciles only as part of an in-place resync.
202
+ */
203
+ readonly reconcileEvery?: Duration.Input;
204
+ /**
205
+ * Hold each resync's `publish` — or, in place, its closing reconcile — until the
206
+ * materialisation writing the feed has caught up with the log, for as long as
207
+ * that takes; see the module doc's Resync section. Absent, a resync publishes at
208
+ * the feed's end, and search can be PARTIAL while a rebuild's replay runs.
209
+ * `materialisationCaughtUp` builds the common case from the read model.
210
+ */
211
+ readonly readiness?: FeedReadiness;
212
+ }
213
+ /**
214
+ * Run the sync worker for `options.source` and `options.sink`; see the module doc.
215
+ * Dies at once, before taking the lease, on a sink that has neither `resync` nor
216
+ * `reconcile`, on a `reconcileEvery` for a sink with no `reconcile`, and on tuning
217
+ * that is out of range.
218
+ */
219
+ export declare const runFeedSync: <D, Receipt>(options: RunFeedSyncOptions<D, Receipt>) => Effect.Effect<never, FeedSyncStalled, Scope.Scope>;
220
+ //# sourceMappingURL=runFeedSync.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runFeedSync.d.ts","sourceRoot":"","sources":["../../src/runFeedSync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyJG;AAGH,OAAO,EAKL,QAAQ,EACR,MAAM,EAIN,KAAK,EAEN,MAAM,QAAQ,CAAA;AAGf,OAAO,EAEL,eAAe,EAQhB,MAAM,UAAU,CAAA;AACjB,OAAO,KAAK,EAAW,UAAU,EAAE,MAAM,QAAQ,CAAA;AACjD,OAAO,EAKL,KAAK,QAAQ,EACd,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAGV,aAAa,EAGd,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAGL,KAAK,QAAQ,EAGb,KAAK,QAAQ,EACd,MAAM,eAAe,CAAA;AACtB,OAAO,KAAK,EAAE,aAAa,EAAsB,MAAM,aAAa,CAAA;AACpE,OAAO,EAEL,KAAK,eAAe,EAGrB,MAAM,aAAa,CAAA;AAYpB,+DAA+D;AAC/D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAA;IACjC,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CACzB;AAED,iFAAiF;AACjF,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,YAAY,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACtC,gFAAgF;IAChF,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,kDAAkD;IAClD,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACzC,qEAAqE;IACrE,QAAQ,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACzC,wFAAwF;IACxF,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAA;IACvC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACnE;;;OAGG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;CACxE;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB,CAAC,CAAC,EAAE,OAAO;IAC5C,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;IAC9B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IACnC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,CAAA;IAChC;;;;OAIG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAA;IACxC;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,aAAa,CAAA;CACnC;AAmBD;;;;;GAKG;AACH,eAAO,MAAM,WAAW,GAAI,CAAC,EAAE,OAAO,EACpC,SAAS,kBAAkB,CAAC,CAAC,EAAE,OAAO,CAAC,KACtC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,KAAK,CAiDhD,CAAA"}
@@ -0,0 +1,40 @@
1
+ /**
2
+ * The driver's tuning: its defaults, and the checks that refuse a nonsensical
3
+ * value at construction rather than somewhere inside a running worker.
4
+ *
5
+ * The page size defaults to core's own catch-up page, and the supervision curve to
6
+ * the read side's — 100 ms doubling to 30 s — so a feed behaves like its
7
+ * neighbours unless told otherwise. The no-progress cap is the read side's too,
8
+ * and for the same reason: forty restarts on that curve is some twenty minutes of
9
+ * a sink refusing everything, which is long enough to ride out an outage and short
10
+ * enough that a broken wiring is reported the same day.
11
+ *
12
+ * Package-internal; the options a caller sets are `FeedSyncTuning`'s, on
13
+ * `runFeedSync`.
14
+ */
15
+ import { ReadLimit } from '@kairos-es/core';
16
+ import { Duration } from 'effect';
17
+ /** Rows read, and changes sent, per page. */
18
+ export declare const DEFAULT_FEED_PAGE_SIZE: number;
19
+ /** The upper bound on an idle worker's sleep: the poll that is the guarantee. */
20
+ export declare const DEFAULT_FEED_POLL_INTERVAL: Duration.Duration;
21
+ /** The first restart back-off. */
22
+ export declare const DEFAULT_FEED_RESTART_MIN_DELAY: Duration.Duration;
23
+ /** The restart back-off's cap, before jitter. */
24
+ export declare const DEFAULT_FEED_RESTART_MAX_DELAY: Duration.Duration;
25
+ /** Consecutive restarts without a cursor advance before the worker gives up. */
26
+ export declare const DEFAULT_FEED_MAX_NO_PROGRESS_RESTARTS = 40;
27
+ /**
28
+ * How many poll intervals a rebuild held on its readiness waits between two
29
+ * warnings that it is still held. Not a bound: the wait goes on, since publishing
30
+ * early is the failure the wait exists to prevent. Counted in polls, not time, so
31
+ * it scales with `pollInterval`: sixty polls is long past any replay that is
32
+ * merely busy at any sensible interval, and rare enough not to flood a log while
33
+ * a long one runs.
34
+ */
35
+ export declare const HELD_WAIT_WARN_POLLS = 60;
36
+ /** The page size as a `ReadLimit`, refusing one that is not a positive integer. */
37
+ export declare const readLimit: (pageSize: number) => ReadLimit;
38
+ /** A duration input decoded, refusing one that is not positive and finite. */
39
+ export declare const positiveDuration: (label: string, input: Duration.Input) => Duration.Duration;
40
+ //# sourceMappingURL=tuning.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tuning.d.ts","sourceRoot":"","sources":["../../src/tuning.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAA8B,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEjC,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB,EAAE,MAAmC,CAAA;AAExE,iFAAiF;AACjF,eAAO,MAAM,0BAA0B,EAAE,QAAQ,CAAC,QAA8B,CAAA;AAEhF,kCAAkC;AAClC,eAAO,MAAM,8BAA8B,EAAE,QAAQ,CAAC,QAChC,CAAA;AAEtB,iDAAiD;AACjD,eAAO,MAAM,8BAA8B,EAAE,QAAQ,CAAC,QAChC,CAAA;AAEtB,gFAAgF;AAChF,eAAO,MAAM,qCAAqC,KAAK,CAAA;AAEvD;;;;;;;GAOG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAA;AAEtC,mFAAmF;AACnF,eAAO,MAAM,SAAS,GAAI,UAAU,MAAM,KAAG,SAO5C,CAAA;AAED,8EAA8E;AAC9E,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,EACb,OAAO,QAAQ,CAAC,KAAK,KACpB,QAAQ,CAAC,QAMX,CAAA"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=Feed.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"Feed.js","names":[],"sources":["../../src/Feed.ts"],"sourcesContent":[null],"mappings":"","ignoreList":[]}
@@ -0,0 +1,46 @@
1
+ /**
2
+ * The sink port: what a target a feed is copied to must offer the driver.
3
+ *
4
+ * ## Three parts, two of them optional
5
+ *
6
+ * - `apply` and `confirm` are the whole of the steady state. `apply` sends a
7
+ * batch and returns a `Receipt`; `confirm` resolves once that batch is applied
8
+ * AND VISIBLE in the target. The driver moves its cursor only after `confirm`
9
+ * (ADR-0019 §6), so a crash between the two resends the batch — which is why
10
+ * every sink must be IDEMPOTENT: the same batch applied twice leaves the target
11
+ * as once. Full-record replace satisfies that; a partial operation such as an
12
+ * increment does not, and a sink must never use one.
13
+ * - `resync` is how a sink rebuilds after the feed's generation moves: load every
14
+ * row into a fresh target that is not yet serving — tombstones too, as deletes,
15
+ * since the feed can delete a key the same resync has already loaded — then swap
16
+ * it over the live one atomically, so search keeps serving the old copy until the new one is
17
+ * complete and records deleted from the feed do not survive the rebuild.
18
+ * - `reconcile` lists the keys the target holds, so the driver can delete those
19
+ * the feed no longer has. Deletes go through the ordinary `apply`.
20
+ *
21
+ * A sink with no atomic swap leaves out `resync`, and the driver then resyncs IN
22
+ * PLACE: it sends every row to the live target and reconciles away what the feed
23
+ * no longer holds. That needs `reconcile`, so `runFeedSync` refuses at
24
+ * construction a sink that has neither part: such a sink could never be rebuilt.
25
+ *
26
+ * ## Why `FeedChange` is not `FeedRow`
27
+ *
28
+ * A sink is handed the change to make, not the row it came from: an `Upsert` with
29
+ * the document or a `Delete`. The sequence and the event position are the
30
+ * driver's business, and a sink that could see them could be tempted to keep a
31
+ * cursor of its own, which would be a second record of progress.
32
+ */
33
+ import { Data, Option } from 'effect';
34
+ /** `FeedChange.Upsert({ key, doc })`, `FeedChange.Delete({ key })`, and matchers. */
35
+ export const FeedChange = /*#__PURE__*/Data.taggedEnum();
36
+ /** The change a row asks of a sink: its document as an upsert, a tombstone as a delete. */
37
+ export const changeOf = row => Option.match(row.doc, {
38
+ onNone: () => FeedChange.Delete({
39
+ key: row.key
40
+ }),
41
+ onSome: doc => FeedChange.Upsert({
42
+ key: row.key,
43
+ doc
44
+ })
45
+ });
46
+ //# sourceMappingURL=FeedSink.js.map