@kairos-es/feed 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/LICENSE +28 -0
  2. package/README.md +109 -2
  3. package/dist/cjs/Feed.js +6 -0
  4. package/dist/cjs/Feed.js.map +1 -0
  5. package/dist/cjs/FeedSink.js +54 -0
  6. package/dist/cjs/FeedSink.js.map +1 -0
  7. package/dist/cjs/FeedSyncStore.js +10 -0
  8. package/dist/cjs/FeedSyncStore.js.map +1 -0
  9. package/dist/cjs/deliver.js +86 -0
  10. package/dist/cjs/deliver.js.map +1 -0
  11. package/dist/cjs/errors.js +114 -0
  12. package/dist/cjs/errors.js.map +1 -0
  13. package/dist/cjs/identifiers.js +63 -0
  14. package/dist/cjs/identifiers.js.map +1 -0
  15. package/dist/cjs/inMemoryFeed.js +328 -0
  16. package/dist/cjs/inMemoryFeed.js.map +1 -0
  17. package/dist/cjs/inMemorySink.js +142 -0
  18. package/dist/cjs/inMemorySink.js.map +1 -0
  19. package/dist/cjs/index.js +184 -0
  20. package/dist/cjs/index.js.map +1 -0
  21. package/dist/cjs/inspect.js +68 -0
  22. package/dist/cjs/inspect.js.map +1 -0
  23. package/dist/cjs/readiness.js +100 -0
  24. package/dist/cjs/readiness.js.map +1 -0
  25. package/dist/cjs/reconcile.js +164 -0
  26. package/dist/cjs/reconcile.js.map +1 -0
  27. package/dist/cjs/runFeedSync.js +603 -0
  28. package/dist/cjs/runFeedSync.js.map +1 -0
  29. package/dist/cjs/tuning.js +60 -0
  30. package/dist/cjs/tuning.js.map +1 -0
  31. package/dist/dts-cjs/Feed.d.ts +99 -0
  32. package/dist/dts-cjs/Feed.d.ts.map +1 -0
  33. package/dist/dts-cjs/FeedSink.d.ts +189 -0
  34. package/dist/dts-cjs/FeedSink.d.ts.map +1 -0
  35. package/dist/dts-cjs/FeedSyncStore.d.ts +281 -0
  36. package/dist/dts-cjs/FeedSyncStore.d.ts.map +1 -0
  37. package/dist/dts-cjs/deliver.d.ts +16 -0
  38. package/dist/dts-cjs/deliver.d.ts.map +1 -0
  39. package/dist/dts-cjs/errors.d.ts +131 -0
  40. package/dist/dts-cjs/errors.d.ts.map +1 -0
  41. package/dist/dts-cjs/identifiers.d.ts +67 -0
  42. package/dist/dts-cjs/identifiers.d.ts.map +1 -0
  43. package/dist/dts-cjs/inMemoryFeed.d.ts +32 -0
  44. package/dist/dts-cjs/inMemoryFeed.d.ts.map +1 -0
  45. package/dist/dts-cjs/inMemorySink.d.ts +61 -0
  46. package/dist/dts-cjs/inMemorySink.d.ts.map +1 -0
  47. package/dist/dts-cjs/index.d.ts +63 -0
  48. package/dist/dts-cjs/index.d.ts.map +1 -0
  49. package/dist/dts-cjs/inspect.d.ts +71 -0
  50. package/dist/dts-cjs/inspect.d.ts.map +1 -0
  51. package/dist/dts-cjs/package.json +3 -0
  52. package/dist/dts-cjs/readiness.d.ts +84 -0
  53. package/dist/dts-cjs/readiness.d.ts.map +1 -0
  54. package/dist/dts-cjs/reconcile.d.ts +68 -0
  55. package/dist/dts-cjs/reconcile.d.ts.map +1 -0
  56. package/dist/dts-cjs/runFeedSync.d.ts +220 -0
  57. package/dist/dts-cjs/runFeedSync.d.ts.map +1 -0
  58. package/dist/dts-cjs/tuning.d.ts +40 -0
  59. package/dist/dts-cjs/tuning.d.ts.map +1 -0
  60. package/dist/dts-esm/Feed.d.ts +99 -0
  61. package/dist/dts-esm/Feed.d.ts.map +1 -0
  62. package/dist/dts-esm/FeedSink.d.ts +189 -0
  63. package/dist/dts-esm/FeedSink.d.ts.map +1 -0
  64. package/dist/dts-esm/FeedSyncStore.d.ts +281 -0
  65. package/dist/dts-esm/FeedSyncStore.d.ts.map +1 -0
  66. package/dist/dts-esm/deliver.d.ts +16 -0
  67. package/dist/dts-esm/deliver.d.ts.map +1 -0
  68. package/dist/dts-esm/errors.d.ts +131 -0
  69. package/dist/dts-esm/errors.d.ts.map +1 -0
  70. package/dist/dts-esm/identifiers.d.ts +67 -0
  71. package/dist/dts-esm/identifiers.d.ts.map +1 -0
  72. package/dist/dts-esm/inMemoryFeed.d.ts +32 -0
  73. package/dist/dts-esm/inMemoryFeed.d.ts.map +1 -0
  74. package/dist/dts-esm/inMemorySink.d.ts +61 -0
  75. package/dist/dts-esm/inMemorySink.d.ts.map +1 -0
  76. package/dist/dts-esm/index.d.ts +63 -0
  77. package/dist/dts-esm/index.d.ts.map +1 -0
  78. package/dist/dts-esm/inspect.d.ts +71 -0
  79. package/dist/dts-esm/inspect.d.ts.map +1 -0
  80. package/dist/dts-esm/package.json +3 -0
  81. package/dist/dts-esm/readiness.d.ts +84 -0
  82. package/dist/dts-esm/readiness.d.ts.map +1 -0
  83. package/dist/dts-esm/reconcile.d.ts +68 -0
  84. package/dist/dts-esm/reconcile.d.ts.map +1 -0
  85. package/dist/dts-esm/runFeedSync.d.ts +220 -0
  86. package/dist/dts-esm/runFeedSync.d.ts.map +1 -0
  87. package/dist/dts-esm/tuning.d.ts +40 -0
  88. package/dist/dts-esm/tuning.d.ts.map +1 -0
  89. package/dist/esm/Feed.js +2 -0
  90. package/dist/esm/Feed.js.map +1 -0
  91. package/dist/esm/FeedSink.js +46 -0
  92. package/dist/esm/FeedSink.js.map +1 -0
  93. package/dist/esm/FeedSyncStore.js +3 -0
  94. package/dist/esm/FeedSyncStore.js.map +1 -0
  95. package/dist/esm/deliver.js +78 -0
  96. package/dist/esm/deliver.js.map +1 -0
  97. package/dist/esm/errors.js +98 -0
  98. package/dist/esm/errors.js.map +1 -0
  99. package/dist/esm/identifiers.js +56 -0
  100. package/dist/esm/identifiers.js.map +1 -0
  101. package/dist/esm/inMemoryFeed.js +320 -0
  102. package/dist/esm/inMemoryFeed.js.map +1 -0
  103. package/dist/esm/inMemorySink.js +134 -0
  104. package/dist/esm/inMemorySink.js.map +1 -0
  105. package/dist/esm/index.js +62 -0
  106. package/dist/esm/index.js.map +1 -0
  107. package/dist/esm/inspect.js +58 -0
  108. package/dist/esm/inspect.js.map +1 -0
  109. package/dist/esm/package.json +4 -0
  110. package/dist/esm/readiness.js +92 -0
  111. package/dist/esm/readiness.js.map +1 -0
  112. package/dist/esm/reconcile.js +153 -0
  113. package/dist/esm/reconcile.js.map +1 -0
  114. package/dist/esm/runFeedSync.js +595 -0
  115. package/dist/esm/runFeedSync.js.map +1 -0
  116. package/dist/esm/tuning.js +51 -0
  117. package/dist/esm/tuning.js.map +1 -0
  118. package/package.json +34 -3
  119. package/src/Feed.ts +111 -0
  120. package/src/FeedSink.ts +130 -0
  121. package/src/FeedSyncStore.ts +326 -0
  122. package/src/deliver.ts +108 -0
  123. package/src/errors.ts +134 -0
  124. package/src/identifiers.ts +92 -0
  125. package/src/inMemoryFeed.ts +582 -0
  126. package/src/inMemorySink.ts +215 -0
  127. package/src/index.ts +136 -0
  128. package/src/inspect.ts +114 -0
  129. package/src/readiness.ts +137 -0
  130. package/src/reconcile.ts +224 -0
  131. package/src/runFeedSync.ts +882 -0
  132. package/src/tuning.ts +65 -0
@@ -0,0 +1,134 @@
1
+ /**
2
+ * The in-memory sink: a `FeedSink` over a `Ref` of documents, with an optional
3
+ * `FeedResync` and `FeedReconcile`, and the ORACLE the sink law suite holds every
4
+ * other sink against.
5
+ *
6
+ * It keeps the laws by construction: an upsert replaces the whole document, a
7
+ * delete of an absent key is a no-op, so the same batch applied twice leaves it as
8
+ * once; a batch is visible the moment `apply` returns, so `confirm` has nothing to
9
+ * wait for; a resync target is a separate map that `publish` swaps in whole, so a
10
+ * published resync holds exactly what was loaded into it, and a load into a target
11
+ * a later `begin` dropped fails rather than recreating it. Each `begin` names its
12
+ * target by a counter the sink's own state holds, so two workers sharing one sink
13
+ * value — the way two processes share one real index — never share a target, and
14
+ * the name is the same on every run of a test.
15
+ *
16
+ * Its two optional parts are switched on by option rather than always present,
17
+ * because which parts a sink has CHANGES what the driver does — a sink with no
18
+ * `resync` is resynced in place — and a test of either path needs a sink that
19
+ * lacks the other. It also records every batch it is handed, live or into a
20
+ * target, so a case can say how many sends a change cost.
21
+ *
22
+ * It lives for as long as the value is held, independently of any feed, which is
23
+ * what lets a test keep one sink across two in-memory feeds and watch the second
24
+ * feed's fresh generation force a resync.
25
+ */
26
+ import { Effect, Ref, Stream } from 'effect';
27
+ import { SinkRetryable } from "./errors.js";
28
+ const LIVE = 'live';
29
+ const applyTo = (documents, changes) => {
30
+ const next = new Map(documents);
31
+ for (const change of changes) {
32
+ if (change._tag === 'Upsert') {
33
+ next.set(change.key, change.doc);
34
+ } else {
35
+ next.delete(change.key);
36
+ }
37
+ }
38
+ return next;
39
+ };
40
+ /** What a load into, or a publish of, a target a later `begin` dropped fails with. */
41
+ const droppedTarget = name => new SinkRetryable({
42
+ reason: `the resync target '${name}' was dropped by a later begin`
43
+ });
44
+ /** The resync target's name for one attempt at `generation`; see the module doc. */
45
+ const targetName = (generation, attempt) => `${LIVE}__g${generation}_${attempt}`;
46
+ /** Build an in-memory sink; see the module doc. */
47
+ export const makeInMemorySink = options => Effect.gen(function* () {
48
+ const state = yield* Ref.make({
49
+ live: new Map(),
50
+ targets: new Map(),
51
+ batches: [],
52
+ published: [],
53
+ attempts: 0
54
+ });
55
+ const record = (target, changes) => Effect.flatten(Ref.modify(state, s => {
56
+ const batches = [...s.batches, {
57
+ target,
58
+ changes
59
+ }];
60
+ const receipt = {
61
+ target,
62
+ batch: batches.length
63
+ };
64
+ if (target === LIVE) {
65
+ return [Effect.succeed(receipt), {
66
+ ...s,
67
+ batches,
68
+ live: applyTo(s.live, changes)
69
+ }];
70
+ }
71
+ const built = s.targets.get(target);
72
+ // Dropped by a later begin: recreating it here is what would let a
73
+ // stale worker's publish swap a fragment in. `FeedResync.load` says so.
74
+ if (built === undefined) {
75
+ return [Effect.fail(droppedTarget(target)), s];
76
+ }
77
+ return [Effect.succeed(receipt), {
78
+ ...s,
79
+ batches,
80
+ targets: new Map(s.targets).set(target, applyTo(built, changes))
81
+ }];
82
+ }));
83
+ const resync = {
84
+ // Every other target is dropped: whatever attempt left it, it will never be
85
+ // published now, and its own publish must fail rather than swap it in.
86
+ begin: generation => Ref.modify(state, s => {
87
+ const attempts = s.attempts + 1;
88
+ const name = targetName(generation, attempts);
89
+ return [{
90
+ generation,
91
+ name
92
+ }, {
93
+ ...s,
94
+ attempts,
95
+ targets: new Map([[name, new Map()]])
96
+ }];
97
+ }),
98
+ load: (target, batch) => record(target.name, batch),
99
+ publish: target => Effect.flatten(Ref.modify(state, s => {
100
+ const built = s.targets.get(target.name);
101
+ if (built === undefined) {
102
+ return [Effect.fail(droppedTarget(target.name)), s];
103
+ }
104
+ const targets = new Map(s.targets);
105
+ targets.delete(target.name);
106
+ return [Effect.void, {
107
+ ...s,
108
+ live: built,
109
+ targets,
110
+ published: [...s.published, target.generation]
111
+ }];
112
+ }))
113
+ };
114
+ const sink = {
115
+ name: options.name,
116
+ apply: batch => record(LIVE, batch),
117
+ confirm: () => Effect.void,
118
+ ...(options.resync === true ? {
119
+ resync
120
+ } : {}),
121
+ ...(options.reconcile === true ? {
122
+ reconcile: {
123
+ listKeys: Stream.unwrap(Effect.map(Ref.get(state), s => Stream.fromIterable(Array.from(s.live.keys()))))
124
+ }
125
+ } : {})
126
+ };
127
+ return {
128
+ sink,
129
+ documents: Effect.map(Ref.get(state), s => s.live),
130
+ batches: Effect.map(Ref.get(state), s => s.batches),
131
+ published: Effect.map(Ref.get(state), s => s.published)
132
+ };
133
+ });
134
+ //# sourceMappingURL=inMemorySink.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"inMemorySink.js","names":["Effect","Ref","Stream","SinkRetryable","LIVE","applyTo","documents","changes","next","Map","change","_tag","set","key","doc","delete","droppedTarget","name","reason","targetName","generation","attempt","makeInMemorySink","options","gen","state","make","live","targets","batches","published","attempts","record","target","flatten","modify","s","receipt","batch","length","succeed","built","get","undefined","fail","resync","begin","load","publish","void","sink","apply","confirm","reconcile","listKeys","unwrap","map","fromIterable","Array","from","keys"],"sources":["../../src/inMemorySink.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;AAyBA,SAASA,MAAM,EAAEC,GAAG,EAAEC,MAAM,QAAQ,QAAQ;AAE5C,SAASC,aAAa,QAAQ,aAAU;AA8CxC,MAAMC,IAAI,GAAG,MAAM;AAEnB,MAAMC,OAAO,GAAGA,CACdC,SAAkC,EAClCC,OAA6C,KAClB;EAC3B,MAAMC,IAAI,GAAG,IAAIC,GAAG,CAACH,SAAS,CAAC;EAC/B,KAAK,MAAMI,MAAM,IAAIH,OAAO,EAAE;IAC5B,IAAIG,MAAM,CAACC,IAAI,KAAK,QAAQ,EAAE;MAC5BH,IAAI,CAACI,GAAG,CAACF,MAAM,CAACG,GAAG,EAAEH,MAAM,CAACI,GAAG,CAAC;IAClC,CAAC,MAAM;MACLN,IAAI,CAACO,MAAM,CAACL,MAAM,CAACG,GAAG,CAAC;IACzB;EACF;EACA,OAAOL,IAAI;AACb,CAAC;AAED;AACA,MAAMQ,aAAa,GAAIC,IAAY,IACjC,IAAId,aAAa,CAAC;EAChBe,MAAM,EAAE,sBAAsBD,IAAI;CACnC,CAAC;AAEJ;AACA,MAAME,UAAU,GAAGA,CAACC,UAAsB,EAAEC,OAAe,KACzD,GAAGjB,IAAI,MAAMgB,UAAU,IAAIC,OAAO,EAAE;AAEtC;AACA,OAAO,MAAMC,gBAAgB,GAC3BC,OAA4B,IAE5BvB,MAAM,CAACwB,GAAG,CAAC,aAAS;EAClB,MAAMC,KAAK,GAAG,OAAOxB,GAAG,CAACyB,IAAI,CAAe;IAC1CC,IAAI,EAAE,IAAIlB,GAAG,EAAE;IACfmB,OAAO,EAAE,IAAInB,GAAG,EAAE;IAClBoB,OAAO,EAAE,EAAE;IACXC,SAAS,EAAE,EAAE;IACbC,QAAQ,EAAE;GACX,CAAC;EAEF,MAAMC,MAAM,GAAGA,CACbC,MAAc,EACd1B,OAA6C,KAE7CP,MAAM,CAACkC,OAAO,CACZjC,GAAG,CAACkC,MAAM,CACRV,KAAK,EAEHW,CAAC,IACgE;IACjE,MAAMP,OAAO,GAAG,CAAC,GAAGO,CAAC,CAACP,OAAO,EAAE;MAAEI,MAAM;MAAE1B;IAAO,CAAE,CAAC;IACnD,MAAM8B,OAAO,GAAG;MAAEJ,MAAM;MAAEK,KAAK,EAAET,OAAO,CAACU;IAAM,CAAE;IACjD,IAAIN,MAAM,KAAK7B,IAAI,EAAE;MACnB,OAAO,CACLJ,MAAM,CAACwC,OAAO,CAACH,OAAO,CAAC,EACvB;QAAE,GAAGD,CAAC;QAAEP,OAAO;QAAEF,IAAI,EAAEtB,OAAO,CAAC+B,CAAC,CAACT,IAAI,EAAEpB,OAAO;MAAC,CAAE,CAClD;IACH;IACA,MAAMkC,KAAK,GAAGL,CAAC,CAACR,OAAO,CAACc,GAAG,CAACT,MAAM,CAAC;IACnC;IACA;IACA,IAAIQ,KAAK,KAAKE,SAAS,EAAE;MACvB,OAAO,CAAC3C,MAAM,CAAC4C,IAAI,CAAC5B,aAAa,CAACiB,MAAM,CAAC,CAAC,EAAEG,CAAC,CAAC;IAChD;IACA,OAAO,CACLpC,MAAM,CAACwC,OAAO,CAACH,OAAO,CAAC,EACvB;MACE,GAAGD,CAAC;MACJP,OAAO;MACPD,OAAO,EAAE,IAAInB,GAAG,CAAC2B,CAAC,CAACR,OAAO,CAAC,CAAChB,GAAG,CAC7BqB,MAAM,EACN5B,OAAO,CAACoC,KAAK,EAAElC,OAAO,CAAC;KAE1B,CACF;EACH,CAAC,CACF,CACF;EAEH,MAAMsC,MAAM,GAAmC;IAC7C;IACA;IACAC,KAAK,EAAG1B,UAAU,IAChBnB,GAAG,CAACkC,MAAM,CAACV,KAAK,EAAGW,CAAC,IAAkC;MACpD,MAAML,QAAQ,GAAGK,CAAC,CAACL,QAAQ,GAAG,CAAC;MAC/B,MAAMd,IAAI,GAAGE,UAAU,CAACC,UAAU,EAAEW,QAAQ,CAAC;MAC7C,OAAO,CACL;QAAEX,UAAU;QAAEH;MAAI,CAAE,EACpB;QAAE,GAAGmB,CAAC;QAAEL,QAAQ;QAAEH,OAAO,EAAE,IAAInB,GAAG,CAAC,CAAC,CAACQ,IAAI,EAAE,IAAIR,GAAG,EAAE,CAAC,CAAC;MAAC,CAAE,CAC1D;IACH,CAAC,CAAC;IACJsC,IAAI,EAAEA,CAACd,MAAM,EAAEK,KAAK,KAAKN,MAAM,CAACC,MAAM,CAAChB,IAAI,EAAEqB,KAAK,CAAC;IACnDU,OAAO,EAAGf,MAAM,IACdjC,MAAM,CAACkC,OAAO,CACZjC,GAAG,CAACkC,MAAM,CACRV,KAAK,EACJW,CAAC,IAAwD;MACxD,MAAMK,KAAK,GAAGL,CAAC,CAACR,OAAO,CAACc,GAAG,CAACT,MAAM,CAAChB,IAAI,CAAC;MACxC,IAAIwB,KAAK,KAAKE,SAAS,EAAE;QACvB,OAAO,CAAC3C,MAAM,CAAC4C,IAAI,CAAC5B,aAAa,CAACiB,MAAM,CAAChB,IAAI,CAAC,CAAC,EAAEmB,CAAC,CAAC;MACrD;MACA,MAAMR,OAAO,GAAG,IAAInB,GAAG,CAAC2B,CAAC,CAACR,OAAO,CAAC;MAClCA,OAAO,CAACb,MAAM,CAACkB,MAAM,CAAChB,IAAI,CAAC;MAC3B,OAAO,CACLjB,MAAM,CAACiD,IAAI,EACX;QACE,GAAGb,CAAC;QACJT,IAAI,EAAEc,KAAK;QACXb,OAAO;QACPE,SAAS,EAAE,CAAC,GAAGM,CAAC,CAACN,SAAS,EAAEG,MAAM,CAACb,UAAU;OAC9C,CACF;IACH,CAAC,CACF;GAEN;EAED,MAAM8B,IAAI,GAAiC;IACzCjC,IAAI,EAAEM,OAAO,CAACN,IAAI;IAClBkC,KAAK,EAAGb,KAAK,IAAKN,MAAM,CAAC5B,IAAI,EAAEkC,KAAK,CAAC;IACrCc,OAAO,EAAEA,CAAA,KAAMpD,MAAM,CAACiD,IAAI;IAC1B,IAAI1B,OAAO,CAACsB,MAAM,KAAK,IAAI,GAAG;MAAEA;IAAM,CAAE,GAAG,EAAE,CAAC;IAC9C,IAAItB,OAAO,CAAC8B,SAAS,KAAK,IAAI,GAC1B;MACEA,SAAS,EAAE;QACTC,QAAQ,EAAEpD,MAAM,CAACqD,MAAM,CACrBvD,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAC3BlC,MAAM,CAACuD,YAAY,CAACC,KAAK,CAACC,IAAI,CAACvB,CAAC,CAACT,IAAI,CAACiC,IAAI,EAAE,CAAC,CAAC,CAC/C;;KAGN,GACD,EAAE;GACP;EAED,OAAO;IACLV,IAAI;IACJ5C,SAAS,EAAEN,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAAKA,CAAC,CAACT,IAAI,CAAC;IACpDE,OAAO,EAAE7B,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAAKA,CAAC,CAACP,OAAO,CAAC;IACrDC,SAAS,EAAE9B,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAAKA,CAAC,CAACN,SAAS;GACzD;AACH,CAAC,CAAC","ignoreList":[]}
@@ -0,0 +1,62 @@
1
+ /**
2
+ * @kairos-es/feed — a state-only change feed: the newest state of each record a
3
+ * read model touches, written inside that read model's own commit, and copied by
4
+ * a sync worker to a sink that has no transaction of its own — a search index, a
5
+ * cache, a mirror (ADR-0019).
6
+ *
7
+ * The feed copies STATE. It never tells anyone each thing that happened; that is
8
+ * `@kairos-es/automation`'s job, and the two are deliberately different shapes: an
9
+ * automation's only progress record is the log, while a feed keeps a cursor, a
10
+ * lease and a parked list beside its rows, because what it mirrors is state and
11
+ * its progress is legitimately a cursor.
12
+ *
13
+ * What is here, and where each part's argument lives:
14
+ *
15
+ * - The identifiers (`identifiers.ts`) and the error vocabulary (`errors.ts`).
16
+ * - The feed's write and read sides, `BatchedFeedWriter` / `FeedWriter` and
17
+ * `FeedSource` (`Feed.ts`): why the feed is written inside the commit, and what
18
+ * "coalesced, latest wins" means for a stored row.
19
+ * - The sink port, `FeedSink` with its optional `FeedResync` and `FeedReconcile`
20
+ * (`FeedSink.ts`): what a sink must keep, and why a sink with neither optional
21
+ * part is refused.
22
+ * - The sync store port, `FeedSyncStore` (`FeedSyncStore.ts`): the cursor, the
23
+ * parked list and the worker lease, and the exact contract a backend implements.
24
+ * - The driver, `runFeedSync` (`runFeedSync.ts`): the lifecycle, the rules it
25
+ * keeps, resync, and its own supervision.
26
+ * - `FeedReadiness` and `materialisationCaughtUp` (`readiness.ts`): why a
27
+ * rebuild's publish can wait for the materialisation, and what it waits on.
28
+ * - `reconcileFeed` (`reconcile.ts`), and `inspectFeed` / `retryParked` /
29
+ * `removeSink` (`inspect.ts`).
30
+ * - The in-memory feed (`inMemoryFeed.ts`) and sink (`inMemorySink.ts`): the parity
31
+ * oracles, and what the in-memory feed refuses.
32
+ *
33
+ * Wiring, end to end, in memory:
34
+ *
35
+ * ```ts
36
+ * const feed = yield* makeInMemoryFeed({ feed: FeedName.make('course-search'), doc: CourseDoc })
37
+ * const store = yield* withInMemoryFeed(yield* makeInMemoryProjectionStore, feed, key)
38
+ *
39
+ * // In the read model's `apply`, as the LAST step of the batch:
40
+ * const apply = (batch) =>
41
+ * feed.write(Arr.lastNonEmpty(batch).sequenced.position, (writer) =>
42
+ * Effect.forEach(touchedCourses(batch), (course) =>
43
+ * writer.upsert(FeedKey.make(course.id), toDoc(course)),
44
+ * ),
45
+ * )
46
+ *
47
+ * // Beside the projections, in a fibre of its own:
48
+ * const { sink } = yield* makeInMemorySink<CourseDoc>({ name: SinkName.make('search'), resync: true })
49
+ * yield* Effect.forkScoped(runFeedSync({ source: feed.source, sink, store: feed.syncStore }))
50
+ * ```
51
+ */
52
+ export { CursorSuperseded, FeedSyncStalled, FeedUnreadable, FeedWriteFailed, LeaseHeld, LeaseLost, SinkHalted, SinkRejected, SinkRetryable } from "./errors.js";
53
+ export { changeOf, FeedChange } from "./FeedSink.js";
54
+ export { sameCursor } from "./FeedSyncStore.js";
55
+ export { FEED_ORIGIN, FeedKey, FeedName, FeedSeq, Generation, LeaseHolder, SinkName } from "./identifiers.js";
56
+ export { makeInMemoryFeed, withInMemoryFeed } from "./inMemoryFeed.js";
57
+ export { makeInMemorySink } from "./inMemorySink.js";
58
+ export { inspectFeed, removeSink, retryParked } from "./inspect.js";
59
+ export { materialisationCaughtUp } from "./readiness.js";
60
+ export { reconcileFeed } from "./reconcile.js";
61
+ export { runFeedSync } from "./runFeedSync.js";
62
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","names":["CursorSuperseded","FeedSyncStalled","FeedUnreadable","FeedWriteFailed","LeaseHeld","LeaseLost","SinkHalted","SinkRejected","SinkRetryable","changeOf","FeedChange","sameCursor","FEED_ORIGIN","FeedKey","FeedName","FeedSeq","Generation","LeaseHolder","SinkName","makeInMemoryFeed","withInMemoryFeed","makeInMemorySink","inspectFeed","removeSink","retryParked","materialisationCaughtUp","reconcileFeed","runFeedSync"],"sources":["../../src/index.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SACEA,gBAAgB,EAChBC,eAAe,EACfC,cAAc,EACdC,eAAe,EACfC,SAAS,EACTC,SAAS,EACTC,UAAU,EACVC,YAAY,EACZC,aAAa,QACR,aAAU;AAOjB,SACEC,QAAQ,EACRC,UAAU,QAML,eAAY;AACnB,SASEC,UAAU,QACL,oBAAiB;AACxB,SACEC,WAAW,EACXC,OAAO,EACPC,QAAQ,EACRC,OAAO,EAEPC,UAAU,EACVC,WAAW,EACXC,QAAQ,QACH,kBAAe;AACtB,SAGEC,gBAAgB,EAChBC,gBAAgB,QACX,mBAAgB;AACvB,SAKEC,gBAAgB,QACX,mBAAgB;AACvB,SAEEC,WAAW,EACXC,UAAU,EACVC,WAAW,QAEN,cAAW;AAClB,SAIEC,uBAAuB,QAClB,gBAAa;AACpB,SAGEC,aAAa,QACR,gBAAa;AACpB,SAIEC,WAAW,QACN,kBAAe","ignoreList":[]}
@@ -0,0 +1,58 @@
1
+ /**
2
+ * `inspectFeed`, `retryParked` and `removeSink`: an operator's view of a feed's
3
+ * sinks, and the two repairs an operator makes by hand.
4
+ *
5
+ * None of them needs the sink's client, because everything they read or change
6
+ * lives in the source database beside the feed: an operator can see a sink's lag,
7
+ * its lease, its parked records and when it last advanced, send a parked record
8
+ * again, and retire a sink, without the credentials of the index concerned. The
9
+ * same choice `@kairos-es/automation` made for its own `inspect`. There is no CLI
10
+ * or UI over any of them; they are effects an application wires into whatever it
11
+ * already has.
12
+ *
13
+ * `lastAdvance` and `removeSink` are a pair. Every sink registered on a feed holds
14
+ * its tombstone collection back, so a sink the application stopped running and
15
+ * never removed lets tombstones pile up for good; `lastAdvance` is how such a sink
16
+ * shows — still, while the head moves — and `removeSink` is what lets collection
17
+ * proceed without it. `FeedSyncStore.ts` owns both rules.
18
+ */
19
+ import { Effect, Option } from 'effect';
20
+ /** Every sink of `source`'s feed: its cursor, lag, lease and parked records. */
21
+ export const inspectFeed = (source, store) => Effect.gen(function* () {
22
+ const generation = yield* source.generation;
23
+ const description = yield* store.describe(source.feed);
24
+ return {
25
+ feed: source.feed,
26
+ generation,
27
+ head: description.head,
28
+ sinks: description.sinks.map(state => ({
29
+ sink: state.sink,
30
+ generation: Option.map(state.cursor, c => c.generation),
31
+ cursor: state.cursor,
32
+ lastAdvance: state.lastAdvance,
33
+ resyncDue: Option.match(state.cursor, {
34
+ onNone: () => true,
35
+ onSome: c => c.generation !== generation
36
+ }),
37
+ lag: state.pending,
38
+ lease: state.lease,
39
+ parked: state.parked
40
+ }))
41
+ };
42
+ });
43
+ /**
44
+ * Send one parked record again: delete its park and re-stamp its row's `seq`, so
45
+ * the next pass of every sink on the feed sends it. Succeeds with whether the key
46
+ * was parked at all. Every sink on the feed receives the record again, not only
47
+ * `sink`; an idempotent sink absorbs that.
48
+ */
49
+ export const retryParked = (store, args) => store.retryParked(args);
50
+ /**
51
+ * Retire `sink` from `feed`: delete its cursor, its lease record and its parked
52
+ * records, so it no longer holds the feed's tombstone collection back. Succeeds
53
+ * with whether the store held anything for it. Fails `LeaseHeld`, removing
54
+ * nothing, while a worker for the sink is running: stop the worker first. A worker
55
+ * started for the sink later registers it again from scratch, and resyncs.
56
+ */
57
+ export const removeSink = (store, args) => store.removeSink(args);
58
+ //# sourceMappingURL=inspect.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"inspect.js","names":["Effect","Option","inspectFeed","source","store","gen","generation","description","describe","feed","head","sinks","map","state","sink","cursor","c","lastAdvance","resyncDue","match","onNone","onSome","lag","pending","lease","parked","retryParked","args","removeSink"],"sources":["../../src/inspect.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;AAkBA,SAAwBA,MAAM,EAAEC,MAAM,QAAQ,QAAQ;AA0CtD;AACA,OAAO,MAAMC,WAAW,GAAGA,CACzBC,MAAqB,EACrBC,KAAoB,KAEpBJ,MAAM,CAACK,GAAG,CAAC,aAAS;EAClB,MAAMC,UAAU,GAAG,OAAOH,MAAM,CAACG,UAAU;EAC3C,MAAMC,WAAW,GAAG,OAAOH,KAAK,CAACI,QAAQ,CAACL,MAAM,CAACM,IAAI,CAAC;EACtD,OAAO;IACLA,IAAI,EAAEN,MAAM,CAACM,IAAI;IACjBH,UAAU;IACVI,IAAI,EAAEH,WAAW,CAACG,IAAI;IACtBC,KAAK,EAAEJ,WAAW,CAACI,KAAK,CAACC,GAAG,CAAEC,KAAK,KAAM;MACvCC,IAAI,EAAED,KAAK,CAACC,IAAI;MAChBR,UAAU,EAAEL,MAAM,CAACW,GAAG,CAACC,KAAK,CAACE,MAAM,EAAGC,CAAC,IAAKA,CAAC,CAACV,UAAU,CAAC;MACzDS,MAAM,EAAEF,KAAK,CAACE,MAAM;MACpBE,WAAW,EAAEJ,KAAK,CAACI,WAAW;MAC9BC,SAAS,EAAEjB,MAAM,CAACkB,KAAK,CAACN,KAAK,CAACE,MAAM,EAAE;QACpCK,MAAM,EAAEA,CAAA,KAAM,IAAI;QAClBC,MAAM,EAAGL,CAAC,IAAKA,CAAC,CAACV,UAAU,KAAKA;OACjC,CAAC;MACFgB,GAAG,EAAET,KAAK,CAACU,OAAO;MAClBC,KAAK,EAAEX,KAAK,CAACW,KAAK;MAClBC,MAAM,EAAEZ,KAAK,CAACY;KACf,CAAC;GACH;AACH,CAAC,CAAC;AAEJ;;;;;;AAMA,OAAO,MAAMC,WAAW,GAAGA,CACzBtB,KAAoB,EACpBuB,IAIC,KAC2CvB,KAAK,CAACsB,WAAW,CAACC,IAAI,CAAC;AAErE;;;;;;;AAOA,OAAO,MAAMC,UAAU,GAAGA,CACxBxB,KAAoB,EACpBuB,IAA0D,KACFvB,KAAK,CAACwB,UAAU,CAACD,IAAI,CAAC","ignoreList":[]}
@@ -0,0 +1,4 @@
1
+ {
2
+ "type": "module",
3
+ "sideEffects": []
4
+ }
@@ -0,0 +1,92 @@
1
+ /**
2
+ * `FeedReadiness` — when a rebuilt copy may go live — and
3
+ * `materialisationCaughtUp`, the readiness of the common case.
4
+ *
5
+ * ## Why a rebuild has anything to wait for
6
+ *
7
+ * A reset empties the feed and moves its generation on, and the materialisation
8
+ * then replays the log, writing the feed afresh as it goes. The sync worker sees
9
+ * the new generation within a poll interval and starts its resync at once, but the
10
+ * feed it pages through is only as complete as the replay is far along. Reaching
11
+ * the feed's last row says nothing about the replay: it is wherever the replay had
12
+ * got to. So a resync that publishes at the feed's end swaps a complete live copy
13
+ * for whatever fraction of the rebuild the replay had written, and search serves
14
+ * that partial copy until the tail fills the rest in.
15
+ *
16
+ * Only the materialisation knows how far its replay has got, and the feed's ports
17
+ * deliberately name no materialisation, so the driver is TOLD, through an optional
18
+ * readiness signal on `runFeedSync`, and holds the step that makes the rebuild
19
+ * final until the signal says yes. With no signal the resync publishes at the
20
+ * feed's end, as it always did, and a rebuild can serve a partial copy for as long
21
+ * as the replay takes.
22
+ *
23
+ * ## Why it is a function returning an effect
24
+ *
25
+ * "Caught up" has to be relative to a fixed point: the log as it stood when the
26
+ * rebuild began. Judged against the log as it stands at each ask, a read model
27
+ * under steady writes is always a batch or two behind, and the publish would wait
28
+ * for a quiet moment that may never come. So the signal is asked ONCE per resync
29
+ * attempt, as the attempt starts, for the generation that attempt rebuilds, and
30
+ * returns the PROBE the worker then polls — by which time whatever point the probe
31
+ * judges by is fixed. An attempt starts after the reset that moved the generation,
32
+ * so a point taken then is at or after the log as it stood at the reset: the error
33
+ * is towards waiting slightly longer, never towards publishing early.
34
+ *
35
+ * Both halves fail `FeedUnreadable`, the feed's own "a read could not be made":
36
+ * the worker cannot tell a probe that could not read the materialisation's
37
+ * progress from a feed it could not read, and needs to do the same thing about
38
+ * either — end the session and try again after a back-off.
39
+ */
40
+ import { composeProjections, DcbEventStore, DEFAULT_CATCH_UP_PAGE_SIZE, ReadLimit } from '@kairos-es/core';
41
+ import { Array as Arr, Effect, Option, Stream } from 'effect';
42
+ import { FeedUnreadable } from "./errors.js";
43
+ /**
44
+ * The readiness of the common case: the materialisation's checkpoint has reached
45
+ * the last event its query selects, as the log stood when the resync attempt
46
+ * started. Pass the very read model whose `apply` writes the feed, so its query
47
+ * and its checkpoint are the ones the replay moves.
48
+ *
49
+ * The point is the last event the read model's QUERY selects, and not the log's
50
+ * head, because the runner advances its checkpoint to the positions it is
51
+ * delivered and is never delivered an event its query does not select: a
52
+ * checkpoint judged against a head an unrelated event moved would never arrive.
53
+ * The store hands no cheaper way to that position than reading up to it, so each
54
+ * attempt starts with one paged read of the query's events past the checkpoint —
55
+ * the backlog the replay itself is about to read, a page at a time and never held
56
+ * whole — and each probe after it is one checkpoint read.
57
+ *
58
+ * Generation-blind, deliberately: the read model has one checkpoint whatever the
59
+ * feed's generation, and an attempt whose generation moves on while it waits is
60
+ * abandoned by the worker's own generation check, so the probe never has to know.
61
+ */
62
+ export const materialisationCaughtUp = readModel => Effect.gen(function* () {
63
+ const log = yield* DcbEventStore;
64
+ const query = composeProjections(readModel.slices).query;
65
+ const limit = ReadLimit.make(DEFAULT_CATCH_UP_PAGE_SIZE);
66
+ const checkpoint = Effect.mapError(readModel.store.readCheckpoint, cause => new FeedUnreadable({
67
+ reason: `the checkpoint of projection '${readModel.store.key.projection}' could not be read`,
68
+ cause
69
+ }));
70
+ // The last position the query selects after `from`, or `from` itself when it
71
+ // selects none: page by page, so the backlog is never held in memory at once.
72
+ const lastSelectedAfter = from => Effect.gen(function* () {
73
+ let last = from;
74
+ for (;;) {
75
+ const response = yield* log.read(query, {
76
+ after: last,
77
+ limit
78
+ });
79
+ const page = yield* Stream.runCollect(response.events);
80
+ const final = Arr.last(page);
81
+ if (Option.isSome(final)) {
82
+ last = final.value.position;
83
+ }
84
+ if (page.length < limit) {
85
+ return last;
86
+ }
87
+ }
88
+ });
89
+ const readiness = () => Effect.map(Effect.flatMap(checkpoint, lastSelectedAfter), target => Effect.map(checkpoint, reached => reached >= target));
90
+ return readiness;
91
+ });
92
+ //# sourceMappingURL=readiness.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"readiness.js","names":["composeProjections","DcbEventStore","DEFAULT_CATCH_UP_PAGE_SIZE","ReadLimit","Array","Arr","Effect","Option","Stream","FeedUnreadable","materialisationCaughtUp","readModel","gen","log","query","slices","limit","make","checkpoint","mapError","store","readCheckpoint","cause","reason","key","projection","lastSelectedAfter","from","last","response","read","after","page","runCollect","events","final","isSome","value","position","length","readiness","map","flatMap","target","reached"],"sources":["../../src/readiness.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,SAEEA,kBAAkB,EAClBC,aAAa,EACbC,0BAA0B,EAE1BC,SAAS,QACJ,iBAAiB;AAExB,SAASC,KAAK,IAAIC,GAAG,EAAEC,MAAM,EAAEC,MAAM,EAAEC,MAAM,QAAQ,QAAQ;AAC7D,SAASC,cAAc,QAAQ,aAAU;AA2BzC;;;;;;;;;;;;;;;;;;;AAmBA,OAAO,MAAMC,uBAAuB,GAClCC,SAA4B,IAE5BL,MAAM,CAACM,GAAG,CAAC,aAAS;EAClB,MAAMC,GAAG,GAAG,OAAOZ,aAAa;EAChC,MAAMa,KAAK,GAAGd,kBAAkB,CAACW,SAAS,CAACI,MAAM,CAAC,CAACD,KAAK;EACxD,MAAME,KAAK,GAAGb,SAAS,CAACc,IAAI,CAACf,0BAA0B,CAAC;EACxD,MAAMgB,UAAU,GAAGZ,MAAM,CAACa,QAAQ,CAChCR,SAAS,CAACS,KAAK,CAACC,cAAc,EAC7BC,KAAK,IACJ,IAAIb,cAAc,CAAC;IACjBc,MAAM,EAAE,iCAAiCZ,SAAS,CAACS,KAAK,CAACI,GAAG,CAACC,UAAU,qBAAqB;IAC5FH;GACD,CAAC,CACL;EAED;EACA;EACA,MAAMI,iBAAiB,GAAIC,IAAc,IACvCrB,MAAM,CAACM,GAAG,CAAC,aAAS;IAClB,IAAIgB,IAAI,GAAGD,IAAI;IACf,SAAS;MACP,MAAME,QAAQ,GAAG,OAAOhB,GAAG,CAACiB,IAAI,CAAChB,KAAK,EAAE;QAAEiB,KAAK,EAAEH,IAAI;QAAEZ;MAAK,CAAE,CAAC;MAC/D,MAAMgB,IAAI,GAAG,OAAOxB,MAAM,CAACyB,UAAU,CAACJ,QAAQ,CAACK,MAAM,CAAC;MACtD,MAAMC,KAAK,GAAG9B,GAAG,CAACuB,IAAI,CAACI,IAAI,CAAC;MAC5B,IAAIzB,MAAM,CAAC6B,MAAM,CAACD,KAAK,CAAC,EAAE;QACxBP,IAAI,GAAGO,KAAK,CAACE,KAAK,CAACC,QAAQ;MAC7B;MACA,IAAIN,IAAI,CAACO,MAAM,GAAGvB,KAAK,EAAE;QACvB,OAAOY,IAAI;MACb;IACF;EACF,CAAC,CAAC;EAEJ,MAAMY,SAAS,GAAkBA,CAAA,KAC/BlC,MAAM,CAACmC,GAAG,CACRnC,MAAM,CAACoC,OAAO,CAACxB,UAAU,EAAEQ,iBAAiB,CAAC,EAC5CiB,MAAM,IACLrC,MAAM,CAACmC,GAAG,CAACvB,UAAU,EAAG0B,OAAO,IAAKA,OAAO,IAAID,MAAM,CAAC,CACzD;EACH,OAAOH,SAAS;AAClB,CAAC,CAAC","ignoreList":[]}
@@ -0,0 +1,153 @@
1
+ /**
2
+ * Reconcile: delete what a sink holds that the feed no longer has.
3
+ *
4
+ * ## Why a feed needs it at all
5
+ *
6
+ * The steady state deletes through tombstones, and a tombstone lives only until
7
+ * every sink registered on the feed has passed it. A record can still be left
8
+ * behind in a sink: a sink restored from a backup, a key written into the index by
9
+ * hand, or a record deleted before a reset, whose tombstone the reset cleared with
10
+ * every other row, followed by an in-place resync that, having no atomic swap,
11
+ * sends only the rows the new generation holds and so never deletes it. Reconcile is the one operation that compares the sink with the feed
12
+ * rather than replaying the feed into the sink, and so the only one that finds
13
+ * those orphans.
14
+ *
15
+ * ## The order of the two reads, and why it is safe
16
+ *
17
+ * The sink's keys are listed FIRST and the feed's live keys read SECOND. A key the
18
+ * sink lists that has a live row by the time the feed is read is kept, so a record
19
+ * written while reconcile runs is never deleted by it. A key whose row turns into
20
+ * a tombstone after the feed is read is kept too, and its tombstone deletes it on
21
+ * the next pass. And a key whose row comes back to life after the feed is read is
22
+ * deleted here and re-sent by the next pass, because its new `seq` is above the
23
+ * cursor. Reconcile runs inside the worker that holds the lease, between batches,
24
+ * so nothing else is sending to the sink while it does (ADR-0019 §8).
25
+ *
26
+ * ## What it costs
27
+ *
28
+ * It holds every key the sink lists and every live key of the feed in memory at
29
+ * once, and reads every live document to find the keys. It is a periodic repair,
30
+ * not a steady-state path: `reconcileEvery` should be measured in hours.
31
+ */
32
+ import { Array as Arr, Effect, Option, Stream } from 'effect';
33
+ import { deliver } from "./deliver.js";
34
+ import { FeedChange } from "./FeedSink.js";
35
+ import { FEED_ORIGIN } from "./identifiers.js";
36
+ import { DEFAULT_FEED_PAGE_SIZE, readLimit } from "./tuning.js";
37
+ /** Every key of `source` whose row is live, read page by page from the origin. */
38
+ export const liveKeys = (source, pageSize) => Effect.gen(function* () {
39
+ const live = new Set();
40
+ const limit = readLimit(pageSize);
41
+ let after = FEED_ORIGIN;
42
+ for (;;) {
43
+ const rows = yield* source.changesAfter(after, limit);
44
+ for (const row of rows) {
45
+ if (Option.isSome(row.doc)) {
46
+ live.add(row.key);
47
+ } else {
48
+ live.delete(row.key);
49
+ }
50
+ }
51
+ const last = rows.at(-1);
52
+ if (last === undefined || rows.length < pageSize) {
53
+ return live;
54
+ }
55
+ after = last.seq;
56
+ }
57
+ });
58
+ /**
59
+ * Reconcile under a lease the caller already holds. Package-internal: the driver
60
+ * calls it between batches and in an in-place resync, and `reconcileFeed` calls it
61
+ * after taking the lease itself.
62
+ *
63
+ * @internal
64
+ */
65
+ export const reconcileWith = args => Effect.gen(function* () {
66
+ const listed = yield* Stream.runCollect(args.reconcile.listKeys);
67
+ const live = yield* liveKeys(args.source, args.pageSize);
68
+ const orphans = Array.from(new Set(listed)).filter(key => !live.has(key));
69
+ const deleted = [];
70
+ const refused = [];
71
+ for (const chunk of Arr.chunksOf(orphans, args.pageSize)) {
72
+ const pending = Arr.map(chunk, key => ({
73
+ change: FeedChange.Delete({
74
+ key
75
+ }),
76
+ seq: FEED_ORIGIN
77
+ }));
78
+ const delivered = yield* deliver({
79
+ send: args.sink.apply,
80
+ confirm: args.sink.confirm,
81
+ lease: args.lease,
82
+ pending
83
+ });
84
+ deleted.push(...delivered.sent);
85
+ refused.push(...delivered.parked.map(p => ({
86
+ key: p.key,
87
+ reason: p.reason
88
+ })));
89
+ }
90
+ const listedKeys = new Set(listed);
91
+ const deletedKeys = new Set(deleted);
92
+ return {
93
+ report: {
94
+ listed: listedKeys.size,
95
+ deleted,
96
+ refused
97
+ },
98
+ isGone: key => !live.has(key) && (deletedKeys.has(key) || !listedKeys.has(key))
99
+ };
100
+ });
101
+ /**
102
+ * Unpark every parked key a reconcile found gone from the sink for good (see
103
+ * {@link Reconciled}): a park reporting a refusal for such a key — an in-place
104
+ * rebuild's orphan, or a tombstone whose delete the tail was refused — reports a
105
+ * record that is not there. `retryParked` unparks a key with no row and re-stamps
106
+ * one that has a row, which for a tombstone sends every sink one idempotent delete.
107
+ *
108
+ * It judges by what the sink holds, not by what this reconcile deleted, so a park
109
+ * an earlier unpark failed to clear goes on the next reconcile. That is why a
110
+ * failure here is a warning and not an error: it costs a stale entry in
111
+ * `inspectFeed` until then, and failing would end a worker's session, or discard
112
+ * an operator's report, over a record that is already gone.
113
+ *
114
+ * @internal
115
+ */
116
+ export const unparkGone = (store, id, isGone) => Effect.gen(function* () {
117
+ const parked = yield* store.readParked(id);
118
+ yield* Effect.forEach(parked.filter(p => isGone(p.key)), p => store.retryParked({
119
+ ...id,
120
+ key: p.key
121
+ }), {
122
+ discard: true
123
+ });
124
+ }).pipe(Effect.catch(cause => Effect.logWarning('feed reconcile could not unpark a record it found gone; the next reconcile will', {
125
+ feed: id.feed,
126
+ sink: id.sink,
127
+ reason: cause.message
128
+ })));
129
+ /**
130
+ * Reconcile `sink` against `source` once, now. It takes the worker lease itself,
131
+ * so it fails `LeaseHeld` while a sync worker for the same feed and sink runs —
132
+ * reconcile through that worker's `reconcileEvery` instead, or stop it first.
133
+ */
134
+ export const reconcileFeed = options => Effect.scoped(Effect.gen(function* () {
135
+ const id = {
136
+ feed: options.source.feed,
137
+ sink: options.sink.name
138
+ };
139
+ const lease = yield* options.store.acquireLease(id);
140
+ const {
141
+ report,
142
+ isGone
143
+ } = yield* reconcileWith({
144
+ source: options.source,
145
+ sink: options.sink,
146
+ reconcile: options.sink.reconcile,
147
+ lease,
148
+ pageSize: options.pageSize ?? DEFAULT_FEED_PAGE_SIZE
149
+ });
150
+ yield* unparkGone(options.store, id, isGone);
151
+ return report;
152
+ }));
153
+ //# sourceMappingURL=reconcile.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reconcile.js","names":["Array","Arr","Effect","Option","Stream","deliver","FeedChange","FEED_ORIGIN","DEFAULT_FEED_PAGE_SIZE","readLimit","liveKeys","source","pageSize","gen","live","Set","limit","after","rows","changesAfter","row","isSome","doc","add","key","delete","last","at","undefined","length","seq","reconcileWith","args","listed","runCollect","reconcile","listKeys","orphans","from","filter","has","deleted","refused","chunk","chunksOf","pending","map","change","Delete","delivered","send","sink","apply","confirm","lease","push","sent","parked","p","reason","listedKeys","deletedKeys","report","size","isGone","unparkGone","store","id","readParked","forEach","retryParked","discard","pipe","catch","cause","logWarning","feed","message","reconcileFeed","options","scoped","name","acquireLease"],"sources":["../../src/reconcile.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,SAASA,KAAK,IAAIC,GAAG,EAAEC,MAAM,EAAEC,MAAM,EAAEC,MAAM,QAAQ,QAAQ;AAE7D,SAASC,OAAO,QAAsB,cAAW;AAUjD,SAASC,UAAU,QAA2C,eAAY;AAE1E,SAASC,WAAW,QAAuC,kBAAe;AAC1E,SAASC,sBAAsB,EAAEC,SAAS,QAAQ,aAAU;AAe5D;AACA,OAAO,MAAMC,QAAQ,GAAGA,CACtBC,MAAqB,EACrBC,QAAgB,KAEhBV,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAMC,IAAI,GAAG,IAAIC,GAAG,EAAW;EAC/B,MAAMC,KAAK,GAAGP,SAAS,CAACG,QAAQ,CAAC;EACjC,IAAIK,KAAK,GAAGV,WAAW;EACvB,SAAS;IACP,MAAMW,IAAI,GAAG,OAAOP,MAAM,CAACQ,YAAY,CAACF,KAAK,EAAED,KAAK,CAAC;IACrD,KAAK,MAAMI,GAAG,IAAIF,IAAI,EAAE;MACtB,IAAIf,MAAM,CAACkB,MAAM,CAACD,GAAG,CAACE,GAAG,CAAC,EAAE;QAC1BR,IAAI,CAACS,GAAG,CAACH,GAAG,CAACI,GAAG,CAAC;MACnB,CAAC,MAAM;QACLV,IAAI,CAACW,MAAM,CAACL,GAAG,CAACI,GAAG,CAAC;MACtB;IACF;IACA,MAAME,IAAI,GAAGR,IAAI,CAACS,EAAE,CAAC,CAAC,CAAC,CAAC;IACxB,IAAID,IAAI,KAAKE,SAAS,IAAIV,IAAI,CAACW,MAAM,GAAGjB,QAAQ,EAAE;MAChD,OAAOE,IAAI;IACb;IACAG,KAAK,GAAGS,IAAI,CAACI,GAAG;EAClB;AACF,CAAC,CAAC;AAeJ;;;;;;;AAOA,OAAO,MAAMC,aAAa,GAAgBC,IAMzC,IAIC9B,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAMoB,MAAM,GAAG,OAAO7B,MAAM,CAAC8B,UAAU,CAACF,IAAI,CAACG,SAAS,CAACC,QAAQ,CAAC;EAChE,MAAMtB,IAAI,GAAG,OAAOJ,QAAQ,CAACsB,IAAI,CAACrB,MAAM,EAAEqB,IAAI,CAACpB,QAAQ,CAAC;EACxD,MAAMyB,OAAO,GAAGrC,KAAK,CAACsC,IAAI,CAAC,IAAIvB,GAAG,CAACkB,MAAM,CAAC,CAAC,CAACM,MAAM,CAAEf,GAAG,IAAK,CAACV,IAAI,CAAC0B,GAAG,CAAChB,GAAG,CAAC,CAAC;EAC3E,MAAMiB,OAAO,GAAmB,EAAE;EAClC,MAAMC,OAAO,GAA4C,EAAE;EAC3D,KAAK,MAAMC,KAAK,IAAI1C,GAAG,CAAC2C,QAAQ,CAACP,OAAO,EAAEL,IAAI,CAACpB,QAAQ,CAAC,EAAE;IACxD,MAAMiC,OAAO,GAAsC5C,GAAG,CAAC6C,GAAG,CACxDH,KAAK,EACJnB,GAAG,KAAM;MAAEuB,MAAM,EAAEzC,UAAU,CAAC0C,MAAM,CAAC;QAAExB;MAAG,CAAE,CAAC;MAAEM,GAAG,EAAEvB;IAAW,CAAE,CAAC,CACpE;IACD,MAAM0C,SAAS,GAAG,OAAO5C,OAAO,CAAC;MAC/B6C,IAAI,EAAElB,IAAI,CAACmB,IAAI,CAACC,KAAK;MACrBC,OAAO,EAAErB,IAAI,CAACmB,IAAI,CAACE,OAAO;MAC1BC,KAAK,EAAEtB,IAAI,CAACsB,KAAK;MACjBT;KACD,CAAC;IACFJ,OAAO,CAACc,IAAI,CAAC,GAAGN,SAAS,CAACO,IAAI,CAAC;IAC/Bd,OAAO,CAACa,IAAI,CACV,GAAGN,SAAS,CAACQ,MAAM,CAACX,GAAG,CAAEY,CAAC,KAAM;MAAElC,GAAG,EAAEkC,CAAC,CAAClC,GAAG;MAAEmC,MAAM,EAAED,CAAC,CAACC;IAAM,CAAE,CAAC,CAAC,CACnE;EACH;EACA,MAAMC,UAAU,GAAG,IAAI7C,GAAG,CAACkB,MAAM,CAAC;EAClC,MAAM4B,WAAW,GAAG,IAAI9C,GAAG,CAAC0B,OAAO,CAAC;EACpC,OAAO;IACLqB,MAAM,EAAE;MAAE7B,MAAM,EAAE2B,UAAU,CAACG,IAAI;MAAEtB,OAAO;MAAEC;IAAO,CAAE;IACrDsB,MAAM,EAAGxC,GAAG,IACV,CAACV,IAAI,CAAC0B,GAAG,CAAChB,GAAG,CAAC,KAAKqC,WAAW,CAACrB,GAAG,CAAChB,GAAG,CAAC,IAAI,CAACoC,UAAU,CAACpB,GAAG,CAAChB,GAAG,CAAC;GAClE;AACH,CAAC,CAAC;AAEJ;;;;;;;;;;;;;;;AAeA,OAAO,MAAMyC,UAAU,GAAGA,CACxBC,KAAoB,EACpBC,EAAc,EACdH,MAAiC,KAEjC9D,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAM4C,MAAM,GAAG,OAAOS,KAAK,CAACE,UAAU,CAACD,EAAE,CAAC;EAC1C,OAAOjE,MAAM,CAACmE,OAAO,CACnBZ,MAAM,CAAClB,MAAM,CAAEmB,CAAC,IAAKM,MAAM,CAACN,CAAC,CAAClC,GAAG,CAAC,CAAC,EAClCkC,CAAC,IAAKQ,KAAK,CAACI,WAAW,CAAC;IAAE,GAAGH,EAAE;IAAE3C,GAAG,EAAEkC,CAAC,CAAClC;EAAG,CAAE,CAAC,EAC/C;IAAE+C,OAAO,EAAE;EAAI,CAAE,CAClB;AACH,CAAC,CAAC,CAACC,IAAI,CACLtE,MAAM,CAACuE,KAAK,CAAEC,KAAK,IACjBxE,MAAM,CAACyE,UAAU,CACf,iFAAiF,EACjF;EAAEC,IAAI,EAAET,EAAE,CAACS,IAAI;EAAEzB,IAAI,EAAEgB,EAAE,CAAChB,IAAI;EAAEQ,MAAM,EAAEe,KAAK,CAACG;AAAO,CAAE,CACxD,CACF,CACF;AAWH;;;;;AAKA,OAAO,MAAMC,aAAa,GACxBC,OAAyC,IAUzC7E,MAAM,CAAC8E,MAAM,CACX9E,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAMsD,EAAE,GAAG;IAAES,IAAI,EAAEG,OAAO,CAACpE,MAAM,CAACiE,IAAI;IAAEzB,IAAI,EAAE4B,OAAO,CAAC5B,IAAI,CAAC8B;EAAI,CAAE;EACjE,MAAM3B,KAAK,GAAG,OAAOyB,OAAO,CAACb,KAAK,CAACgB,YAAY,CAACf,EAAE,CAAC;EACnD,MAAM;IAAEL,MAAM;IAAEE;EAAM,CAAE,GAAG,OAAOjC,aAAa,CAAC;IAC9CpB,MAAM,EAAEoE,OAAO,CAACpE,MAAM;IACtBwC,IAAI,EAAE4B,OAAO,CAAC5B,IAAI;IAClBhB,SAAS,EAAE4C,OAAO,CAAC5B,IAAI,CAAChB,SAAS;IACjCmB,KAAK;IACL1C,QAAQ,EAAEmE,OAAO,CAACnE,QAAQ,IAAIJ;GAC/B,CAAC;EACF,OAAOyD,UAAU,CAACc,OAAO,CAACb,KAAK,EAAEC,EAAE,EAAEH,MAAM,CAAC;EAC5C,OAAOF,MAAM;AACf,CAAC,CAAC,CACH","ignoreList":[]}