@kairos-es/feed 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +109 -2
- package/dist/cjs/Feed.js +6 -0
- package/dist/cjs/Feed.js.map +1 -0
- package/dist/cjs/FeedSink.js +54 -0
- package/dist/cjs/FeedSink.js.map +1 -0
- package/dist/cjs/FeedSyncStore.js +10 -0
- package/dist/cjs/FeedSyncStore.js.map +1 -0
- package/dist/cjs/deliver.js +86 -0
- package/dist/cjs/deliver.js.map +1 -0
- package/dist/cjs/errors.js +114 -0
- package/dist/cjs/errors.js.map +1 -0
- package/dist/cjs/identifiers.js +63 -0
- package/dist/cjs/identifiers.js.map +1 -0
- package/dist/cjs/inMemoryFeed.js +328 -0
- package/dist/cjs/inMemoryFeed.js.map +1 -0
- package/dist/cjs/inMemorySink.js +142 -0
- package/dist/cjs/inMemorySink.js.map +1 -0
- package/dist/cjs/index.js +184 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/inspect.js +68 -0
- package/dist/cjs/inspect.js.map +1 -0
- package/dist/cjs/readiness.js +100 -0
- package/dist/cjs/readiness.js.map +1 -0
- package/dist/cjs/reconcile.js +164 -0
- package/dist/cjs/reconcile.js.map +1 -0
- package/dist/cjs/runFeedSync.js +603 -0
- package/dist/cjs/runFeedSync.js.map +1 -0
- package/dist/cjs/tuning.js +60 -0
- package/dist/cjs/tuning.js.map +1 -0
- package/dist/dts-cjs/Feed.d.ts +99 -0
- package/dist/dts-cjs/Feed.d.ts.map +1 -0
- package/dist/dts-cjs/FeedSink.d.ts +189 -0
- package/dist/dts-cjs/FeedSink.d.ts.map +1 -0
- package/dist/dts-cjs/FeedSyncStore.d.ts +281 -0
- package/dist/dts-cjs/FeedSyncStore.d.ts.map +1 -0
- package/dist/dts-cjs/deliver.d.ts +16 -0
- package/dist/dts-cjs/deliver.d.ts.map +1 -0
- package/dist/dts-cjs/errors.d.ts +131 -0
- package/dist/dts-cjs/errors.d.ts.map +1 -0
- package/dist/dts-cjs/identifiers.d.ts +67 -0
- package/dist/dts-cjs/identifiers.d.ts.map +1 -0
- package/dist/dts-cjs/inMemoryFeed.d.ts +32 -0
- package/dist/dts-cjs/inMemoryFeed.d.ts.map +1 -0
- package/dist/dts-cjs/inMemorySink.d.ts +61 -0
- package/dist/dts-cjs/inMemorySink.d.ts.map +1 -0
- package/dist/dts-cjs/index.d.ts +63 -0
- package/dist/dts-cjs/index.d.ts.map +1 -0
- package/dist/dts-cjs/inspect.d.ts +71 -0
- package/dist/dts-cjs/inspect.d.ts.map +1 -0
- package/dist/dts-cjs/package.json +3 -0
- package/dist/dts-cjs/readiness.d.ts +84 -0
- package/dist/dts-cjs/readiness.d.ts.map +1 -0
- package/dist/dts-cjs/reconcile.d.ts +68 -0
- package/dist/dts-cjs/reconcile.d.ts.map +1 -0
- package/dist/dts-cjs/runFeedSync.d.ts +220 -0
- package/dist/dts-cjs/runFeedSync.d.ts.map +1 -0
- package/dist/dts-cjs/tuning.d.ts +40 -0
- package/dist/dts-cjs/tuning.d.ts.map +1 -0
- package/dist/dts-esm/Feed.d.ts +99 -0
- package/dist/dts-esm/Feed.d.ts.map +1 -0
- package/dist/dts-esm/FeedSink.d.ts +189 -0
- package/dist/dts-esm/FeedSink.d.ts.map +1 -0
- package/dist/dts-esm/FeedSyncStore.d.ts +281 -0
- package/dist/dts-esm/FeedSyncStore.d.ts.map +1 -0
- package/dist/dts-esm/deliver.d.ts +16 -0
- package/dist/dts-esm/deliver.d.ts.map +1 -0
- package/dist/dts-esm/errors.d.ts +131 -0
- package/dist/dts-esm/errors.d.ts.map +1 -0
- package/dist/dts-esm/identifiers.d.ts +67 -0
- package/dist/dts-esm/identifiers.d.ts.map +1 -0
- package/dist/dts-esm/inMemoryFeed.d.ts +32 -0
- package/dist/dts-esm/inMemoryFeed.d.ts.map +1 -0
- package/dist/dts-esm/inMemorySink.d.ts +61 -0
- package/dist/dts-esm/inMemorySink.d.ts.map +1 -0
- package/dist/dts-esm/index.d.ts +63 -0
- package/dist/dts-esm/index.d.ts.map +1 -0
- package/dist/dts-esm/inspect.d.ts +71 -0
- package/dist/dts-esm/inspect.d.ts.map +1 -0
- package/dist/dts-esm/package.json +3 -0
- package/dist/dts-esm/readiness.d.ts +84 -0
- package/dist/dts-esm/readiness.d.ts.map +1 -0
- package/dist/dts-esm/reconcile.d.ts +68 -0
- package/dist/dts-esm/reconcile.d.ts.map +1 -0
- package/dist/dts-esm/runFeedSync.d.ts +220 -0
- package/dist/dts-esm/runFeedSync.d.ts.map +1 -0
- package/dist/dts-esm/tuning.d.ts +40 -0
- package/dist/dts-esm/tuning.d.ts.map +1 -0
- package/dist/esm/Feed.js +2 -0
- package/dist/esm/Feed.js.map +1 -0
- package/dist/esm/FeedSink.js +46 -0
- package/dist/esm/FeedSink.js.map +1 -0
- package/dist/esm/FeedSyncStore.js +3 -0
- package/dist/esm/FeedSyncStore.js.map +1 -0
- package/dist/esm/deliver.js +78 -0
- package/dist/esm/deliver.js.map +1 -0
- package/dist/esm/errors.js +98 -0
- package/dist/esm/errors.js.map +1 -0
- package/dist/esm/identifiers.js +56 -0
- package/dist/esm/identifiers.js.map +1 -0
- package/dist/esm/inMemoryFeed.js +320 -0
- package/dist/esm/inMemoryFeed.js.map +1 -0
- package/dist/esm/inMemorySink.js +134 -0
- package/dist/esm/inMemorySink.js.map +1 -0
- package/dist/esm/index.js +62 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/inspect.js +58 -0
- package/dist/esm/inspect.js.map +1 -0
- package/dist/esm/package.json +4 -0
- package/dist/esm/readiness.js +92 -0
- package/dist/esm/readiness.js.map +1 -0
- package/dist/esm/reconcile.js +153 -0
- package/dist/esm/reconcile.js.map +1 -0
- package/dist/esm/runFeedSync.js +595 -0
- package/dist/esm/runFeedSync.js.map +1 -0
- package/dist/esm/tuning.js +51 -0
- package/dist/esm/tuning.js.map +1 -0
- package/package.json +34 -3
- package/src/Feed.ts +111 -0
- package/src/FeedSink.ts +130 -0
- package/src/FeedSyncStore.ts +326 -0
- package/src/deliver.ts +108 -0
- package/src/errors.ts +134 -0
- package/src/identifiers.ts +92 -0
- package/src/inMemoryFeed.ts +582 -0
- package/src/inMemorySink.ts +215 -0
- package/src/index.ts +136 -0
- package/src/inspect.ts +114 -0
- package/src/readiness.ts +137 -0
- package/src/reconcile.ts +224 -0
- package/src/runFeedSync.ts +882 -0
- package/src/tuning.ts +65 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The in-memory sink: a `FeedSink` over a `Ref` of documents, with an optional
|
|
3
|
+
* `FeedResync` and `FeedReconcile`, and the ORACLE the sink law suite holds every
|
|
4
|
+
* other sink against.
|
|
5
|
+
*
|
|
6
|
+
* It keeps the laws by construction: an upsert replaces the whole document, a
|
|
7
|
+
* delete of an absent key is a no-op, so the same batch applied twice leaves it as
|
|
8
|
+
* once; a batch is visible the moment `apply` returns, so `confirm` has nothing to
|
|
9
|
+
* wait for; a resync target is a separate map that `publish` swaps in whole, so a
|
|
10
|
+
* published resync holds exactly what was loaded into it, and a load into a target
|
|
11
|
+
* a later `begin` dropped fails rather than recreating it. Each `begin` names its
|
|
12
|
+
* target by a counter the sink's own state holds, so two workers sharing one sink
|
|
13
|
+
* value — the way two processes share one real index — never share a target, and
|
|
14
|
+
* the name is the same on every run of a test.
|
|
15
|
+
*
|
|
16
|
+
* Its two optional parts are switched on by option rather than always present,
|
|
17
|
+
* because which parts a sink has CHANGES what the driver does — a sink with no
|
|
18
|
+
* `resync` is resynced in place — and a test of either path needs a sink that
|
|
19
|
+
* lacks the other. It also records every batch it is handed, live or into a
|
|
20
|
+
* target, so a case can say how many sends a change cost.
|
|
21
|
+
*
|
|
22
|
+
* It lives for as long as the value is held, independently of any feed, which is
|
|
23
|
+
* what lets a test keep one sink across two in-memory feeds and watch the second
|
|
24
|
+
* feed's fresh generation force a resync.
|
|
25
|
+
*/
|
|
26
|
+
import { Effect, Ref, Stream } from 'effect';
|
|
27
|
+
import { SinkRetryable } from "./errors.js";
|
|
28
|
+
const LIVE = 'live';
|
|
29
|
+
const applyTo = (documents, changes) => {
|
|
30
|
+
const next = new Map(documents);
|
|
31
|
+
for (const change of changes) {
|
|
32
|
+
if (change._tag === 'Upsert') {
|
|
33
|
+
next.set(change.key, change.doc);
|
|
34
|
+
} else {
|
|
35
|
+
next.delete(change.key);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return next;
|
|
39
|
+
};
|
|
40
|
+
/** What a load into, or a publish of, a target a later `begin` dropped fails with. */
|
|
41
|
+
const droppedTarget = name => new SinkRetryable({
|
|
42
|
+
reason: `the resync target '${name}' was dropped by a later begin`
|
|
43
|
+
});
|
|
44
|
+
/** The resync target's name for one attempt at `generation`; see the module doc. */
|
|
45
|
+
const targetName = (generation, attempt) => `${LIVE}__g${generation}_${attempt}`;
|
|
46
|
+
/** Build an in-memory sink; see the module doc. */
|
|
47
|
+
export const makeInMemorySink = options => Effect.gen(function* () {
|
|
48
|
+
const state = yield* Ref.make({
|
|
49
|
+
live: new Map(),
|
|
50
|
+
targets: new Map(),
|
|
51
|
+
batches: [],
|
|
52
|
+
published: [],
|
|
53
|
+
attempts: 0
|
|
54
|
+
});
|
|
55
|
+
const record = (target, changes) => Effect.flatten(Ref.modify(state, s => {
|
|
56
|
+
const batches = [...s.batches, {
|
|
57
|
+
target,
|
|
58
|
+
changes
|
|
59
|
+
}];
|
|
60
|
+
const receipt = {
|
|
61
|
+
target,
|
|
62
|
+
batch: batches.length
|
|
63
|
+
};
|
|
64
|
+
if (target === LIVE) {
|
|
65
|
+
return [Effect.succeed(receipt), {
|
|
66
|
+
...s,
|
|
67
|
+
batches,
|
|
68
|
+
live: applyTo(s.live, changes)
|
|
69
|
+
}];
|
|
70
|
+
}
|
|
71
|
+
const built = s.targets.get(target);
|
|
72
|
+
// Dropped by a later begin: recreating it here is what would let a
|
|
73
|
+
// stale worker's publish swap a fragment in. `FeedResync.load` says so.
|
|
74
|
+
if (built === undefined) {
|
|
75
|
+
return [Effect.fail(droppedTarget(target)), s];
|
|
76
|
+
}
|
|
77
|
+
return [Effect.succeed(receipt), {
|
|
78
|
+
...s,
|
|
79
|
+
batches,
|
|
80
|
+
targets: new Map(s.targets).set(target, applyTo(built, changes))
|
|
81
|
+
}];
|
|
82
|
+
}));
|
|
83
|
+
const resync = {
|
|
84
|
+
// Every other target is dropped: whatever attempt left it, it will never be
|
|
85
|
+
// published now, and its own publish must fail rather than swap it in.
|
|
86
|
+
begin: generation => Ref.modify(state, s => {
|
|
87
|
+
const attempts = s.attempts + 1;
|
|
88
|
+
const name = targetName(generation, attempts);
|
|
89
|
+
return [{
|
|
90
|
+
generation,
|
|
91
|
+
name
|
|
92
|
+
}, {
|
|
93
|
+
...s,
|
|
94
|
+
attempts,
|
|
95
|
+
targets: new Map([[name, new Map()]])
|
|
96
|
+
}];
|
|
97
|
+
}),
|
|
98
|
+
load: (target, batch) => record(target.name, batch),
|
|
99
|
+
publish: target => Effect.flatten(Ref.modify(state, s => {
|
|
100
|
+
const built = s.targets.get(target.name);
|
|
101
|
+
if (built === undefined) {
|
|
102
|
+
return [Effect.fail(droppedTarget(target.name)), s];
|
|
103
|
+
}
|
|
104
|
+
const targets = new Map(s.targets);
|
|
105
|
+
targets.delete(target.name);
|
|
106
|
+
return [Effect.void, {
|
|
107
|
+
...s,
|
|
108
|
+
live: built,
|
|
109
|
+
targets,
|
|
110
|
+
published: [...s.published, target.generation]
|
|
111
|
+
}];
|
|
112
|
+
}))
|
|
113
|
+
};
|
|
114
|
+
const sink = {
|
|
115
|
+
name: options.name,
|
|
116
|
+
apply: batch => record(LIVE, batch),
|
|
117
|
+
confirm: () => Effect.void,
|
|
118
|
+
...(options.resync === true ? {
|
|
119
|
+
resync
|
|
120
|
+
} : {}),
|
|
121
|
+
...(options.reconcile === true ? {
|
|
122
|
+
reconcile: {
|
|
123
|
+
listKeys: Stream.unwrap(Effect.map(Ref.get(state), s => Stream.fromIterable(Array.from(s.live.keys()))))
|
|
124
|
+
}
|
|
125
|
+
} : {})
|
|
126
|
+
};
|
|
127
|
+
return {
|
|
128
|
+
sink,
|
|
129
|
+
documents: Effect.map(Ref.get(state), s => s.live),
|
|
130
|
+
batches: Effect.map(Ref.get(state), s => s.batches),
|
|
131
|
+
published: Effect.map(Ref.get(state), s => s.published)
|
|
132
|
+
};
|
|
133
|
+
});
|
|
134
|
+
//# sourceMappingURL=inMemorySink.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inMemorySink.js","names":["Effect","Ref","Stream","SinkRetryable","LIVE","applyTo","documents","changes","next","Map","change","_tag","set","key","doc","delete","droppedTarget","name","reason","targetName","generation","attempt","makeInMemorySink","options","gen","state","make","live","targets","batches","published","attempts","record","target","flatten","modify","s","receipt","batch","length","succeed","built","get","undefined","fail","resync","begin","load","publish","void","sink","apply","confirm","reconcile","listKeys","unwrap","map","fromIterable","Array","from","keys"],"sources":["../../src/inMemorySink.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;AAyBA,SAASA,MAAM,EAAEC,GAAG,EAAEC,MAAM,QAAQ,QAAQ;AAE5C,SAASC,aAAa,QAAQ,aAAU;AA8CxC,MAAMC,IAAI,GAAG,MAAM;AAEnB,MAAMC,OAAO,GAAGA,CACdC,SAAkC,EAClCC,OAA6C,KAClB;EAC3B,MAAMC,IAAI,GAAG,IAAIC,GAAG,CAACH,SAAS,CAAC;EAC/B,KAAK,MAAMI,MAAM,IAAIH,OAAO,EAAE;IAC5B,IAAIG,MAAM,CAACC,IAAI,KAAK,QAAQ,EAAE;MAC5BH,IAAI,CAACI,GAAG,CAACF,MAAM,CAACG,GAAG,EAAEH,MAAM,CAACI,GAAG,CAAC;IAClC,CAAC,MAAM;MACLN,IAAI,CAACO,MAAM,CAACL,MAAM,CAACG,GAAG,CAAC;IACzB;EACF;EACA,OAAOL,IAAI;AACb,CAAC;AAED;AACA,MAAMQ,aAAa,GAAIC,IAAY,IACjC,IAAId,aAAa,CAAC;EAChBe,MAAM,EAAE,sBAAsBD,IAAI;CACnC,CAAC;AAEJ;AACA,MAAME,UAAU,GAAGA,CAACC,UAAsB,EAAEC,OAAe,KACzD,GAAGjB,IAAI,MAAMgB,UAAU,IAAIC,OAAO,EAAE;AAEtC;AACA,OAAO,MAAMC,gBAAgB,GAC3BC,OAA4B,IAE5BvB,MAAM,CAACwB,GAAG,CAAC,aAAS;EAClB,MAAMC,KAAK,GAAG,OAAOxB,GAAG,CAACyB,IAAI,CAAe;IAC1CC,IAAI,EAAE,IAAIlB,GAAG,EAAE;IACfmB,OAAO,EAAE,IAAInB,GAAG,EAAE;IAClBoB,OAAO,EAAE,EAAE;IACXC,SAAS,EAAE,EAAE;IACbC,QAAQ,EAAE;GACX,CAAC;EAEF,MAAMC,MAAM,GAAGA,CACbC,MAAc,EACd1B,OAA6C,KAE7CP,MAAM,CAACkC,OAAO,CACZjC,GAAG,CAACkC,MAAM,CACRV,KAAK,EAEHW,CAAC,IACgE;IACjE,MAAMP,OAAO,GAAG,CAAC,GAAGO,CAAC,CAACP,OAAO,EAAE;MAAEI,MAAM;MAAE1B;IAAO,CAAE,CAAC;IACnD,MAAM8B,OAAO,GAAG;MAAEJ,MAAM;MAAEK,KAAK,EAAET,OAAO,CAACU;IAAM,CAAE;IACjD,IAAIN,MAAM,KAAK7B,IAAI,EAAE;MACnB,OAAO,CACLJ,MAAM,CAACwC,OAAO,CAACH,OAAO,CAAC,EACvB;QAAE,GAAGD,CAAC;QAAEP,OAAO;QAAEF,IAAI,EAAEtB,OAAO,CAAC+B,CAAC,CAACT,IAAI,EAAEpB,OAAO;MAAC,CAAE,CAClD;IACH;IACA,MAAMkC,KAAK,GAAGL,CAAC,CAACR,OAAO,CAACc,GAAG,CAACT,MAAM,CAAC;IACnC;IACA;IACA,IAAIQ,KAAK,KAAKE,SAAS,EAAE;MACvB,OAAO,CAAC3C,MAAM,CAAC4C,IAAI,CAAC5B,aAAa,CAACiB,MAAM,CAAC,CAAC,EAAEG,CAAC,CAAC;IAChD;IACA,OAAO,CACLpC,MAAM,CAACwC,OAAO,CAACH,OAAO,CAAC,EACvB;MACE,GAAGD,CAAC;MACJP,OAAO;MACPD,OAAO,EAAE,IAAInB,GAAG,CAAC2B,CAAC,CAACR,OAAO,CAAC,CAAChB,GAAG,CAC7BqB,MAAM,EACN5B,OAAO,CAACoC,KAAK,EAAElC,OAAO,CAAC;KAE1B,CACF;EACH,CAAC,CACF,CACF;EAEH,MAAMsC,MAAM,GAAmC;IAC7C;IACA;IACAC,KAAK,EAAG1B,UAAU,IAChBnB,GAAG,CAACkC,MAAM,CAACV,KAAK,EAAGW,CAAC,IAAkC;MACpD,MAAML,QAAQ,GAAGK,CAAC,CAACL,QAAQ,GAAG,CAAC;MAC/B,MAAMd,IAAI,GAAGE,UAAU,CAACC,UAAU,EAAEW,QAAQ,CAAC;MAC7C,OAAO,CACL;QAAEX,UAAU;QAAEH;MAAI,CAAE,EACpB;QAAE,GAAGmB,CAAC;QAAEL,QAAQ;QAAEH,OAAO,EAAE,IAAInB,GAAG,CAAC,CAAC,CAACQ,IAAI,EAAE,IAAIR,GAAG,EAAE,CAAC,CAAC;MAAC,CAAE,CAC1D;IACH,CAAC,CAAC;IACJsC,IAAI,EAAEA,CAACd,MAAM,EAAEK,KAAK,KAAKN,MAAM,CAACC,MAAM,CAAChB,IAAI,EAAEqB,KAAK,CAAC;IACnDU,OAAO,EAAGf,MAAM,IACdjC,MAAM,CAACkC,OAAO,CACZjC,GAAG,CAACkC,MAAM,CACRV,KAAK,EACJW,CAAC,IAAwD;MACxD,MAAMK,KAAK,GAAGL,CAAC,CAACR,OAAO,CAACc,GAAG,CAACT,MAAM,CAAChB,IAAI,CAAC;MACxC,IAAIwB,KAAK,KAAKE,SAAS,EAAE;QACvB,OAAO,CAAC3C,MAAM,CAAC4C,IAAI,CAAC5B,aAAa,CAACiB,MAAM,CAAChB,IAAI,CAAC,CAAC,EAAEmB,CAAC,CAAC;MACrD;MACA,MAAMR,OAAO,GAAG,IAAInB,GAAG,CAAC2B,CAAC,CAACR,OAAO,CAAC;MAClCA,OAAO,CAACb,MAAM,CAACkB,MAAM,CAAChB,IAAI,CAAC;MAC3B,OAAO,CACLjB,MAAM,CAACiD,IAAI,EACX;QACE,GAAGb,CAAC;QACJT,IAAI,EAAEc,KAAK;QACXb,OAAO;QACPE,SAAS,EAAE,CAAC,GAAGM,CAAC,CAACN,SAAS,EAAEG,MAAM,CAACb,UAAU;OAC9C,CACF;IACH,CAAC,CACF;GAEN;EAED,MAAM8B,IAAI,GAAiC;IACzCjC,IAAI,EAAEM,OAAO,CAACN,IAAI;IAClBkC,KAAK,EAAGb,KAAK,IAAKN,MAAM,CAAC5B,IAAI,EAAEkC,KAAK,CAAC;IACrCc,OAAO,EAAEA,CAAA,KAAMpD,MAAM,CAACiD,IAAI;IAC1B,IAAI1B,OAAO,CAACsB,MAAM,KAAK,IAAI,GAAG;MAAEA;IAAM,CAAE,GAAG,EAAE,CAAC;IAC9C,IAAItB,OAAO,CAAC8B,SAAS,KAAK,IAAI,GAC1B;MACEA,SAAS,EAAE;QACTC,QAAQ,EAAEpD,MAAM,CAACqD,MAAM,CACrBvD,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAC3BlC,MAAM,CAACuD,YAAY,CAACC,KAAK,CAACC,IAAI,CAACvB,CAAC,CAACT,IAAI,CAACiC,IAAI,EAAE,CAAC,CAAC,CAC/C;;KAGN,GACD,EAAE;GACP;EAED,OAAO;IACLV,IAAI;IACJ5C,SAAS,EAAEN,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAAKA,CAAC,CAACT,IAAI,CAAC;IACpDE,OAAO,EAAE7B,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAAKA,CAAC,CAACP,OAAO,CAAC;IACrDC,SAAS,EAAE9B,MAAM,CAACwD,GAAG,CAACvD,GAAG,CAACyC,GAAG,CAACjB,KAAK,CAAC,EAAGW,CAAC,IAAKA,CAAC,CAACN,SAAS;GACzD;AACH,CAAC,CAAC","ignoreList":[]}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @kairos-es/feed — a state-only change feed: the newest state of each record a
|
|
3
|
+
* read model touches, written inside that read model's own commit, and copied by
|
|
4
|
+
* a sync worker to a sink that has no transaction of its own — a search index, a
|
|
5
|
+
* cache, a mirror (ADR-0019).
|
|
6
|
+
*
|
|
7
|
+
* The feed copies STATE. It never tells anyone each thing that happened; that is
|
|
8
|
+
* `@kairos-es/automation`'s job, and the two are deliberately different shapes: an
|
|
9
|
+
* automation's only progress record is the log, while a feed keeps a cursor, a
|
|
10
|
+
* lease and a parked list beside its rows, because what it mirrors is state and
|
|
11
|
+
* its progress is legitimately a cursor.
|
|
12
|
+
*
|
|
13
|
+
* What is here, and where each part's argument lives:
|
|
14
|
+
*
|
|
15
|
+
* - The identifiers (`identifiers.ts`) and the error vocabulary (`errors.ts`).
|
|
16
|
+
* - The feed's write and read sides, `BatchedFeedWriter` / `FeedWriter` and
|
|
17
|
+
* `FeedSource` (`Feed.ts`): why the feed is written inside the commit, and what
|
|
18
|
+
* "coalesced, latest wins" means for a stored row.
|
|
19
|
+
* - The sink port, `FeedSink` with its optional `FeedResync` and `FeedReconcile`
|
|
20
|
+
* (`FeedSink.ts`): what a sink must keep, and why a sink with neither optional
|
|
21
|
+
* part is refused.
|
|
22
|
+
* - The sync store port, `FeedSyncStore` (`FeedSyncStore.ts`): the cursor, the
|
|
23
|
+
* parked list and the worker lease, and the exact contract a backend implements.
|
|
24
|
+
* - The driver, `runFeedSync` (`runFeedSync.ts`): the lifecycle, the rules it
|
|
25
|
+
* keeps, resync, and its own supervision.
|
|
26
|
+
* - `FeedReadiness` and `materialisationCaughtUp` (`readiness.ts`): why a
|
|
27
|
+
* rebuild's publish can wait for the materialisation, and what it waits on.
|
|
28
|
+
* - `reconcileFeed` (`reconcile.ts`), and `inspectFeed` / `retryParked` /
|
|
29
|
+
* `removeSink` (`inspect.ts`).
|
|
30
|
+
* - The in-memory feed (`inMemoryFeed.ts`) and sink (`inMemorySink.ts`): the parity
|
|
31
|
+
* oracles, and what the in-memory feed refuses.
|
|
32
|
+
*
|
|
33
|
+
* Wiring, end to end, in memory:
|
|
34
|
+
*
|
|
35
|
+
* ```ts
|
|
36
|
+
* const feed = yield* makeInMemoryFeed({ feed: FeedName.make('course-search'), doc: CourseDoc })
|
|
37
|
+
* const store = yield* withInMemoryFeed(yield* makeInMemoryProjectionStore, feed, key)
|
|
38
|
+
*
|
|
39
|
+
* // In the read model's `apply`, as the LAST step of the batch:
|
|
40
|
+
* const apply = (batch) =>
|
|
41
|
+
* feed.write(Arr.lastNonEmpty(batch).sequenced.position, (writer) =>
|
|
42
|
+
* Effect.forEach(touchedCourses(batch), (course) =>
|
|
43
|
+
* writer.upsert(FeedKey.make(course.id), toDoc(course)),
|
|
44
|
+
* ),
|
|
45
|
+
* )
|
|
46
|
+
*
|
|
47
|
+
* // Beside the projections, in a fibre of its own:
|
|
48
|
+
* const { sink } = yield* makeInMemorySink<CourseDoc>({ name: SinkName.make('search'), resync: true })
|
|
49
|
+
* yield* Effect.forkScoped(runFeedSync({ source: feed.source, sink, store: feed.syncStore }))
|
|
50
|
+
* ```
|
|
51
|
+
*/
|
|
52
|
+
export { CursorSuperseded, FeedSyncStalled, FeedUnreadable, FeedWriteFailed, LeaseHeld, LeaseLost, SinkHalted, SinkRejected, SinkRetryable } from "./errors.js";
|
|
53
|
+
export { changeOf, FeedChange } from "./FeedSink.js";
|
|
54
|
+
export { sameCursor } from "./FeedSyncStore.js";
|
|
55
|
+
export { FEED_ORIGIN, FeedKey, FeedName, FeedSeq, Generation, LeaseHolder, SinkName } from "./identifiers.js";
|
|
56
|
+
export { makeInMemoryFeed, withInMemoryFeed } from "./inMemoryFeed.js";
|
|
57
|
+
export { makeInMemorySink } from "./inMemorySink.js";
|
|
58
|
+
export { inspectFeed, removeSink, retryParked } from "./inspect.js";
|
|
59
|
+
export { materialisationCaughtUp } from "./readiness.js";
|
|
60
|
+
export { reconcileFeed } from "./reconcile.js";
|
|
61
|
+
export { runFeedSync } from "./runFeedSync.js";
|
|
62
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","names":["CursorSuperseded","FeedSyncStalled","FeedUnreadable","FeedWriteFailed","LeaseHeld","LeaseLost","SinkHalted","SinkRejected","SinkRetryable","changeOf","FeedChange","sameCursor","FEED_ORIGIN","FeedKey","FeedName","FeedSeq","Generation","LeaseHolder","SinkName","makeInMemoryFeed","withInMemoryFeed","makeInMemorySink","inspectFeed","removeSink","retryParked","materialisationCaughtUp","reconcileFeed","runFeedSync"],"sources":["../../src/index.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoDA,SACEA,gBAAgB,EAChBC,eAAe,EACfC,cAAc,EACdC,eAAe,EACfC,SAAS,EACTC,SAAS,EACTC,UAAU,EACVC,YAAY,EACZC,aAAa,QACR,aAAU;AAOjB,SACEC,QAAQ,EACRC,UAAU,QAML,eAAY;AACnB,SASEC,UAAU,QACL,oBAAiB;AACxB,SACEC,WAAW,EACXC,OAAO,EACPC,QAAQ,EACRC,OAAO,EAEPC,UAAU,EACVC,WAAW,EACXC,QAAQ,QACH,kBAAe;AACtB,SAGEC,gBAAgB,EAChBC,gBAAgB,QACX,mBAAgB;AACvB,SAKEC,gBAAgB,QACX,mBAAgB;AACvB,SAEEC,WAAW,EACXC,UAAU,EACVC,WAAW,QAEN,cAAW;AAClB,SAIEC,uBAAuB,QAClB,gBAAa;AACpB,SAGEC,aAAa,QACR,gBAAa;AACpB,SAIEC,WAAW,QACN,kBAAe","ignoreList":[]}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `inspectFeed`, `retryParked` and `removeSink`: an operator's view of a feed's
|
|
3
|
+
* sinks, and the two repairs an operator makes by hand.
|
|
4
|
+
*
|
|
5
|
+
* None of them needs the sink's client, because everything they read or change
|
|
6
|
+
* lives in the source database beside the feed: an operator can see a sink's lag,
|
|
7
|
+
* its lease, its parked records and when it last advanced, send a parked record
|
|
8
|
+
* again, and retire a sink, without the credentials of the index concerned. The
|
|
9
|
+
* same choice `@kairos-es/automation` made for its own `inspect`. There is no CLI
|
|
10
|
+
* or UI over any of them; they are effects an application wires into whatever it
|
|
11
|
+
* already has.
|
|
12
|
+
*
|
|
13
|
+
* `lastAdvance` and `removeSink` are a pair. Every sink registered on a feed holds
|
|
14
|
+
* its tombstone collection back, so a sink the application stopped running and
|
|
15
|
+
* never removed lets tombstones pile up for good; `lastAdvance` is how such a sink
|
|
16
|
+
* shows — still, while the head moves — and `removeSink` is what lets collection
|
|
17
|
+
* proceed without it. `FeedSyncStore.ts` owns both rules.
|
|
18
|
+
*/
|
|
19
|
+
import { Effect, Option } from 'effect';
|
|
20
|
+
/** Every sink of `source`'s feed: its cursor, lag, lease and parked records. */
|
|
21
|
+
export const inspectFeed = (source, store) => Effect.gen(function* () {
|
|
22
|
+
const generation = yield* source.generation;
|
|
23
|
+
const description = yield* store.describe(source.feed);
|
|
24
|
+
return {
|
|
25
|
+
feed: source.feed,
|
|
26
|
+
generation,
|
|
27
|
+
head: description.head,
|
|
28
|
+
sinks: description.sinks.map(state => ({
|
|
29
|
+
sink: state.sink,
|
|
30
|
+
generation: Option.map(state.cursor, c => c.generation),
|
|
31
|
+
cursor: state.cursor,
|
|
32
|
+
lastAdvance: state.lastAdvance,
|
|
33
|
+
resyncDue: Option.match(state.cursor, {
|
|
34
|
+
onNone: () => true,
|
|
35
|
+
onSome: c => c.generation !== generation
|
|
36
|
+
}),
|
|
37
|
+
lag: state.pending,
|
|
38
|
+
lease: state.lease,
|
|
39
|
+
parked: state.parked
|
|
40
|
+
}))
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
/**
|
|
44
|
+
* Send one parked record again: delete its park and re-stamp its row's `seq`, so
|
|
45
|
+
* the next pass of every sink on the feed sends it. Succeeds with whether the key
|
|
46
|
+
* was parked at all. Every sink on the feed receives the record again, not only
|
|
47
|
+
* `sink`; an idempotent sink absorbs that.
|
|
48
|
+
*/
|
|
49
|
+
export const retryParked = (store, args) => store.retryParked(args);
|
|
50
|
+
/**
|
|
51
|
+
* Retire `sink` from `feed`: delete its cursor, its lease record and its parked
|
|
52
|
+
* records, so it no longer holds the feed's tombstone collection back. Succeeds
|
|
53
|
+
* with whether the store held anything for it. Fails `LeaseHeld`, removing
|
|
54
|
+
* nothing, while a worker for the sink is running: stop the worker first. A worker
|
|
55
|
+
* started for the sink later registers it again from scratch, and resyncs.
|
|
56
|
+
*/
|
|
57
|
+
export const removeSink = (store, args) => store.removeSink(args);
|
|
58
|
+
//# sourceMappingURL=inspect.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inspect.js","names":["Effect","Option","inspectFeed","source","store","gen","generation","description","describe","feed","head","sinks","map","state","sink","cursor","c","lastAdvance","resyncDue","match","onNone","onSome","lag","pending","lease","parked","retryParked","args","removeSink"],"sources":["../../src/inspect.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;AAkBA,SAAwBA,MAAM,EAAEC,MAAM,QAAQ,QAAQ;AA0CtD;AACA,OAAO,MAAMC,WAAW,GAAGA,CACzBC,MAAqB,EACrBC,KAAoB,KAEpBJ,MAAM,CAACK,GAAG,CAAC,aAAS;EAClB,MAAMC,UAAU,GAAG,OAAOH,MAAM,CAACG,UAAU;EAC3C,MAAMC,WAAW,GAAG,OAAOH,KAAK,CAACI,QAAQ,CAACL,MAAM,CAACM,IAAI,CAAC;EACtD,OAAO;IACLA,IAAI,EAAEN,MAAM,CAACM,IAAI;IACjBH,UAAU;IACVI,IAAI,EAAEH,WAAW,CAACG,IAAI;IACtBC,KAAK,EAAEJ,WAAW,CAACI,KAAK,CAACC,GAAG,CAAEC,KAAK,KAAM;MACvCC,IAAI,EAAED,KAAK,CAACC,IAAI;MAChBR,UAAU,EAAEL,MAAM,CAACW,GAAG,CAACC,KAAK,CAACE,MAAM,EAAGC,CAAC,IAAKA,CAAC,CAACV,UAAU,CAAC;MACzDS,MAAM,EAAEF,KAAK,CAACE,MAAM;MACpBE,WAAW,EAAEJ,KAAK,CAACI,WAAW;MAC9BC,SAAS,EAAEjB,MAAM,CAACkB,KAAK,CAACN,KAAK,CAACE,MAAM,EAAE;QACpCK,MAAM,EAAEA,CAAA,KAAM,IAAI;QAClBC,MAAM,EAAGL,CAAC,IAAKA,CAAC,CAACV,UAAU,KAAKA;OACjC,CAAC;MACFgB,GAAG,EAAET,KAAK,CAACU,OAAO;MAClBC,KAAK,EAAEX,KAAK,CAACW,KAAK;MAClBC,MAAM,EAAEZ,KAAK,CAACY;KACf,CAAC;GACH;AACH,CAAC,CAAC;AAEJ;;;;;;AAMA,OAAO,MAAMC,WAAW,GAAGA,CACzBtB,KAAoB,EACpBuB,IAIC,KAC2CvB,KAAK,CAACsB,WAAW,CAACC,IAAI,CAAC;AAErE;;;;;;;AAOA,OAAO,MAAMC,UAAU,GAAGA,CACxBxB,KAAoB,EACpBuB,IAA0D,KACFvB,KAAK,CAACwB,UAAU,CAACD,IAAI,CAAC","ignoreList":[]}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `FeedReadiness` — when a rebuilt copy may go live — and
|
|
3
|
+
* `materialisationCaughtUp`, the readiness of the common case.
|
|
4
|
+
*
|
|
5
|
+
* ## Why a rebuild has anything to wait for
|
|
6
|
+
*
|
|
7
|
+
* A reset empties the feed and moves its generation on, and the materialisation
|
|
8
|
+
* then replays the log, writing the feed afresh as it goes. The sync worker sees
|
|
9
|
+
* the new generation within a poll interval and starts its resync at once, but the
|
|
10
|
+
* feed it pages through is only as complete as the replay is far along. Reaching
|
|
11
|
+
* the feed's last row says nothing about the replay: it is wherever the replay had
|
|
12
|
+
* got to. So a resync that publishes at the feed's end swaps a complete live copy
|
|
13
|
+
* for whatever fraction of the rebuild the replay had written, and search serves
|
|
14
|
+
* that partial copy until the tail fills the rest in.
|
|
15
|
+
*
|
|
16
|
+
* Only the materialisation knows how far its replay has got, and the feed's ports
|
|
17
|
+
* deliberately name no materialisation, so the driver is TOLD, through an optional
|
|
18
|
+
* readiness signal on `runFeedSync`, and holds the step that makes the rebuild
|
|
19
|
+
* final until the signal says yes. With no signal the resync publishes at the
|
|
20
|
+
* feed's end, as it always did, and a rebuild can serve a partial copy for as long
|
|
21
|
+
* as the replay takes.
|
|
22
|
+
*
|
|
23
|
+
* ## Why it is a function returning an effect
|
|
24
|
+
*
|
|
25
|
+
* "Caught up" has to be relative to a fixed point: the log as it stood when the
|
|
26
|
+
* rebuild began. Judged against the log as it stands at each ask, a read model
|
|
27
|
+
* under steady writes is always a batch or two behind, and the publish would wait
|
|
28
|
+
* for a quiet moment that may never come. So the signal is asked ONCE per resync
|
|
29
|
+
* attempt, as the attempt starts, for the generation that attempt rebuilds, and
|
|
30
|
+
* returns the PROBE the worker then polls — by which time whatever point the probe
|
|
31
|
+
* judges by is fixed. An attempt starts after the reset that moved the generation,
|
|
32
|
+
* so a point taken then is at or after the log as it stood at the reset: the error
|
|
33
|
+
* is towards waiting slightly longer, never towards publishing early.
|
|
34
|
+
*
|
|
35
|
+
* Both halves fail `FeedUnreadable`, the feed's own "a read could not be made":
|
|
36
|
+
* the worker cannot tell a probe that could not read the materialisation's
|
|
37
|
+
* progress from a feed it could not read, and needs to do the same thing about
|
|
38
|
+
* either — end the session and try again after a back-off.
|
|
39
|
+
*/
|
|
40
|
+
import { composeProjections, DcbEventStore, DEFAULT_CATCH_UP_PAGE_SIZE, ReadLimit } from '@kairos-es/core';
|
|
41
|
+
import { Array as Arr, Effect, Option, Stream } from 'effect';
|
|
42
|
+
import { FeedUnreadable } from "./errors.js";
|
|
43
|
+
/**
|
|
44
|
+
* The readiness of the common case: the materialisation's checkpoint has reached
|
|
45
|
+
* the last event its query selects, as the log stood when the resync attempt
|
|
46
|
+
* started. Pass the very read model whose `apply` writes the feed, so its query
|
|
47
|
+
* and its checkpoint are the ones the replay moves.
|
|
48
|
+
*
|
|
49
|
+
* The point is the last event the read model's QUERY selects, and not the log's
|
|
50
|
+
* head, because the runner advances its checkpoint to the positions it is
|
|
51
|
+
* delivered and is never delivered an event its query does not select: a
|
|
52
|
+
* checkpoint judged against a head an unrelated event moved would never arrive.
|
|
53
|
+
* The store hands no cheaper way to that position than reading up to it, so each
|
|
54
|
+
* attempt starts with one paged read of the query's events past the checkpoint —
|
|
55
|
+
* the backlog the replay itself is about to read, a page at a time and never held
|
|
56
|
+
* whole — and each probe after it is one checkpoint read.
|
|
57
|
+
*
|
|
58
|
+
* Generation-blind, deliberately: the read model has one checkpoint whatever the
|
|
59
|
+
* feed's generation, and an attempt whose generation moves on while it waits is
|
|
60
|
+
* abandoned by the worker's own generation check, so the probe never has to know.
|
|
61
|
+
*/
|
|
62
|
+
export const materialisationCaughtUp = readModel => Effect.gen(function* () {
|
|
63
|
+
const log = yield* DcbEventStore;
|
|
64
|
+
const query = composeProjections(readModel.slices).query;
|
|
65
|
+
const limit = ReadLimit.make(DEFAULT_CATCH_UP_PAGE_SIZE);
|
|
66
|
+
const checkpoint = Effect.mapError(readModel.store.readCheckpoint, cause => new FeedUnreadable({
|
|
67
|
+
reason: `the checkpoint of projection '${readModel.store.key.projection}' could not be read`,
|
|
68
|
+
cause
|
|
69
|
+
}));
|
|
70
|
+
// The last position the query selects after `from`, or `from` itself when it
|
|
71
|
+
// selects none: page by page, so the backlog is never held in memory at once.
|
|
72
|
+
const lastSelectedAfter = from => Effect.gen(function* () {
|
|
73
|
+
let last = from;
|
|
74
|
+
for (;;) {
|
|
75
|
+
const response = yield* log.read(query, {
|
|
76
|
+
after: last,
|
|
77
|
+
limit
|
|
78
|
+
});
|
|
79
|
+
const page = yield* Stream.runCollect(response.events);
|
|
80
|
+
const final = Arr.last(page);
|
|
81
|
+
if (Option.isSome(final)) {
|
|
82
|
+
last = final.value.position;
|
|
83
|
+
}
|
|
84
|
+
if (page.length < limit) {
|
|
85
|
+
return last;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
const readiness = () => Effect.map(Effect.flatMap(checkpoint, lastSelectedAfter), target => Effect.map(checkpoint, reached => reached >= target));
|
|
90
|
+
return readiness;
|
|
91
|
+
});
|
|
92
|
+
//# sourceMappingURL=readiness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"readiness.js","names":["composeProjections","DcbEventStore","DEFAULT_CATCH_UP_PAGE_SIZE","ReadLimit","Array","Arr","Effect","Option","Stream","FeedUnreadable","materialisationCaughtUp","readModel","gen","log","query","slices","limit","make","checkpoint","mapError","store","readCheckpoint","cause","reason","key","projection","lastSelectedAfter","from","last","response","read","after","page","runCollect","events","final","isSome","value","position","length","readiness","map","flatMap","target","reached"],"sources":["../../src/readiness.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuCA,SAEEA,kBAAkB,EAClBC,aAAa,EACbC,0BAA0B,EAE1BC,SAAS,QACJ,iBAAiB;AAExB,SAASC,KAAK,IAAIC,GAAG,EAAEC,MAAM,EAAEC,MAAM,EAAEC,MAAM,QAAQ,QAAQ;AAC7D,SAASC,cAAc,QAAQ,aAAU;AA2BzC;;;;;;;;;;;;;;;;;;;AAmBA,OAAO,MAAMC,uBAAuB,GAClCC,SAA4B,IAE5BL,MAAM,CAACM,GAAG,CAAC,aAAS;EAClB,MAAMC,GAAG,GAAG,OAAOZ,aAAa;EAChC,MAAMa,KAAK,GAAGd,kBAAkB,CAACW,SAAS,CAACI,MAAM,CAAC,CAACD,KAAK;EACxD,MAAME,KAAK,GAAGb,SAAS,CAACc,IAAI,CAACf,0BAA0B,CAAC;EACxD,MAAMgB,UAAU,GAAGZ,MAAM,CAACa,QAAQ,CAChCR,SAAS,CAACS,KAAK,CAACC,cAAc,EAC7BC,KAAK,IACJ,IAAIb,cAAc,CAAC;IACjBc,MAAM,EAAE,iCAAiCZ,SAAS,CAACS,KAAK,CAACI,GAAG,CAACC,UAAU,qBAAqB;IAC5FH;GACD,CAAC,CACL;EAED;EACA;EACA,MAAMI,iBAAiB,GAAIC,IAAc,IACvCrB,MAAM,CAACM,GAAG,CAAC,aAAS;IAClB,IAAIgB,IAAI,GAAGD,IAAI;IACf,SAAS;MACP,MAAME,QAAQ,GAAG,OAAOhB,GAAG,CAACiB,IAAI,CAAChB,KAAK,EAAE;QAAEiB,KAAK,EAAEH,IAAI;QAAEZ;MAAK,CAAE,CAAC;MAC/D,MAAMgB,IAAI,GAAG,OAAOxB,MAAM,CAACyB,UAAU,CAACJ,QAAQ,CAACK,MAAM,CAAC;MACtD,MAAMC,KAAK,GAAG9B,GAAG,CAACuB,IAAI,CAACI,IAAI,CAAC;MAC5B,IAAIzB,MAAM,CAAC6B,MAAM,CAACD,KAAK,CAAC,EAAE;QACxBP,IAAI,GAAGO,KAAK,CAACE,KAAK,CAACC,QAAQ;MAC7B;MACA,IAAIN,IAAI,CAACO,MAAM,GAAGvB,KAAK,EAAE;QACvB,OAAOY,IAAI;MACb;IACF;EACF,CAAC,CAAC;EAEJ,MAAMY,SAAS,GAAkBA,CAAA,KAC/BlC,MAAM,CAACmC,GAAG,CACRnC,MAAM,CAACoC,OAAO,CAACxB,UAAU,EAAEQ,iBAAiB,CAAC,EAC5CiB,MAAM,IACLrC,MAAM,CAACmC,GAAG,CAACvB,UAAU,EAAG0B,OAAO,IAAKA,OAAO,IAAID,MAAM,CAAC,CACzD;EACH,OAAOH,SAAS;AAClB,CAAC,CAAC","ignoreList":[]}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reconcile: delete what a sink holds that the feed no longer has.
|
|
3
|
+
*
|
|
4
|
+
* ## Why a feed needs it at all
|
|
5
|
+
*
|
|
6
|
+
* The steady state deletes through tombstones, and a tombstone lives only until
|
|
7
|
+
* every sink registered on the feed has passed it. A record can still be left
|
|
8
|
+
* behind in a sink: a sink restored from a backup, a key written into the index by
|
|
9
|
+
* hand, or a record deleted before a reset, whose tombstone the reset cleared with
|
|
10
|
+
* every other row, followed by an in-place resync that, having no atomic swap,
|
|
11
|
+
* sends only the rows the new generation holds and so never deletes it. Reconcile is the one operation that compares the sink with the feed
|
|
12
|
+
* rather than replaying the feed into the sink, and so the only one that finds
|
|
13
|
+
* those orphans.
|
|
14
|
+
*
|
|
15
|
+
* ## The order of the two reads, and why it is safe
|
|
16
|
+
*
|
|
17
|
+
* The sink's keys are listed FIRST and the feed's live keys read SECOND. A key the
|
|
18
|
+
* sink lists that has a live row by the time the feed is read is kept, so a record
|
|
19
|
+
* written while reconcile runs is never deleted by it. A key whose row turns into
|
|
20
|
+
* a tombstone after the feed is read is kept too, and its tombstone deletes it on
|
|
21
|
+
* the next pass. And a key whose row comes back to life after the feed is read is
|
|
22
|
+
* deleted here and re-sent by the next pass, because its new `seq` is above the
|
|
23
|
+
* cursor. Reconcile runs inside the worker that holds the lease, between batches,
|
|
24
|
+
* so nothing else is sending to the sink while it does (ADR-0019 §8).
|
|
25
|
+
*
|
|
26
|
+
* ## What it costs
|
|
27
|
+
*
|
|
28
|
+
* It holds every key the sink lists and every live key of the feed in memory at
|
|
29
|
+
* once, and reads every live document to find the keys. It is a periodic repair,
|
|
30
|
+
* not a steady-state path: `reconcileEvery` should be measured in hours.
|
|
31
|
+
*/
|
|
32
|
+
import { Array as Arr, Effect, Option, Stream } from 'effect';
|
|
33
|
+
import { deliver } from "./deliver.js";
|
|
34
|
+
import { FeedChange } from "./FeedSink.js";
|
|
35
|
+
import { FEED_ORIGIN } from "./identifiers.js";
|
|
36
|
+
import { DEFAULT_FEED_PAGE_SIZE, readLimit } from "./tuning.js";
|
|
37
|
+
/** Every key of `source` whose row is live, read page by page from the origin. */
|
|
38
|
+
export const liveKeys = (source, pageSize) => Effect.gen(function* () {
|
|
39
|
+
const live = new Set();
|
|
40
|
+
const limit = readLimit(pageSize);
|
|
41
|
+
let after = FEED_ORIGIN;
|
|
42
|
+
for (;;) {
|
|
43
|
+
const rows = yield* source.changesAfter(after, limit);
|
|
44
|
+
for (const row of rows) {
|
|
45
|
+
if (Option.isSome(row.doc)) {
|
|
46
|
+
live.add(row.key);
|
|
47
|
+
} else {
|
|
48
|
+
live.delete(row.key);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
const last = rows.at(-1);
|
|
52
|
+
if (last === undefined || rows.length < pageSize) {
|
|
53
|
+
return live;
|
|
54
|
+
}
|
|
55
|
+
after = last.seq;
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
/**
|
|
59
|
+
* Reconcile under a lease the caller already holds. Package-internal: the driver
|
|
60
|
+
* calls it between batches and in an in-place resync, and `reconcileFeed` calls it
|
|
61
|
+
* after taking the lease itself.
|
|
62
|
+
*
|
|
63
|
+
* @internal
|
|
64
|
+
*/
|
|
65
|
+
export const reconcileWith = args => Effect.gen(function* () {
|
|
66
|
+
const listed = yield* Stream.runCollect(args.reconcile.listKeys);
|
|
67
|
+
const live = yield* liveKeys(args.source, args.pageSize);
|
|
68
|
+
const orphans = Array.from(new Set(listed)).filter(key => !live.has(key));
|
|
69
|
+
const deleted = [];
|
|
70
|
+
const refused = [];
|
|
71
|
+
for (const chunk of Arr.chunksOf(orphans, args.pageSize)) {
|
|
72
|
+
const pending = Arr.map(chunk, key => ({
|
|
73
|
+
change: FeedChange.Delete({
|
|
74
|
+
key
|
|
75
|
+
}),
|
|
76
|
+
seq: FEED_ORIGIN
|
|
77
|
+
}));
|
|
78
|
+
const delivered = yield* deliver({
|
|
79
|
+
send: args.sink.apply,
|
|
80
|
+
confirm: args.sink.confirm,
|
|
81
|
+
lease: args.lease,
|
|
82
|
+
pending
|
|
83
|
+
});
|
|
84
|
+
deleted.push(...delivered.sent);
|
|
85
|
+
refused.push(...delivered.parked.map(p => ({
|
|
86
|
+
key: p.key,
|
|
87
|
+
reason: p.reason
|
|
88
|
+
})));
|
|
89
|
+
}
|
|
90
|
+
const listedKeys = new Set(listed);
|
|
91
|
+
const deletedKeys = new Set(deleted);
|
|
92
|
+
return {
|
|
93
|
+
report: {
|
|
94
|
+
listed: listedKeys.size,
|
|
95
|
+
deleted,
|
|
96
|
+
refused
|
|
97
|
+
},
|
|
98
|
+
isGone: key => !live.has(key) && (deletedKeys.has(key) || !listedKeys.has(key))
|
|
99
|
+
};
|
|
100
|
+
});
|
|
101
|
+
/**
|
|
102
|
+
* Unpark every parked key a reconcile found gone from the sink for good (see
|
|
103
|
+
* {@link Reconciled}): a park reporting a refusal for such a key — an in-place
|
|
104
|
+
* rebuild's orphan, or a tombstone whose delete the tail was refused — reports a
|
|
105
|
+
* record that is not there. `retryParked` unparks a key with no row and re-stamps
|
|
106
|
+
* one that has a row, which for a tombstone sends every sink one idempotent delete.
|
|
107
|
+
*
|
|
108
|
+
* It judges by what the sink holds, not by what this reconcile deleted, so a park
|
|
109
|
+
* an earlier unpark failed to clear goes on the next reconcile. That is why a
|
|
110
|
+
* failure here is a warning and not an error: it costs a stale entry in
|
|
111
|
+
* `inspectFeed` until then, and failing would end a worker's session, or discard
|
|
112
|
+
* an operator's report, over a record that is already gone.
|
|
113
|
+
*
|
|
114
|
+
* @internal
|
|
115
|
+
*/
|
|
116
|
+
export const unparkGone = (store, id, isGone) => Effect.gen(function* () {
|
|
117
|
+
const parked = yield* store.readParked(id);
|
|
118
|
+
yield* Effect.forEach(parked.filter(p => isGone(p.key)), p => store.retryParked({
|
|
119
|
+
...id,
|
|
120
|
+
key: p.key
|
|
121
|
+
}), {
|
|
122
|
+
discard: true
|
|
123
|
+
});
|
|
124
|
+
}).pipe(Effect.catch(cause => Effect.logWarning('feed reconcile could not unpark a record it found gone; the next reconcile will', {
|
|
125
|
+
feed: id.feed,
|
|
126
|
+
sink: id.sink,
|
|
127
|
+
reason: cause.message
|
|
128
|
+
})));
|
|
129
|
+
/**
|
|
130
|
+
* Reconcile `sink` against `source` once, now. It takes the worker lease itself,
|
|
131
|
+
* so it fails `LeaseHeld` while a sync worker for the same feed and sink runs —
|
|
132
|
+
* reconcile through that worker's `reconcileEvery` instead, or stop it first.
|
|
133
|
+
*/
|
|
134
|
+
export const reconcileFeed = options => Effect.scoped(Effect.gen(function* () {
|
|
135
|
+
const id = {
|
|
136
|
+
feed: options.source.feed,
|
|
137
|
+
sink: options.sink.name
|
|
138
|
+
};
|
|
139
|
+
const lease = yield* options.store.acquireLease(id);
|
|
140
|
+
const {
|
|
141
|
+
report,
|
|
142
|
+
isGone
|
|
143
|
+
} = yield* reconcileWith({
|
|
144
|
+
source: options.source,
|
|
145
|
+
sink: options.sink,
|
|
146
|
+
reconcile: options.sink.reconcile,
|
|
147
|
+
lease,
|
|
148
|
+
pageSize: options.pageSize ?? DEFAULT_FEED_PAGE_SIZE
|
|
149
|
+
});
|
|
150
|
+
yield* unparkGone(options.store, id, isGone);
|
|
151
|
+
return report;
|
|
152
|
+
}));
|
|
153
|
+
//# sourceMappingURL=reconcile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reconcile.js","names":["Array","Arr","Effect","Option","Stream","deliver","FeedChange","FEED_ORIGIN","DEFAULT_FEED_PAGE_SIZE","readLimit","liveKeys","source","pageSize","gen","live","Set","limit","after","rows","changesAfter","row","isSome","doc","add","key","delete","last","at","undefined","length","seq","reconcileWith","args","listed","runCollect","reconcile","listKeys","orphans","from","filter","has","deleted","refused","chunk","chunksOf","pending","map","change","Delete","delivered","send","sink","apply","confirm","lease","push","sent","parked","p","reason","listedKeys","deletedKeys","report","size","isGone","unparkGone","store","id","readParked","forEach","retryParked","discard","pipe","catch","cause","logWarning","feed","message","reconcileFeed","options","scoped","name","acquireLease"],"sources":["../../src/reconcile.ts"],"sourcesContent":[null],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+BA,SAASA,KAAK,IAAIC,GAAG,EAAEC,MAAM,EAAEC,MAAM,EAAEC,MAAM,QAAQ,QAAQ;AAE7D,SAASC,OAAO,QAAsB,cAAW;AAUjD,SAASC,UAAU,QAA2C,eAAY;AAE1E,SAASC,WAAW,QAAuC,kBAAe;AAC1E,SAASC,sBAAsB,EAAEC,SAAS,QAAQ,aAAU;AAe5D;AACA,OAAO,MAAMC,QAAQ,GAAGA,CACtBC,MAAqB,EACrBC,QAAgB,KAEhBV,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAMC,IAAI,GAAG,IAAIC,GAAG,EAAW;EAC/B,MAAMC,KAAK,GAAGP,SAAS,CAACG,QAAQ,CAAC;EACjC,IAAIK,KAAK,GAAGV,WAAW;EACvB,SAAS;IACP,MAAMW,IAAI,GAAG,OAAOP,MAAM,CAACQ,YAAY,CAACF,KAAK,EAAED,KAAK,CAAC;IACrD,KAAK,MAAMI,GAAG,IAAIF,IAAI,EAAE;MACtB,IAAIf,MAAM,CAACkB,MAAM,CAACD,GAAG,CAACE,GAAG,CAAC,EAAE;QAC1BR,IAAI,CAACS,GAAG,CAACH,GAAG,CAACI,GAAG,CAAC;MACnB,CAAC,MAAM;QACLV,IAAI,CAACW,MAAM,CAACL,GAAG,CAACI,GAAG,CAAC;MACtB;IACF;IACA,MAAME,IAAI,GAAGR,IAAI,CAACS,EAAE,CAAC,CAAC,CAAC,CAAC;IACxB,IAAID,IAAI,KAAKE,SAAS,IAAIV,IAAI,CAACW,MAAM,GAAGjB,QAAQ,EAAE;MAChD,OAAOE,IAAI;IACb;IACAG,KAAK,GAAGS,IAAI,CAACI,GAAG;EAClB;AACF,CAAC,CAAC;AAeJ;;;;;;;AAOA,OAAO,MAAMC,aAAa,GAAgBC,IAMzC,IAIC9B,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAMoB,MAAM,GAAG,OAAO7B,MAAM,CAAC8B,UAAU,CAACF,IAAI,CAACG,SAAS,CAACC,QAAQ,CAAC;EAChE,MAAMtB,IAAI,GAAG,OAAOJ,QAAQ,CAACsB,IAAI,CAACrB,MAAM,EAAEqB,IAAI,CAACpB,QAAQ,CAAC;EACxD,MAAMyB,OAAO,GAAGrC,KAAK,CAACsC,IAAI,CAAC,IAAIvB,GAAG,CAACkB,MAAM,CAAC,CAAC,CAACM,MAAM,CAAEf,GAAG,IAAK,CAACV,IAAI,CAAC0B,GAAG,CAAChB,GAAG,CAAC,CAAC;EAC3E,MAAMiB,OAAO,GAAmB,EAAE;EAClC,MAAMC,OAAO,GAA4C,EAAE;EAC3D,KAAK,MAAMC,KAAK,IAAI1C,GAAG,CAAC2C,QAAQ,CAACP,OAAO,EAAEL,IAAI,CAACpB,QAAQ,CAAC,EAAE;IACxD,MAAMiC,OAAO,GAAsC5C,GAAG,CAAC6C,GAAG,CACxDH,KAAK,EACJnB,GAAG,KAAM;MAAEuB,MAAM,EAAEzC,UAAU,CAAC0C,MAAM,CAAC;QAAExB;MAAG,CAAE,CAAC;MAAEM,GAAG,EAAEvB;IAAW,CAAE,CAAC,CACpE;IACD,MAAM0C,SAAS,GAAG,OAAO5C,OAAO,CAAC;MAC/B6C,IAAI,EAAElB,IAAI,CAACmB,IAAI,CAACC,KAAK;MACrBC,OAAO,EAAErB,IAAI,CAACmB,IAAI,CAACE,OAAO;MAC1BC,KAAK,EAAEtB,IAAI,CAACsB,KAAK;MACjBT;KACD,CAAC;IACFJ,OAAO,CAACc,IAAI,CAAC,GAAGN,SAAS,CAACO,IAAI,CAAC;IAC/Bd,OAAO,CAACa,IAAI,CACV,GAAGN,SAAS,CAACQ,MAAM,CAACX,GAAG,CAAEY,CAAC,KAAM;MAAElC,GAAG,EAAEkC,CAAC,CAAClC,GAAG;MAAEmC,MAAM,EAAED,CAAC,CAACC;IAAM,CAAE,CAAC,CAAC,CACnE;EACH;EACA,MAAMC,UAAU,GAAG,IAAI7C,GAAG,CAACkB,MAAM,CAAC;EAClC,MAAM4B,WAAW,GAAG,IAAI9C,GAAG,CAAC0B,OAAO,CAAC;EACpC,OAAO;IACLqB,MAAM,EAAE;MAAE7B,MAAM,EAAE2B,UAAU,CAACG,IAAI;MAAEtB,OAAO;MAAEC;IAAO,CAAE;IACrDsB,MAAM,EAAGxC,GAAG,IACV,CAACV,IAAI,CAAC0B,GAAG,CAAChB,GAAG,CAAC,KAAKqC,WAAW,CAACrB,GAAG,CAAChB,GAAG,CAAC,IAAI,CAACoC,UAAU,CAACpB,GAAG,CAAChB,GAAG,CAAC;GAClE;AACH,CAAC,CAAC;AAEJ;;;;;;;;;;;;;;;AAeA,OAAO,MAAMyC,UAAU,GAAGA,CACxBC,KAAoB,EACpBC,EAAc,EACdH,MAAiC,KAEjC9D,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAM4C,MAAM,GAAG,OAAOS,KAAK,CAACE,UAAU,CAACD,EAAE,CAAC;EAC1C,OAAOjE,MAAM,CAACmE,OAAO,CACnBZ,MAAM,CAAClB,MAAM,CAAEmB,CAAC,IAAKM,MAAM,CAACN,CAAC,CAAClC,GAAG,CAAC,CAAC,EAClCkC,CAAC,IAAKQ,KAAK,CAACI,WAAW,CAAC;IAAE,GAAGH,EAAE;IAAE3C,GAAG,EAAEkC,CAAC,CAAClC;EAAG,CAAE,CAAC,EAC/C;IAAE+C,OAAO,EAAE;EAAI,CAAE,CAClB;AACH,CAAC,CAAC,CAACC,IAAI,CACLtE,MAAM,CAACuE,KAAK,CAAEC,KAAK,IACjBxE,MAAM,CAACyE,UAAU,CACf,iFAAiF,EACjF;EAAEC,IAAI,EAAET,EAAE,CAACS,IAAI;EAAEzB,IAAI,EAAEgB,EAAE,CAAChB,IAAI;EAAEQ,MAAM,EAAEe,KAAK,CAACG;AAAO,CAAE,CACxD,CACF,CACF;AAWH;;;;;AAKA,OAAO,MAAMC,aAAa,GACxBC,OAAyC,IAUzC7E,MAAM,CAAC8E,MAAM,CACX9E,MAAM,CAACW,GAAG,CAAC,aAAS;EAClB,MAAMsD,EAAE,GAAG;IAAES,IAAI,EAAEG,OAAO,CAACpE,MAAM,CAACiE,IAAI;IAAEzB,IAAI,EAAE4B,OAAO,CAAC5B,IAAI,CAAC8B;EAAI,CAAE;EACjE,MAAM3B,KAAK,GAAG,OAAOyB,OAAO,CAACb,KAAK,CAACgB,YAAY,CAACf,EAAE,CAAC;EACnD,MAAM;IAAEL,MAAM;IAAEE;EAAM,CAAE,GAAG,OAAOjC,aAAa,CAAC;IAC9CpB,MAAM,EAAEoE,OAAO,CAACpE,MAAM;IACtBwC,IAAI,EAAE4B,OAAO,CAAC5B,IAAI;IAClBhB,SAAS,EAAE4C,OAAO,CAAC5B,IAAI,CAAChB,SAAS;IACjCmB,KAAK;IACL1C,QAAQ,EAAEmE,OAAO,CAACnE,QAAQ,IAAIJ;GAC/B,CAAC;EACF,OAAOyD,UAAU,CAACc,OAAO,CAACb,KAAK,EAAEC,EAAE,EAAEH,MAAM,CAAC;EAC5C,OAAOF,MAAM;AACf,CAAC,CAAC,CACH","ignoreList":[]}
|