@powerhousedao/reactor 6.2.3-dev.5 → 6.2.3-dev.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/document-indexer-3fkVWCaN.js +1856 -0
- package/dist/document-indexer-3fkVWCaN.js.map +1 -0
- package/dist/drive-container-types-BAfFzBOf.js +3892 -0
- package/dist/drive-container-types-BAfFzBOf.js.map +1 -0
- package/dist/entry.d.ts +2 -1
- package/dist/entry.js +111 -7
- package/dist/entry.js.map +1 -1
- package/dist/{errors-D3S6Eysd.js → errors-BM_ULVWd.js} +9 -1
- package/dist/{errors-D3S6Eysd.js.map → errors-BM_ULVWd.js.map} +1 -1
- package/dist/forwarding-logger-B9LLXItm.js +3667 -0
- package/dist/forwarding-logger-B9LLXItm.js.map +1 -0
- package/dist/index-dDm0fUBT.d.ts +6833 -0
- package/dist/index-dDm0fUBT.d.ts.map +1 -0
- package/dist/index.d.ts +1912 -5584
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7937 -1436
- package/dist/index.js.map +1 -1
- package/dist/load-spec-ZLjuyhxz.js +246 -0
- package/dist/load-spec-ZLjuyhxz.js.map +1 -0
- package/dist/projection-entry.d.ts +2 -1
- package/dist/projection-entry.js +99 -21
- package/dist/projection-entry.js.map +1 -1
- package/dist/{projection-shard-manager-CPQc5XHL.js → projection-shard-manager-Drl9w8Eo.js} +25 -15
- package/dist/projection-shard-manager-Drl9w8Eo.js.map +1 -0
- package/dist/{projection-worker-wI4PwcV2.js → projection-worker-D8uNFdwX.js} +1 -1
- package/dist/{projection-worker-wI4PwcV2.js.map → projection-worker-D8uNFdwX.js.map} +1 -1
- package/dist/rpc.d.ts +358 -0
- package/dist/rpc.d.ts.map +1 -0
- package/dist/rpc.js +575 -0
- package/dist/rpc.js.map +1 -0
- package/dist/{transport-CuogVKN_.js → transport-B3G50KVd.js} +1 -1
- package/dist/{transport-CuogVKN_.js.map → transport-B3G50KVd.js.map} +1 -1
- package/dist/{transport-ByGviWdZ.js → transport-CZDHzfvy.js} +1 -1
- package/dist/{transport-ByGviWdZ.js.map → transport-CZDHzfvy.js.map} +1 -1
- package/dist/{worker-handle-CrERzl8s.js → worker-handle-Nso8i5eU.js} +30 -17
- package/dist/worker-handle-Nso8i5eU.js.map +1 -0
- package/dist/worker-pool-router-BROOEQSE.js +35 -0
- package/dist/worker-pool-router-BROOEQSE.js.map +1 -0
- package/package.json +34 -18
- package/dist/build-worker-executor-DnU3ZU4H.js +0 -83
- package/dist/build-worker-executor-DnU3ZU4H.js.map +0 -1
- package/dist/document-indexer-C5Gsa1B3.js +0 -958
- package/dist/document-indexer-C5Gsa1B3.js.map +0 -1
- package/dist/drive-container-types-yZrksiJR.js +0 -5310
- package/dist/drive-container-types-yZrksiJR.js.map +0 -1
- package/dist/error-info-Cpu4OY3o.js +0 -62
- package/dist/error-info-Cpu4OY3o.js.map +0 -1
- package/dist/forwarding-logger-BBkMSxuJ.js +0 -85
- package/dist/forwarding-logger-BBkMSxuJ.js.map +0 -1
- package/dist/projection-shard-manager-CPQc5XHL.js.map +0 -1
- package/dist/types-DMKLa0Ok.js +0 -33
- package/dist/types-DMKLa0Ok.js.map +0 -1
- package/dist/worker-HPysBfhx.js +0 -22
- package/dist/worker-HPysBfhx.js.map +0 -1
- package/dist/worker-handle-CrERzl8s.js.map +0 -1
|
@@ -0,0 +1,1856 @@
|
|
|
1
|
+
import { At as yieldToMain, Ct as ReactorEventTypes, S as createKyselyWatermarkProbe, _ as acquirePurgeLocks, lt as DocumentNotFoundError, v as findPurged, x as SettledWatermark } from "./forwarding-logger-B9LLXItm.js";
|
|
2
|
+
import { createAuthState, isDenied, isPurgeMarker } from "@powerhousedao/shared/document-model";
|
|
3
|
+
import { v4 } from "uuid";
|
|
4
|
+
import { childLogger } from "document-model";
|
|
5
|
+
//#region src/catch-up/scheduler.ts
|
|
6
|
+
const noHooks = {
|
|
7
|
+
onSwept: () => {},
|
|
8
|
+
describeSessions: () => Promise.resolve([])
|
|
9
|
+
};
|
|
10
|
+
function isCatchUpConsumer(value) {
|
|
11
|
+
if (typeof value !== "object" || value === null) return false;
|
|
12
|
+
const candidate = value;
|
|
13
|
+
return typeof candidate.sweep === "function" && typeof candidate.consumerId === "string" && typeof candidate.appliedThrough === "number";
|
|
14
|
+
}
|
|
15
|
+
/** Periodically sweeps every consumer up to the settled watermark. */
|
|
16
|
+
var CatchUpScheduler = class {
|
|
17
|
+
watermark;
|
|
18
|
+
operationIndex;
|
|
19
|
+
config;
|
|
20
|
+
logger;
|
|
21
|
+
hooks;
|
|
22
|
+
fixed = [];
|
|
23
|
+
sources = [];
|
|
24
|
+
sourced = /* @__PURE__ */ new WeakMap();
|
|
25
|
+
timer;
|
|
26
|
+
running;
|
|
27
|
+
started = false;
|
|
28
|
+
stopped = false;
|
|
29
|
+
warnedStallSince;
|
|
30
|
+
statusSources = [];
|
|
31
|
+
constructor(watermark, operationIndex, config, logger, hooks = noHooks) {
|
|
32
|
+
this.watermark = watermark;
|
|
33
|
+
this.operationIndex = operationIndex;
|
|
34
|
+
this.config = config;
|
|
35
|
+
this.logger = logger;
|
|
36
|
+
this.hooks = hooks;
|
|
37
|
+
}
|
|
38
|
+
get settledWatermark() {
|
|
39
|
+
return this.watermark;
|
|
40
|
+
}
|
|
41
|
+
addConsumer(consumer, thread) {
|
|
42
|
+
if (this.fixed.some((entry) => entry.consumer === consumer)) return;
|
|
43
|
+
this.fixed.push({
|
|
44
|
+
consumer,
|
|
45
|
+
thread,
|
|
46
|
+
lastAdvanceUtcMs: Date.now(),
|
|
47
|
+
warnedHeld: false
|
|
48
|
+
});
|
|
49
|
+
this.ensureTimer();
|
|
50
|
+
}
|
|
51
|
+
/** Consumers read on every tick, so models registered later are swept too. */
|
|
52
|
+
addSource(consumers, thread) {
|
|
53
|
+
this.sources.push({
|
|
54
|
+
consumers,
|
|
55
|
+
thread
|
|
56
|
+
});
|
|
57
|
+
this.ensureTimer();
|
|
58
|
+
}
|
|
59
|
+
/** Consumers another thread sweeps, reported in status(). */
|
|
60
|
+
addStatusSource(source) {
|
|
61
|
+
this.statusSources.push(source);
|
|
62
|
+
}
|
|
63
|
+
start() {
|
|
64
|
+
this.started = true;
|
|
65
|
+
this.ensureTimer();
|
|
66
|
+
}
|
|
67
|
+
async stop() {
|
|
68
|
+
this.stopped = true;
|
|
69
|
+
if (this.timer !== void 0) {
|
|
70
|
+
clearInterval(this.timer);
|
|
71
|
+
this.timer = void 0;
|
|
72
|
+
}
|
|
73
|
+
if (this.running !== void 0) await this.running.catch(() => []);
|
|
74
|
+
}
|
|
75
|
+
async sweepNow() {
|
|
76
|
+
if (this.running !== void 0) await this.running.catch(() => []);
|
|
77
|
+
return this.run();
|
|
78
|
+
}
|
|
79
|
+
status() {
|
|
80
|
+
return {
|
|
81
|
+
watermark: this.watermark.status(),
|
|
82
|
+
consumers: this.entries().map((entry) => ({
|
|
83
|
+
consumerId: entry.consumer.consumerId,
|
|
84
|
+
thread: entry.thread,
|
|
85
|
+
appliedThrough: entry.consumer.appliedThrough,
|
|
86
|
+
trackedAbove: entry.consumer.trackedAbove,
|
|
87
|
+
...entry.blockedAt !== void 0 ? { blockedAt: entry.blockedAt } : {},
|
|
88
|
+
lastAdvanceUtcMs: entry.lastAdvanceUtcMs
|
|
89
|
+
})).concat(this.statusSources.flatMap((source) => source()))
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
ensureTimer() {
|
|
93
|
+
if (!this.started || this.stopped || this.timer !== void 0) return;
|
|
94
|
+
if (this.fixed.length === 0 && this.sources.length === 0) return;
|
|
95
|
+
const timer = setInterval(() => {
|
|
96
|
+
if (this.running !== void 0) return;
|
|
97
|
+
this.run().catch((error) => {
|
|
98
|
+
this.logger.error("catch-up tick failed: @error", error);
|
|
99
|
+
});
|
|
100
|
+
}, this.config.intervalMs);
|
|
101
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
102
|
+
this.timer = timer;
|
|
103
|
+
}
|
|
104
|
+
run() {
|
|
105
|
+
const run = this.tick().finally(() => {
|
|
106
|
+
if (this.running === run) this.running = void 0;
|
|
107
|
+
});
|
|
108
|
+
this.running = run;
|
|
109
|
+
return run;
|
|
110
|
+
}
|
|
111
|
+
async tick() {
|
|
112
|
+
const settled = await this.watermark.refresh();
|
|
113
|
+
await this.warnIfWatermarkHeld();
|
|
114
|
+
const entries = this.entries();
|
|
115
|
+
if (entries.length === 0) return [];
|
|
116
|
+
let lowest = Number.POSITIVE_INFINITY;
|
|
117
|
+
for (const entry of entries) lowest = Math.min(lowest, entry.consumer.appliedThrough);
|
|
118
|
+
const shared = await this.readPage(lowest, settled);
|
|
119
|
+
const results = [];
|
|
120
|
+
for (const entry of entries) {
|
|
121
|
+
if (this.stopped) break;
|
|
122
|
+
const result = await this.sweepOne(entry, shared, settled);
|
|
123
|
+
if (result !== void 0) results.push(result);
|
|
124
|
+
}
|
|
125
|
+
return results;
|
|
126
|
+
}
|
|
127
|
+
entries() {
|
|
128
|
+
const entries = [...this.fixed];
|
|
129
|
+
const seen = new Set(entries.map((entry) => entry.consumer));
|
|
130
|
+
for (const source of this.sources) for (const consumer of source.consumers()) {
|
|
131
|
+
if (seen.has(consumer)) continue;
|
|
132
|
+
seen.add(consumer);
|
|
133
|
+
let entry = this.sourced.get(consumer);
|
|
134
|
+
if (entry === void 0) {
|
|
135
|
+
entry = {
|
|
136
|
+
consumer,
|
|
137
|
+
thread: source.thread,
|
|
138
|
+
lastAdvanceUtcMs: Date.now(),
|
|
139
|
+
warnedHeld: false
|
|
140
|
+
};
|
|
141
|
+
this.sourced.set(consumer, entry);
|
|
142
|
+
}
|
|
143
|
+
entries.push(entry);
|
|
144
|
+
}
|
|
145
|
+
return entries;
|
|
146
|
+
}
|
|
147
|
+
async sweepOne(entry, shared, settled) {
|
|
148
|
+
const applied = entry.consumer.appliedThrough;
|
|
149
|
+
let page = shared;
|
|
150
|
+
if (applied >= shared.bound && shared.bound < settled) try {
|
|
151
|
+
page = await this.readPage(applied, settled);
|
|
152
|
+
} catch (error) {
|
|
153
|
+
this.logger.error("@consumer catch-up could not read present ordinals: @error", entry.consumer.consumerId, error);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
let result;
|
|
157
|
+
try {
|
|
158
|
+
result = await entry.consumer.sweep(page.bound, page.present);
|
|
159
|
+
} catch (error) {
|
|
160
|
+
this.logger.error("@consumer catch-up sweep failed: @error", entry.consumer.consumerId, error);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
if (result.to > result.from) {
|
|
164
|
+
entry.lastAdvanceUtcMs = Date.now();
|
|
165
|
+
entry.warnedHeld = false;
|
|
166
|
+
}
|
|
167
|
+
entry.blockedAt = result.blockedAt;
|
|
168
|
+
this.warnIfConsumerHeld(entry, settled);
|
|
169
|
+
if (result.to > result.from || result.replayed > 0 || result.blockedAt !== void 0) this.hooks.onSwept(result, entry.thread);
|
|
170
|
+
return result;
|
|
171
|
+
}
|
|
172
|
+
async warnIfWatermarkHeld() {
|
|
173
|
+
const status = this.watermark.status();
|
|
174
|
+
const since = status.stalledSinceUtcMs;
|
|
175
|
+
if (since === void 0) {
|
|
176
|
+
this.warnedStallSince = void 0;
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
const heldMs = Date.now() - since;
|
|
180
|
+
if (heldMs < this.config.stuckWarnMs || this.warnedStallSince === since) return;
|
|
181
|
+
this.warnedStallSince = since;
|
|
182
|
+
const sessions = await this.hooks.describeSessions(status.waitingOn).catch(() => []);
|
|
183
|
+
this.logger.warn("settled watermark held at @settled (head @head) for @ms ms, waiting on xid @xids: @sessions", status.settledThrough, status.head, heldMs, status.waitingOn.join(", "), sessions.map((session) => `pid ${session.pid} ${session.applicationName} ${session.state} since ${session.xactStart ?? "?"}`).join("; "));
|
|
184
|
+
}
|
|
185
|
+
warnIfConsumerHeld(entry, settled) {
|
|
186
|
+
const applied = entry.consumer.appliedThrough;
|
|
187
|
+
if (applied >= settled || entry.warnedHeld) return;
|
|
188
|
+
const heldMs = Date.now() - entry.lastAdvanceUtcMs;
|
|
189
|
+
if (heldMs < this.config.stuckWarnMs) return;
|
|
190
|
+
entry.warnedHeld = true;
|
|
191
|
+
this.logger.warn("@consumer cursor held at @applied for @ms ms (settled @settled)", entry.consumer.consumerId, applied, heldMs, settled);
|
|
192
|
+
}
|
|
193
|
+
async readPage(after, settled) {
|
|
194
|
+
const limit = this.config.sweepPageSize;
|
|
195
|
+
const present = await this.operationIndex.getOrdinalsInRange(after, settled, limit);
|
|
196
|
+
return {
|
|
197
|
+
bound: present.length >= limit ? present[present.length - 1] : settled,
|
|
198
|
+
present
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
//#endregion
|
|
203
|
+
//#region src/catch-up/types.ts
|
|
204
|
+
const defaultCatchUpConfig = {
|
|
205
|
+
intervalMs: 2e3,
|
|
206
|
+
stuckWarnMs: 6e4,
|
|
207
|
+
maxTrackedAboveCursor: 1e5,
|
|
208
|
+
sweepPageSize: 1e4
|
|
209
|
+
};
|
|
210
|
+
//#endregion
|
|
211
|
+
//#region src/catch-up/contiguous-cursor.ts
|
|
212
|
+
/** In memory, per consumer; tracks ordinals above appliedThrough only. */
|
|
213
|
+
var ContiguousCursor = class {
|
|
214
|
+
through;
|
|
215
|
+
limit;
|
|
216
|
+
claimed = /* @__PURE__ */ new Set();
|
|
217
|
+
applied = /* @__PURE__ */ new Set();
|
|
218
|
+
constructor(through, limit) {
|
|
219
|
+
this.through = through;
|
|
220
|
+
this.limit = limit;
|
|
221
|
+
}
|
|
222
|
+
setLimit(limit) {
|
|
223
|
+
this.limit = limit;
|
|
224
|
+
}
|
|
225
|
+
/** Every present ordinal at or below this has been applied. */
|
|
226
|
+
get appliedThrough() {
|
|
227
|
+
return this.through;
|
|
228
|
+
}
|
|
229
|
+
get trackedAbove() {
|
|
230
|
+
return this.claimed.size + this.applied.size;
|
|
231
|
+
}
|
|
232
|
+
/** Ordinals the caller must apply; the rest are already covered. */
|
|
233
|
+
claim(ordinals) {
|
|
234
|
+
const mine = /* @__PURE__ */ new Set();
|
|
235
|
+
for (const ordinal of ordinals) {
|
|
236
|
+
if (ordinal <= this.through) continue;
|
|
237
|
+
if (this.claimed.has(ordinal) || this.applied.has(ordinal)) continue;
|
|
238
|
+
this.claimed.add(ordinal);
|
|
239
|
+
mine.add(ordinal);
|
|
240
|
+
}
|
|
241
|
+
return mine;
|
|
242
|
+
}
|
|
243
|
+
/** Releases claims; committed ones join the applied set. */
|
|
244
|
+
settle(ordinals, committed) {
|
|
245
|
+
for (const ordinal of ordinals) {
|
|
246
|
+
if (!this.claimed.delete(ordinal)) continue;
|
|
247
|
+
if (committed && ordinal > this.through) this.applied.add(ordinal);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/** Past the limit, clears the applied set; the next sweep re-applies it. */
|
|
251
|
+
enforceLimit() {
|
|
252
|
+
if (this.claimed.size + this.applied.size > this.limit) this.applied.clear();
|
|
253
|
+
}
|
|
254
|
+
/** Present ordinals above the cursor that no path has claimed or applied. */
|
|
255
|
+
missing(present) {
|
|
256
|
+
return present.filter((ordinal) => ordinal > this.through && !this.claimed.has(ordinal) && !this.applied.has(ordinal));
|
|
257
|
+
}
|
|
258
|
+
/** `settled`, held below the lowest claimed or unapplied ordinal. */
|
|
259
|
+
target(settled, present) {
|
|
260
|
+
let to = settled;
|
|
261
|
+
for (const ordinal of this.claimed) if (ordinal <= to) to = ordinal - 1;
|
|
262
|
+
for (const ordinal of present) {
|
|
263
|
+
if (ordinal <= this.through || ordinal > to) continue;
|
|
264
|
+
if (!this.applied.has(ordinal)) to = ordinal - 1;
|
|
265
|
+
}
|
|
266
|
+
return Math.max(to, this.through);
|
|
267
|
+
}
|
|
268
|
+
advance(to) {
|
|
269
|
+
if (to <= this.through) return;
|
|
270
|
+
this.through = to;
|
|
271
|
+
for (const ordinal of this.applied) if (ordinal <= to) this.applied.delete(ordinal);
|
|
272
|
+
for (const ordinal of this.claimed) if (ordinal <= to) this.claimed.delete(ordinal);
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
//#endregion
|
|
276
|
+
//#region src/read-models/base-read-model.ts
|
|
277
|
+
/** Small enough that a chunk's transaction rarely outlasts the yield deadline. */
|
|
278
|
+
const DEFAULT_COMMIT_CHUNK_SIZE = 50;
|
|
279
|
+
/** Matches the executor's own default, so both paths yield on the same cadence. */
|
|
280
|
+
const DEFAULT_READ_MODEL_YIELD_DEADLINE_MS = 50;
|
|
281
|
+
const defaultReadModelIndexingConfig = {
|
|
282
|
+
commitChunkSize: 50,
|
|
283
|
+
yieldDeadlineMs: 50
|
|
284
|
+
};
|
|
285
|
+
/** For read models whose callers can observe where a batch was split. */
|
|
286
|
+
const unchunkedReadModelIndexingConfig = {
|
|
287
|
+
commitChunkSize: Number.MAX_SAFE_INTEGER,
|
|
288
|
+
yieldDeadlineMs: 50
|
|
289
|
+
};
|
|
290
|
+
/**
|
|
291
|
+
* Keeps the chunk size at one operation or more: a chunk of zero or less never
|
|
292
|
+
* advances the indexing loop, so the pass would spin without ever resolving.
|
|
293
|
+
*/
|
|
294
|
+
function normalizeIndexingConfig(config) {
|
|
295
|
+
if (Number.isNaN(config.commitChunkSize)) return {
|
|
296
|
+
...config,
|
|
297
|
+
commitChunkSize: 50
|
|
298
|
+
};
|
|
299
|
+
return {
|
|
300
|
+
...config,
|
|
301
|
+
commitChunkSize: Math.max(1, Math.min(Math.floor(config.commitChunkSize), Number.MAX_SAFE_INTEGER))
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
function ordinalOf(item) {
|
|
305
|
+
return item.context.ordinal;
|
|
306
|
+
}
|
|
307
|
+
function isTracked(ordinal) {
|
|
308
|
+
return Number.isFinite(ordinal) && ordinal > 0;
|
|
309
|
+
}
|
|
310
|
+
function streamKeyOf(stream) {
|
|
311
|
+
return `${stream.documentId}\u0000${stream.scope}\u0000${stream.branch}`;
|
|
312
|
+
}
|
|
313
|
+
function groupByStream(items) {
|
|
314
|
+
const groups = /* @__PURE__ */ new Map();
|
|
315
|
+
for (const item of items) {
|
|
316
|
+
const { documentId, scope, branch } = item.context;
|
|
317
|
+
const key = streamKeyOf(item.context);
|
|
318
|
+
let group = groups.get(key);
|
|
319
|
+
if (group === void 0) {
|
|
320
|
+
group = {
|
|
321
|
+
documentId,
|
|
322
|
+
scope,
|
|
323
|
+
branch,
|
|
324
|
+
lowest: ordinalOf(item),
|
|
325
|
+
late: []
|
|
326
|
+
};
|
|
327
|
+
groups.set(key, group);
|
|
328
|
+
}
|
|
329
|
+
group.lowest = Math.min(group.lowest, ordinalOf(item));
|
|
330
|
+
group.late.push(item);
|
|
331
|
+
}
|
|
332
|
+
return [...groups.values()].sort((a, b) => a.lowest - b.lowest);
|
|
333
|
+
}
|
|
334
|
+
function mergeByOrdinal(...lists) {
|
|
335
|
+
const byOrdinal = /* @__PURE__ */ new Map();
|
|
336
|
+
for (const list of lists) for (const item of list) byOrdinal.set(ordinalOf(item), item);
|
|
337
|
+
return [...byOrdinal.values()].sort((a, b) => ordinalOf(a) - ordinalOf(b));
|
|
338
|
+
}
|
|
339
|
+
/** The ids whose purge an operation's rows must not outlive. */
|
|
340
|
+
function purgeFenceIds(items) {
|
|
341
|
+
const ids = /* @__PURE__ */ new Set();
|
|
342
|
+
for (const { operation, context } of items) {
|
|
343
|
+
ids.add(context.documentId);
|
|
344
|
+
if (operation.action.type !== "ADD_RELATIONSHIP") continue;
|
|
345
|
+
const input = operation.action.input;
|
|
346
|
+
if (typeof input.sourceId === "string") ids.add(input.sourceId);
|
|
347
|
+
if (typeof input.targetId === "string") ids.add(input.targetId);
|
|
348
|
+
}
|
|
349
|
+
return [...ids].sort();
|
|
350
|
+
}
|
|
351
|
+
function dropPurged(items, purged) {
|
|
352
|
+
if (purged.size === 0) return items;
|
|
353
|
+
return items.filter((item) => isPurgeMarker(item.operation) || !purged.has(item.context.documentId));
|
|
354
|
+
}
|
|
355
|
+
function errorMessage(error) {
|
|
356
|
+
return error instanceof Error ? error.message : String(error);
|
|
357
|
+
}
|
|
358
|
+
/** A read model whose cursor only sweeps and boot replay advance. */
|
|
359
|
+
var BaseReadModel = class BaseReadModel {
|
|
360
|
+
db;
|
|
361
|
+
operationIndex;
|
|
362
|
+
writeCache;
|
|
363
|
+
consistencyTracker;
|
|
364
|
+
config;
|
|
365
|
+
name;
|
|
366
|
+
indexing;
|
|
367
|
+
catchUpLogger;
|
|
368
|
+
cursor;
|
|
369
|
+
persisted = 0;
|
|
370
|
+
watermark;
|
|
371
|
+
maxTrackedAboveCursor = defaultCatchUpConfig.maxTrackedAboveCursor;
|
|
372
|
+
loggedFailure;
|
|
373
|
+
failedItem;
|
|
374
|
+
initialized = false;
|
|
375
|
+
liveInFlight = /* @__PURE__ */ new Map();
|
|
376
|
+
sweptListeners = /* @__PURE__ */ new Set();
|
|
377
|
+
constructor(db, operationIndex, writeCache, consistencyTracker, config) {
|
|
378
|
+
this.db = db;
|
|
379
|
+
this.operationIndex = operationIndex;
|
|
380
|
+
this.writeCache = writeCache;
|
|
381
|
+
this.consistencyTracker = consistencyTracker;
|
|
382
|
+
this.config = config;
|
|
383
|
+
this.name = config.readModelId;
|
|
384
|
+
this.catchUpLogger = childLogger([
|
|
385
|
+
"reactor",
|
|
386
|
+
"read-model",
|
|
387
|
+
config.readModelId
|
|
388
|
+
]);
|
|
389
|
+
this.indexing = normalizeIndexingConfig(config.indexing ?? defaultReadModelIndexingConfig);
|
|
390
|
+
this.cursor = new ContiguousCursor(0, this.maxTrackedAboveCursor);
|
|
391
|
+
if ((config.purgeFence ?? "locked") === "locked" && this.commitOperations !== BaseReadModel.prototype.commitOperations && !this.constructor.commitsInFenceTransaction) throw new Error(`Read model ${config.readModelId}: purgeFence "locked" commits in a transaction its commitOperations must write through. Write through the trx argument and set static commitsInFenceTransaction to true, or set purgeFence "skip" (rows on another handle) or "none" (no rows here, or its own transaction).`);
|
|
392
|
+
}
|
|
393
|
+
/** Set by a subclass whose commitOperations writes through its trx argument. */
|
|
394
|
+
static commitsInFenceTransaction = false;
|
|
395
|
+
get consumerId() {
|
|
396
|
+
return this.config.readModelId;
|
|
397
|
+
}
|
|
398
|
+
get appliedThrough() {
|
|
399
|
+
return this.cursor.appliedThrough;
|
|
400
|
+
}
|
|
401
|
+
get trackedAbove() {
|
|
402
|
+
return this.cursor.trackedAbove;
|
|
403
|
+
}
|
|
404
|
+
/** Shares a thread's watermark; otherwise boot probes through this db. */
|
|
405
|
+
attachCatchUp(watermark, maxTrackedAboveCursor) {
|
|
406
|
+
this.watermark = watermark;
|
|
407
|
+
this.maxTrackedAboveCursor = maxTrackedAboveCursor;
|
|
408
|
+
this.cursor.setLimit(maxTrackedAboveCursor);
|
|
409
|
+
}
|
|
410
|
+
/** Notified with the coordinates each sweep applied. */
|
|
411
|
+
onSwept(listener) {
|
|
412
|
+
this.sweptListeners.add(listener);
|
|
413
|
+
return () => {
|
|
414
|
+
this.sweptListeners.delete(listener);
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
/** A chunk that throws ends the replay; sweeps continue from below it. */
|
|
418
|
+
async init() {
|
|
419
|
+
if (this.initialized) {
|
|
420
|
+
await this.replayFromCursor();
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
let stored = await this.loadState();
|
|
424
|
+
if (stored === void 0) {
|
|
425
|
+
if (this.config.startFrom === "head") {
|
|
426
|
+
const watermark = this.settledWatermark();
|
|
427
|
+
const settled = await watermark.refresh();
|
|
428
|
+
const head = Math.max(settled, watermark.status().head);
|
|
429
|
+
await this.initializeState(head);
|
|
430
|
+
this.resetCursor(head);
|
|
431
|
+
this.initialized = true;
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
await this.initializeState(0);
|
|
435
|
+
stored = 0;
|
|
436
|
+
}
|
|
437
|
+
this.resetCursor(stored);
|
|
438
|
+
this.initialized = true;
|
|
439
|
+
await this.replayFromCursor();
|
|
440
|
+
}
|
|
441
|
+
/** The live path: never moves the cursor. */
|
|
442
|
+
indexOperations(items) {
|
|
443
|
+
if (items.length === 0) return Promise.resolve();
|
|
444
|
+
return this.reserveOperations(items).apply();
|
|
445
|
+
}
|
|
446
|
+
/** Holds a queued batch as live, so a sweep neither takes nor passes it. */
|
|
447
|
+
reserveOperations(items) {
|
|
448
|
+
const cursor = this.cursor;
|
|
449
|
+
const owned = this.claimLive(items);
|
|
450
|
+
const unmark = this.markLive(owned);
|
|
451
|
+
let done = false;
|
|
452
|
+
return {
|
|
453
|
+
apply: () => {
|
|
454
|
+
if (done) return Promise.resolve();
|
|
455
|
+
done = true;
|
|
456
|
+
if (cursor !== this.cursor) {
|
|
457
|
+
unmark();
|
|
458
|
+
return this.indexOperations(items);
|
|
459
|
+
}
|
|
460
|
+
if (owned.length === 0) {
|
|
461
|
+
unmark();
|
|
462
|
+
return Promise.resolve();
|
|
463
|
+
}
|
|
464
|
+
return this.applyChunked(owned, unmark);
|
|
465
|
+
},
|
|
466
|
+
release: () => {
|
|
467
|
+
if (done) return;
|
|
468
|
+
done = true;
|
|
469
|
+
cursor.settle(owned.map(ordinalOf), false);
|
|
470
|
+
unmark();
|
|
471
|
+
}
|
|
472
|
+
};
|
|
473
|
+
}
|
|
474
|
+
async sweep(settledThrough, present, signal) {
|
|
475
|
+
const startedAt = performance.now();
|
|
476
|
+
const from = this.cursor.appliedThrough;
|
|
477
|
+
const range = present.filter((ordinal) => ordinal > from && ordinal <= settledThrough);
|
|
478
|
+
const mine = this.cursor.claim(this.cursor.missing(range));
|
|
479
|
+
let replayed = 0;
|
|
480
|
+
let reapplied = 0;
|
|
481
|
+
let blockedAt;
|
|
482
|
+
if (mine.size > 0) {
|
|
483
|
+
let late;
|
|
484
|
+
try {
|
|
485
|
+
late = await this.operationIndex.getByOrdinals([...mine], signal);
|
|
486
|
+
} catch (error) {
|
|
487
|
+
this.cursor.settle(mine, false);
|
|
488
|
+
throw error;
|
|
489
|
+
}
|
|
490
|
+
const found = new Set(late.map(ordinalOf));
|
|
491
|
+
this.cursor.settle([...mine].filter((ordinal) => !found.has(ordinal)), true);
|
|
492
|
+
for (const group of groupByStream(late)) {
|
|
493
|
+
const outcome = await this.sweepStream(group, signal);
|
|
494
|
+
replayed += outcome.replayed;
|
|
495
|
+
reapplied += outcome.reapplied;
|
|
496
|
+
blockedAt ??= outcome.blockedAt;
|
|
497
|
+
}
|
|
498
|
+
if (replayed > 0) this.catchUpLogger.warn("@consumer applied @n operations its live path never received: ordinals @first..@last", this.consumerId, replayed, Math.min(...found), Math.max(...found));
|
|
499
|
+
}
|
|
500
|
+
const to = this.cursor.target(settledThrough, range);
|
|
501
|
+
if (to > from) await this.moveCursor(to);
|
|
502
|
+
this.cursor.enforceLimit();
|
|
503
|
+
return {
|
|
504
|
+
consumerId: this.consumerId,
|
|
505
|
+
from,
|
|
506
|
+
to: this.cursor.appliedThrough,
|
|
507
|
+
durationMs: performance.now() - startedAt,
|
|
508
|
+
replayed,
|
|
509
|
+
reapplied,
|
|
510
|
+
...blockedAt !== void 0 ? { blockedAt } : {}
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* Waits for the read model to reach the specified consistency level.
|
|
515
|
+
*/
|
|
516
|
+
async waitForConsistency(token, timeoutMs, signal) {
|
|
517
|
+
if (token.coordinates.length === 0) return;
|
|
518
|
+
await this.consistencyTracker.waitFor(token.coordinates, timeoutMs, signal);
|
|
519
|
+
}
|
|
520
|
+
/** The cursor, for subclasses: every present ordinal at or below it is applied. */
|
|
521
|
+
get lastOrdinal() {
|
|
522
|
+
return this.cursor.appliedThrough;
|
|
523
|
+
}
|
|
524
|
+
/** Called after the cursor advances. */
|
|
525
|
+
onCursorAdvanced(appliedThrough) {}
|
|
526
|
+
/** Writes through `trx` when given: the purge fence's transaction. */
|
|
527
|
+
async commitOperations(items, trx) {}
|
|
528
|
+
/** The handle document_purges is read through; `db` may be the fence trx. */
|
|
529
|
+
purgeLookup(db) {
|
|
530
|
+
return db;
|
|
531
|
+
}
|
|
532
|
+
/** False when the batch writes no rows, so the fence opens no transaction. */
|
|
533
|
+
writesRows(items) {
|
|
534
|
+
return true;
|
|
535
|
+
}
|
|
536
|
+
/** Every delivery commits through here. */
|
|
537
|
+
async commitFenced(items) {
|
|
538
|
+
const fence = this.config.purgeFence ?? "locked";
|
|
539
|
+
if (fence === "none") {
|
|
540
|
+
await this.commitOperations(items);
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
543
|
+
if (items.length === 0 || !this.writesRows(items)) {
|
|
544
|
+
if (items.length > 0) await this.commitOperations(items);
|
|
545
|
+
return;
|
|
546
|
+
}
|
|
547
|
+
const ids = purgeFenceIds(items);
|
|
548
|
+
if (fence === "skip") {
|
|
549
|
+
const live = dropPurged(items, await findPurged(this.purgeLookup(this.db), ids));
|
|
550
|
+
if (live.length > 0) await this.commitOperations(live);
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
await this.db.transaction().execute(async (trx) => {
|
|
554
|
+
await acquirePurgeLocks(trx, ids, "shared");
|
|
555
|
+
const live = dropPurged(items, await findPurged(this.purgeLookup(trx), ids));
|
|
556
|
+
if (live.length > 0) await this.commitOperations(live, trx);
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
/** Rebuilds resultingState as the executor writes it: scopes plus header. */
|
|
560
|
+
async rebuildStateForOperations(operations) {
|
|
561
|
+
const result = [];
|
|
562
|
+
for (const op of operations) {
|
|
563
|
+
const { documentId, scope, branch } = op.context;
|
|
564
|
+
const targetRevision = op.operation.index;
|
|
565
|
+
const document = await this.writeCache.getState(documentId, scope, branch, targetRevision);
|
|
566
|
+
result.push({
|
|
567
|
+
operation: op.operation,
|
|
568
|
+
context: {
|
|
569
|
+
...op.context,
|
|
570
|
+
resultingState: JSON.stringify({
|
|
571
|
+
...document.state,
|
|
572
|
+
header: document.header
|
|
573
|
+
})
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
return result;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Loads the last processed ordinal from the ViewState table.
|
|
581
|
+
* Returns undefined if no state exists for this read model.
|
|
582
|
+
*/
|
|
583
|
+
async loadState() {
|
|
584
|
+
return (await this.db.selectFrom("ViewState").select("lastOrdinal").where("readModelId", "=", this.config.readModelId).executeTakeFirst())?.lastOrdinal;
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* Initializes the ViewState row for this read model.
|
|
588
|
+
*/
|
|
589
|
+
async initializeState(lastOrdinal = 0) {
|
|
590
|
+
await this.db.insertInto("ViewState").values({
|
|
591
|
+
readModelId: this.config.readModelId,
|
|
592
|
+
lastOrdinal
|
|
593
|
+
}).onConflict((oc) => oc.column("readModelId").doNothing()).execute();
|
|
594
|
+
}
|
|
595
|
+
/**
|
|
596
|
+
* Updates the consistency tracker with the processed operations.
|
|
597
|
+
*/
|
|
598
|
+
updateConsistencyTracker(items) {
|
|
599
|
+
this.consistencyTracker.update(this.coordinatesOf(items));
|
|
600
|
+
}
|
|
601
|
+
coordinatesOf(items) {
|
|
602
|
+
const coordinates = [];
|
|
603
|
+
for (let i = 0; i < items.length; i++) {
|
|
604
|
+
const item = items[i];
|
|
605
|
+
coordinates.push({
|
|
606
|
+
documentId: item.context.documentId,
|
|
607
|
+
scope: item.context.scope,
|
|
608
|
+
branch: item.context.branch,
|
|
609
|
+
operationIndex: item.operation.index
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
return coordinates;
|
|
613
|
+
}
|
|
614
|
+
settledWatermark() {
|
|
615
|
+
this.watermark ??= new SettledWatermark(createKyselyWatermarkProbe(this.db), this.catchUpLogger);
|
|
616
|
+
return this.watermark;
|
|
617
|
+
}
|
|
618
|
+
resetCursor(appliedThrough) {
|
|
619
|
+
this.cursor = new ContiguousCursor(appliedThrough, this.maxTrackedAboveCursor);
|
|
620
|
+
this.persisted = appliedThrough;
|
|
621
|
+
}
|
|
622
|
+
/** Items this pass must apply; untracked ordinals always apply. */
|
|
623
|
+
claimLive(items) {
|
|
624
|
+
const mine = this.cursor.claim(items.map(ordinalOf));
|
|
625
|
+
return items.filter((item) => {
|
|
626
|
+
const ordinal = ordinalOf(item);
|
|
627
|
+
return !isTracked(ordinal) || mine.has(ordinal);
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
/** Marks items in flight on their streams; returns the release. */
|
|
631
|
+
markLive(items) {
|
|
632
|
+
const keys = /* @__PURE__ */ new Set();
|
|
633
|
+
for (const item of items) {
|
|
634
|
+
if (!isTracked(ordinalOf(item))) continue;
|
|
635
|
+
const key = streamKeyOf(item.context);
|
|
636
|
+
let ordinals = this.liveInFlight.get(key);
|
|
637
|
+
if (ordinals === void 0) {
|
|
638
|
+
ordinals = /* @__PURE__ */ new Set();
|
|
639
|
+
this.liveInFlight.set(key, ordinals);
|
|
640
|
+
}
|
|
641
|
+
ordinals.add(ordinalOf(item));
|
|
642
|
+
keys.add(key);
|
|
643
|
+
}
|
|
644
|
+
return () => {
|
|
645
|
+
for (const item of items) this.liveInFlight.get(streamKeyOf(item.context))?.delete(ordinalOf(item));
|
|
646
|
+
for (const key of keys) if (this.liveInFlight.get(key)?.size === 0) this.liveInFlight.delete(key);
|
|
647
|
+
};
|
|
648
|
+
}
|
|
649
|
+
/** True while a live pass applies an earlier operation of the stream. */
|
|
650
|
+
liveBelow(group) {
|
|
651
|
+
const ordinals = this.liveInFlight.get(streamKeyOf(group));
|
|
652
|
+
if (ordinals === void 0) return false;
|
|
653
|
+
for (const ordinal of ordinals) if (ordinal < group.lowest) return true;
|
|
654
|
+
return false;
|
|
655
|
+
}
|
|
656
|
+
/** Commits in chunks, yielding with no transaction open. */
|
|
657
|
+
async applyChunked(items, release = this.markLive(items)) {
|
|
658
|
+
try {
|
|
659
|
+
const { commitChunkSize, yieldDeadlineMs } = this.indexing;
|
|
660
|
+
let lastYield = performance.now();
|
|
661
|
+
let committed = 0;
|
|
662
|
+
for (let start = 0; start < items.length; start += commitChunkSize) {
|
|
663
|
+
if (start > 0 && performance.now() - lastYield > yieldDeadlineMs) {
|
|
664
|
+
await yieldToMain();
|
|
665
|
+
lastYield = performance.now();
|
|
666
|
+
}
|
|
667
|
+
const chunk = items.slice(start, start + commitChunkSize);
|
|
668
|
+
try {
|
|
669
|
+
await this.commitFenced(chunk);
|
|
670
|
+
} catch (error) {
|
|
671
|
+
this.failedItem = chunk[0];
|
|
672
|
+
const prefix = items.slice(0, committed);
|
|
673
|
+
this.cursor.settle(prefix.map(ordinalOf), true);
|
|
674
|
+
this.cursor.settle(items.slice(committed).map(ordinalOf), false);
|
|
675
|
+
if (prefix.length > 0) this.updateConsistencyTracker(prefix);
|
|
676
|
+
throw error;
|
|
677
|
+
}
|
|
678
|
+
committed += chunk.length;
|
|
679
|
+
}
|
|
680
|
+
this.cursor.settle(items.map(ordinalOf), true);
|
|
681
|
+
this.cursor.enforceLimit();
|
|
682
|
+
this.updateConsistencyTracker(items);
|
|
683
|
+
} finally {
|
|
684
|
+
release();
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
async replayFromCursor() {
|
|
688
|
+
const settledAtBoot = await this.settledAtBoot();
|
|
689
|
+
let page = await this.operationIndex.getSinceOrdinal(this.cursor.appliedThrough);
|
|
690
|
+
while (page.results.length > 0) {
|
|
691
|
+
const ordinals = page.results.map(ordinalOf);
|
|
692
|
+
const failed = await this.replayPage(page.results);
|
|
693
|
+
const pageMax = Math.max(...ordinals);
|
|
694
|
+
const to = this.cursor.target(Math.min(settledAtBoot, pageMax), ordinals);
|
|
695
|
+
if (to > this.cursor.appliedThrough) await this.moveCursor(to);
|
|
696
|
+
if (failed || !page.next) break;
|
|
697
|
+
page = await page.next();
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
/** Without a probe, boot replay applies but leaves the cursor to sweeps. */
|
|
701
|
+
async settledAtBoot() {
|
|
702
|
+
try {
|
|
703
|
+
return await this.settledWatermark().refresh();
|
|
704
|
+
} catch (error) {
|
|
705
|
+
this.catchUpLogger.warn("@consumer could not probe the settled watermark at boot; the cursor waits for a sweep: @error", this.consumerId, error);
|
|
706
|
+
return this.cursor.appliedThrough;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
/** Returns true when a chunk failed, which ends the replay. */
|
|
710
|
+
async replayPage(results) {
|
|
711
|
+
const owned = this.claimLive(results);
|
|
712
|
+
if (owned.length === 0) return false;
|
|
713
|
+
const release = this.markLive(owned);
|
|
714
|
+
try {
|
|
715
|
+
return await this.replayOwned(owned);
|
|
716
|
+
} finally {
|
|
717
|
+
release();
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
async replayOwned(owned) {
|
|
721
|
+
let rebuilt;
|
|
722
|
+
try {
|
|
723
|
+
rebuilt = await this.rebuildIfConfigured(owned);
|
|
724
|
+
} catch (error) {
|
|
725
|
+
this.cursor.settle(owned.map(ordinalOf), false);
|
|
726
|
+
this.block(owned[0], error);
|
|
727
|
+
return true;
|
|
728
|
+
}
|
|
729
|
+
this.cursor.settle(rebuilt.absent, true);
|
|
730
|
+
if (rebuilt.items.length === 0) return false;
|
|
731
|
+
try {
|
|
732
|
+
await this.applyChunked(rebuilt.items);
|
|
733
|
+
} catch (error) {
|
|
734
|
+
this.block(this.failedItem ?? rebuilt.items[0], error);
|
|
735
|
+
return true;
|
|
736
|
+
}
|
|
737
|
+
return false;
|
|
738
|
+
}
|
|
739
|
+
async sweepStream(group, signal) {
|
|
740
|
+
const owned = group.late.map(ordinalOf);
|
|
741
|
+
if (this.liveBelow(group)) {
|
|
742
|
+
this.cursor.settle(owned, false);
|
|
743
|
+
return {
|
|
744
|
+
replayed: 0,
|
|
745
|
+
reapplied: 0
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
const ownedSet = new Set(owned);
|
|
749
|
+
let items = group.late;
|
|
750
|
+
if (this.config.replayStreamSuffix ?? true) {
|
|
751
|
+
let suffix;
|
|
752
|
+
try {
|
|
753
|
+
suffix = await this.operationIndex.getStreamAfter(group, group.lowest, signal);
|
|
754
|
+
} catch (error) {
|
|
755
|
+
this.cursor.settle(owned, false);
|
|
756
|
+
return {
|
|
757
|
+
replayed: 0,
|
|
758
|
+
reapplied: 0,
|
|
759
|
+
blockedAt: this.block(group.late[0], error)
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
items = mergeByOrdinal(group.late, suffix);
|
|
763
|
+
}
|
|
764
|
+
let rebuilt;
|
|
765
|
+
try {
|
|
766
|
+
rebuilt = await this.rebuildIfConfigured(items);
|
|
767
|
+
} catch (error) {
|
|
768
|
+
this.cursor.settle(owned, false);
|
|
769
|
+
return {
|
|
770
|
+
replayed: 0,
|
|
771
|
+
reapplied: 0,
|
|
772
|
+
blockedAt: this.block(items[0], error)
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
try {
|
|
776
|
+
await this.commitFenced(rebuilt.items);
|
|
777
|
+
} catch (error) {
|
|
778
|
+
this.cursor.settle(owned, false);
|
|
779
|
+
return {
|
|
780
|
+
replayed: 0,
|
|
781
|
+
reapplied: 0,
|
|
782
|
+
blockedAt: this.block(rebuilt.items[0] ?? items[0], error)
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
this.cursor.settle(owned, true);
|
|
786
|
+
this.updateConsistencyTracker(rebuilt.items);
|
|
787
|
+
this.notifySwept(rebuilt.items);
|
|
788
|
+
let replayed = 0;
|
|
789
|
+
for (const item of rebuilt.items) if (ownedSet.has(ordinalOf(item))) replayed++;
|
|
790
|
+
return {
|
|
791
|
+
replayed,
|
|
792
|
+
reapplied: rebuilt.items.length - replayed
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
/** A document gone from the write cache, or purged, is absent. */
|
|
796
|
+
async rebuildIfConfigured(items) {
|
|
797
|
+
if (!this.config.rebuildStateOnInit) return {
|
|
798
|
+
items,
|
|
799
|
+
absent: []
|
|
800
|
+
};
|
|
801
|
+
const rebuilt = [];
|
|
802
|
+
const absent = [];
|
|
803
|
+
for (const item of items) {
|
|
804
|
+
if (isPurgeMarker(item.operation)) {
|
|
805
|
+
rebuilt.push(item);
|
|
806
|
+
continue;
|
|
807
|
+
}
|
|
808
|
+
let result;
|
|
809
|
+
try {
|
|
810
|
+
result = await this.rebuildStateForOperations([item]);
|
|
811
|
+
} catch (error) {
|
|
812
|
+
if (!DocumentNotFoundError.isError(error) && !await this.isPurged(item.context.documentId)) throw error;
|
|
813
|
+
this.catchUpLogger.warn("@consumer dropped ordinal @ordinal: document @documentId is gone", this.consumerId, ordinalOf(item), item.context.documentId);
|
|
814
|
+
absent.push(ordinalOf(item));
|
|
815
|
+
continue;
|
|
816
|
+
}
|
|
817
|
+
rebuilt.push(...result);
|
|
818
|
+
}
|
|
819
|
+
return {
|
|
820
|
+
items: rebuilt,
|
|
821
|
+
absent
|
|
822
|
+
};
|
|
823
|
+
}
|
|
824
|
+
/** A failed lookup is no answer: the caller's own error stands. */
|
|
825
|
+
async isPurged(documentId) {
|
|
826
|
+
try {
|
|
827
|
+
return (await findPurged(this.purgeLookup(this.db), [documentId])).has(documentId);
|
|
828
|
+
} catch {
|
|
829
|
+
return false;
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
block(item, error) {
|
|
833
|
+
const blockedAt = {
|
|
834
|
+
ordinal: ordinalOf(item),
|
|
835
|
+
documentId: item.context.documentId,
|
|
836
|
+
scope: item.context.scope,
|
|
837
|
+
branch: item.context.branch,
|
|
838
|
+
type: item.operation.action.type,
|
|
839
|
+
error: errorMessage(error)
|
|
840
|
+
};
|
|
841
|
+
if (this.loggedFailure !== blockedAt.ordinal) {
|
|
842
|
+
this.loggedFailure = blockedAt.ordinal;
|
|
843
|
+
this.catchUpLogger.error("@consumer cursor held at @applied: ordinal @ordinal (@documentId/@scope/@branch, @type) failed: @error", this.consumerId, this.cursor.appliedThrough, blockedAt.ordinal, blockedAt.documentId, blockedAt.scope, blockedAt.branch, blockedAt.type, blockedAt.error);
|
|
844
|
+
}
|
|
845
|
+
return blockedAt;
|
|
846
|
+
}
|
|
847
|
+
notifySwept(items) {
|
|
848
|
+
if (this.sweptListeners.size === 0 || items.length === 0) return;
|
|
849
|
+
const coordinates = this.coordinatesOf(items);
|
|
850
|
+
for (const listener of this.sweptListeners) listener(coordinates);
|
|
851
|
+
}
|
|
852
|
+
/** Compare-and-set against the value this process last wrote. */
|
|
853
|
+
async moveCursor(to) {
|
|
854
|
+
if (to <= this.persisted) {
|
|
855
|
+
this.advanceCursor(to);
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
const expected = this.persisted;
|
|
859
|
+
const result = await this.db.updateTable("ViewState").set({
|
|
860
|
+
lastOrdinal: to,
|
|
861
|
+
lastOperationTimestamp: /* @__PURE__ */ new Date()
|
|
862
|
+
}).where("readModelId", "=", this.config.readModelId).where("lastOrdinal", "=", expected).executeTakeFirst();
|
|
863
|
+
if (Number(result.numUpdatedRows) > 0) {
|
|
864
|
+
this.persisted = to;
|
|
865
|
+
this.advanceCursor(to);
|
|
866
|
+
return;
|
|
867
|
+
}
|
|
868
|
+
const stored = await this.loadState();
|
|
869
|
+
if (stored === void 0) {
|
|
870
|
+
await this.initializeState(0);
|
|
871
|
+
this.lowered(0);
|
|
872
|
+
return;
|
|
873
|
+
}
|
|
874
|
+
if (stored < this.cursor.appliedThrough) {
|
|
875
|
+
this.lowered(stored);
|
|
876
|
+
return;
|
|
877
|
+
}
|
|
878
|
+
this.persisted = stored;
|
|
879
|
+
this.advanceCursor(Math.min(to, stored));
|
|
880
|
+
}
|
|
881
|
+
advanceCursor(to) {
|
|
882
|
+
const before = this.cursor.appliedThrough;
|
|
883
|
+
this.cursor.advance(to);
|
|
884
|
+
if (this.cursor.appliedThrough > before) this.onCursorAdvanced(this.cursor.appliedThrough);
|
|
885
|
+
}
|
|
886
|
+
lowered(to) {
|
|
887
|
+
this.catchUpLogger.info("@consumer cursor lowered externally from @old to @new; replaying", this.consumerId, this.cursor.appliedThrough, to);
|
|
888
|
+
this.resetCursor(to);
|
|
889
|
+
}
|
|
890
|
+
};
|
|
891
|
+
//#endregion
|
|
892
|
+
//#region src/read-models/coordinator.ts
|
|
893
|
+
/** Claims a batch for every model that supports it, as the batch queues. */
|
|
894
|
+
function reserveBatch(readModels, operations, logger) {
|
|
895
|
+
const reservations = /* @__PURE__ */ new Map();
|
|
896
|
+
for (const readModel of readModels) {
|
|
897
|
+
if (readModel.reserveOperations === void 0) continue;
|
|
898
|
+
try {
|
|
899
|
+
reservations.set(readModel, readModel.reserveOperations(operations));
|
|
900
|
+
} catch (error) {
|
|
901
|
+
logger.error("Read model @name could not reserve a batch: @Error", readModel.name, error);
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
return reservations;
|
|
905
|
+
}
|
|
906
|
+
/** Applies the model's reservation, or indexes a model registered since. */
|
|
907
|
+
function indexReserved(readModel, operations, reservations) {
|
|
908
|
+
const reservation = reservations.get(readModel);
|
|
909
|
+
return reservation ? reservation.apply() : readModel.indexOperations(operations);
|
|
910
|
+
}
|
|
911
|
+
/** Splices the instance out of each array; true when any held it. */
|
|
912
|
+
function spliceReadModel(readModel, ...lists) {
|
|
913
|
+
let found = false;
|
|
914
|
+
for (const list of lists) {
|
|
915
|
+
const index = list.indexOf(readModel);
|
|
916
|
+
if (index < 0) continue;
|
|
917
|
+
list.splice(index, 1);
|
|
918
|
+
found = true;
|
|
919
|
+
}
|
|
920
|
+
return found;
|
|
921
|
+
}
|
|
922
|
+
/** Waits for every model, so a sibling that throws early cannot let the
|
|
923
|
+
* chain start the next batch while another model still applies this one. */
|
|
924
|
+
async function indexStage(readModels, index, onError) {
|
|
925
|
+
const results = await Promise.allSettled(readModels.map(index));
|
|
926
|
+
for (const result of results) if (result.status === "rejected") onError(result.reason);
|
|
927
|
+
}
|
|
928
|
+
/** Frees every claim a run did not apply, so a sweep can take it. */
|
|
929
|
+
function releaseBatch(reservations) {
|
|
930
|
+
for (const reservation of reservations.values()) reservation.release();
|
|
931
|
+
}
|
|
932
|
+
/**
|
|
933
|
+
* Coordinates read model synchronization by listening to operation write events
|
|
934
|
+
* and updating all registered read models on per-`documentId:scope:branch`
|
|
935
|
+
* serial chains. Cross-key projection runs in parallel; same-key projection is
|
|
936
|
+
* serialized so the executor can return to dispatch without holding ordering
|
|
937
|
+
* implicitly.
|
|
938
|
+
*/
|
|
939
|
+
var ReadModelCoordinator = class {
|
|
940
|
+
eventBus;
|
|
941
|
+
preReady;
|
|
942
|
+
postReady;
|
|
943
|
+
unsubscribe;
|
|
944
|
+
isRunning = false;
|
|
945
|
+
chains = /* @__PURE__ */ new Map();
|
|
946
|
+
logger;
|
|
947
|
+
readModels;
|
|
948
|
+
constructor(eventBus, preReady, postReady) {
|
|
949
|
+
this.eventBus = eventBus;
|
|
950
|
+
this.preReady = preReady;
|
|
951
|
+
this.postReady = postReady;
|
|
952
|
+
this.readModels = [...preReady, ...postReady];
|
|
953
|
+
this.logger = childLogger(["reactor", "read-model-coordinator"]);
|
|
954
|
+
}
|
|
955
|
+
start() {
|
|
956
|
+
if (this.isRunning) return;
|
|
957
|
+
this.unsubscribe = this.eventBus.subscribe(ReactorEventTypes.JOB_WRITE_READY, (type, event) => {
|
|
958
|
+
return this.handleWriteReady(event);
|
|
959
|
+
});
|
|
960
|
+
this.isRunning = true;
|
|
961
|
+
}
|
|
962
|
+
stop() {
|
|
963
|
+
if (!this.isRunning) return;
|
|
964
|
+
if (this.unsubscribe) {
|
|
965
|
+
this.unsubscribe();
|
|
966
|
+
this.unsubscribe = void 0;
|
|
967
|
+
}
|
|
968
|
+
this.isRunning = false;
|
|
969
|
+
}
|
|
970
|
+
/**
|
|
971
|
+
* Resolves when every per-queueKey projection chain has flushed. Intended
|
|
972
|
+
* for test fixtures and explicit shutdown; production callers use
|
|
973
|
+
* consistency tokens instead.
|
|
974
|
+
*/
|
|
975
|
+
async drain() {
|
|
976
|
+
while (this.chains.size > 0) {
|
|
977
|
+
const pending = Array.from(this.chains.values());
|
|
978
|
+
await Promise.allSettled(pending);
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
getChainDepth() {
|
|
982
|
+
return this.chains.size;
|
|
983
|
+
}
|
|
984
|
+
indexedReadModels() {
|
|
985
|
+
return [...this.preReady, ...this.postReady];
|
|
986
|
+
}
|
|
987
|
+
addReadModel(readModel, stage) {
|
|
988
|
+
if (this.readModels.some(({ name }) => name === readModel.name)) throw new Error(`Read model "${readModel.name}" is already registered`);
|
|
989
|
+
if (stage === "pre_ready") this.preReady.push(readModel);
|
|
990
|
+
else this.postReady.push(readModel);
|
|
991
|
+
this.readModels.push(readModel);
|
|
992
|
+
}
|
|
993
|
+
removeReadModel(readModel) {
|
|
994
|
+
return spliceReadModel(readModel, this.preReady, this.postReady, this.readModels);
|
|
995
|
+
}
|
|
996
|
+
handleWriteReady(event) {
|
|
997
|
+
if (event.operations.length === 0) {
|
|
998
|
+
this.emitEmptyReadReady(event);
|
|
999
|
+
return;
|
|
1000
|
+
}
|
|
1001
|
+
const enqueuedAt = performance.now();
|
|
1002
|
+
const key = this.queueKeyFor(event);
|
|
1003
|
+
const reservations = reserveBatch(this.readModels, event.operations, this.logger);
|
|
1004
|
+
const run = () => this.runChain(event, enqueuedAt, reservations);
|
|
1005
|
+
const current = (this.chains.get(key) ?? Promise.resolve()).then(run, run);
|
|
1006
|
+
this.chains.set(key, current);
|
|
1007
|
+
current.finally(() => {
|
|
1008
|
+
if (this.chains.get(key) === current) this.chains.delete(key);
|
|
1009
|
+
});
|
|
1010
|
+
}
|
|
1011
|
+
async emitEmptyReadReady(event) {
|
|
1012
|
+
const readyEvent = {
|
|
1013
|
+
jobId: event.jobId,
|
|
1014
|
+
operations: event.operations
|
|
1015
|
+
};
|
|
1016
|
+
try {
|
|
1017
|
+
await this.eventBus.emit(ReactorEventTypes.JOB_READ_READY, readyEvent);
|
|
1018
|
+
} catch (error) {
|
|
1019
|
+
this.logger.error("JOB_READ_READY emit failed for job @jobId: @Error", { jobId: event.jobId }, error);
|
|
1020
|
+
}
|
|
1021
|
+
this.emitBatchCompleted({
|
|
1022
|
+
jobId: event.jobId,
|
|
1023
|
+
batchSize: 0,
|
|
1024
|
+
chainWaitDurationMs: 0,
|
|
1025
|
+
preReadyDurationMs: 0,
|
|
1026
|
+
emitDurationMs: 0,
|
|
1027
|
+
postReadyDurationMs: 0
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
async runChain(event, enqueuedAt, reservations) {
|
|
1031
|
+
try {
|
|
1032
|
+
await this.runBatch(event, enqueuedAt, reservations);
|
|
1033
|
+
} finally {
|
|
1034
|
+
releaseBatch(reservations);
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
async runBatch(event, enqueuedAt, reservations) {
|
|
1038
|
+
const chainWaitDurationMs = performance.now() - enqueuedAt;
|
|
1039
|
+
const preReadyStart = performance.now();
|
|
1040
|
+
await indexStage([...this.preReady], (readModel) => this.indexWithTiming(readModel, "pre_ready", event, reservations), (error) => this.logger.error("Pre-ready read model indexing failed for job @jobId: @Error", { jobId: event.jobId }, error));
|
|
1041
|
+
const preReadyDurationMs = performance.now() - preReadyStart;
|
|
1042
|
+
const readyEvent = {
|
|
1043
|
+
jobId: event.jobId,
|
|
1044
|
+
operations: event.operations
|
|
1045
|
+
};
|
|
1046
|
+
const emitStart = performance.now();
|
|
1047
|
+
try {
|
|
1048
|
+
await this.eventBus.emit(ReactorEventTypes.JOB_READ_READY, readyEvent);
|
|
1049
|
+
} catch (error) {
|
|
1050
|
+
this.logger.error("JOB_READ_READY emit failed for job @jobId: @Error", { jobId: event.jobId }, error);
|
|
1051
|
+
}
|
|
1052
|
+
const emitDurationMs = performance.now() - emitStart;
|
|
1053
|
+
const postReadyStart = performance.now();
|
|
1054
|
+
await indexStage([...this.postReady], (readModel) => this.indexWithTiming(readModel, "post_ready", event, reservations), (error) => this.logger.error("Post-ready read model indexing failed for job @jobId: @Error", { jobId: event.jobId }, error));
|
|
1055
|
+
const postReadyDurationMs = performance.now() - postReadyStart;
|
|
1056
|
+
this.emitBatchCompleted({
|
|
1057
|
+
jobId: event.jobId,
|
|
1058
|
+
batchSize: event.operations.length,
|
|
1059
|
+
chainWaitDurationMs,
|
|
1060
|
+
preReadyDurationMs,
|
|
1061
|
+
emitDurationMs,
|
|
1062
|
+
postReadyDurationMs
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
async indexWithTiming(readModel, stage, event, reservations) {
|
|
1066
|
+
const start = performance.now();
|
|
1067
|
+
let success = false;
|
|
1068
|
+
try {
|
|
1069
|
+
await indexReserved(readModel, event.operations, reservations);
|
|
1070
|
+
success = true;
|
|
1071
|
+
} finally {
|
|
1072
|
+
this.emitReadModelIndexed({
|
|
1073
|
+
jobId: event.jobId,
|
|
1074
|
+
readModelName: readModel.name,
|
|
1075
|
+
stage,
|
|
1076
|
+
durationMs: performance.now() - start,
|
|
1077
|
+
operationCount: event.operations.length,
|
|
1078
|
+
success
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
emitReadModelIndexed(payload) {
|
|
1083
|
+
this.eventBus.emit(ReactorEventTypes.READMODEL_INDEXED, payload).catch((err) => this.logger.error("READMODEL_INDEXED emit failed for job @jobId: @Error", { jobId: payload.jobId }, err));
|
|
1084
|
+
}
|
|
1085
|
+
emitBatchCompleted(payload) {
|
|
1086
|
+
this.eventBus.emit(ReactorEventTypes.READMODEL_BATCH_COMPLETED, payload).catch((err) => this.logger.error("READMODEL_BATCH_COMPLETED emit failed for job @jobId: @Error", { jobId: payload.jobId }, err));
|
|
1087
|
+
}
|
|
1088
|
+
queueKeyFor(event) {
|
|
1089
|
+
const ctx = event.operations[0].context;
|
|
1090
|
+
return `${ctx.documentId}:${ctx.scope}:${ctx.branch}`;
|
|
1091
|
+
}
|
|
1092
|
+
};
|
|
1093
|
+
//#endregion
|
|
1094
|
+
//#region src/read-models/names.ts
|
|
1095
|
+
/** `getReadModel` name of the document view; see {@link ReactorReadModels}. */
|
|
1096
|
+
const DOCUMENT_VIEW_READ_MODEL = "document-view";
|
|
1097
|
+
/** `getReadModel` name of the document indexer; see {@link ReactorReadModels}. */
|
|
1098
|
+
const DOCUMENT_INDEXER_READ_MODEL = "document-indexer";
|
|
1099
|
+
//#endregion
|
|
1100
|
+
//#region src/read-models/document-view.ts
|
|
1101
|
+
/**
|
|
1102
|
+
* What a single-document read of a deleted document returns. A listing omits a
|
|
1103
|
+
* deleted document under either value.
|
|
1104
|
+
*/
|
|
1105
|
+
let DeletedDocumentRead = /* @__PURE__ */ function(DeletedDocumentRead) {
|
|
1106
|
+
/**
|
|
1107
|
+
* The document reads as missing: `get` throws and `resolveIdOrSlug` does not
|
|
1108
|
+
* match its id.
|
|
1109
|
+
*/
|
|
1110
|
+
DeletedDocumentRead["NotFound"] = "NotFound";
|
|
1111
|
+
/**
|
|
1112
|
+
* The document's state as of the deletion, with `state.document.isDeleted`
|
|
1113
|
+
* telling the caller what it holds. Only meaningful with `documentDecisions`,
|
|
1114
|
+
* which is what makes deletion positional.
|
|
1115
|
+
*/
|
|
1116
|
+
DeletedDocumentRead["StateAtDeletion"] = "StateAtDeletion";
|
|
1117
|
+
return DeletedDocumentRead;
|
|
1118
|
+
}({});
|
|
1119
|
+
var KyselyDocumentView = class extends BaseReadModel {
|
|
1120
|
+
operationStore;
|
|
1121
|
+
deletedDocumentRead;
|
|
1122
|
+
static commitsInFenceTransaction = true;
|
|
1123
|
+
_db;
|
|
1124
|
+
logger = childLogger([
|
|
1125
|
+
"reactor",
|
|
1126
|
+
"read-model",
|
|
1127
|
+
DOCUMENT_VIEW_READ_MODEL
|
|
1128
|
+
]);
|
|
1129
|
+
constructor(db, operationStore, operationIndex, writeCache, consistencyTracker, deletedDocumentRead, indexing = defaultReadModelIndexingConfig) {
|
|
1130
|
+
super(db, operationIndex, writeCache, consistencyTracker, {
|
|
1131
|
+
readModelId: DOCUMENT_VIEW_READ_MODEL,
|
|
1132
|
+
rebuildStateOnInit: true,
|
|
1133
|
+
indexing,
|
|
1134
|
+
replayStreamSuffix: false,
|
|
1135
|
+
purgeFence: "locked"
|
|
1136
|
+
});
|
|
1137
|
+
this.operationStore = operationStore;
|
|
1138
|
+
this.deletedDocumentRead = deletedDocumentRead;
|
|
1139
|
+
this._db = db;
|
|
1140
|
+
}
|
|
1141
|
+
/**
|
|
1142
|
+
* Indexes committed operations into DocumentSnapshot rows. CREATE_DOCUMENT
|
|
1143
|
+
* only seeds header/document/auth. UPGRADE_DOCUMENT reindexes every scope
|
|
1144
|
+
* present in resultingState when the operation vouches for them — a seed
|
|
1145
|
+
* carrying initialState or a migration stamped with the __migrated marker
|
|
1146
|
+
* — since the upgrade reducer may have reshaped any of them; upgrades
|
|
1147
|
+
* without either fall back to header/document/auth, because their sibling
|
|
1148
|
+
* echoes may be stale. All other action types index only header and their
|
|
1149
|
+
* own scope.
|
|
1150
|
+
*
|
|
1151
|
+
* Every row accepts a write only from an operation whose global ordinal is
|
|
1152
|
+
* at least the one the row already carries, so an older duplicate from a
|
|
1153
|
+
* sweep, boot replay or chunk interleave never rolls a scope back.
|
|
1154
|
+
*/
|
|
1155
|
+
async commitOperations(items, fenced) {
|
|
1156
|
+
const write = async (trx) => {
|
|
1157
|
+
for (const item of items) {
|
|
1158
|
+
const { operation, context } = item;
|
|
1159
|
+
if (isPurgeMarker(operation)) continue;
|
|
1160
|
+
const { documentId, scope, branch, documentType, resultingState, ordinal } = context;
|
|
1161
|
+
const { index, hash } = operation;
|
|
1162
|
+
if (!resultingState) throw new Error(`Missing resultingState in context for operation ${operation.id || "unknown"}. IDocumentView requires resultingState from upstream - it does not rebuild documents.`);
|
|
1163
|
+
let fullState;
|
|
1164
|
+
try {
|
|
1165
|
+
fullState = JSON.parse(resultingState);
|
|
1166
|
+
} catch (error) {
|
|
1167
|
+
throw new Error(`Failed to parse resultingState for operation ${operation.id || "unknown"}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
1168
|
+
}
|
|
1169
|
+
const operationType = operation.action.type;
|
|
1170
|
+
const denied = isDenied(operation);
|
|
1171
|
+
const preserveHeaderMeta = denied || operationType !== "CREATE_DOCUMENT" && operationType !== "UPGRADE_DOCUMENT" && operationType !== "DELETE_DOCUMENT" && scope !== "header";
|
|
1172
|
+
if (operationType === "DELETE_DOCUMENT" && !denied) {
|
|
1173
|
+
const now = /* @__PURE__ */ new Date();
|
|
1174
|
+
await trx.updateTable("DocumentSnapshot").set({
|
|
1175
|
+
isDeleted: true,
|
|
1176
|
+
deletedAt: now,
|
|
1177
|
+
lastOperationIndex: index,
|
|
1178
|
+
lastOperationHash: hash,
|
|
1179
|
+
lastOperationOrdinal: ordinal,
|
|
1180
|
+
lastUpdatedAt: now
|
|
1181
|
+
}).where("documentId", "=", documentId).where("branch", "=", branch).where("lastOperationOrdinal", "<=", ordinal).execute();
|
|
1182
|
+
const deletedDocumentScope = fullState?.document;
|
|
1183
|
+
if (deletedDocumentScope !== void 0) await trx.updateTable("DocumentSnapshot").set({ content: deletedDocumentScope }).where("documentId", "=", documentId).where("branch", "=", branch).where("scope", "=", "document").where("lastOperationOrdinal", "<=", ordinal).execute();
|
|
1184
|
+
await trx.deleteFrom("SlugMapping").where("documentId", "=", documentId).where("branch", "=", branch).execute();
|
|
1185
|
+
continue;
|
|
1186
|
+
}
|
|
1187
|
+
let scopesToIndex;
|
|
1188
|
+
if (operationType === "CREATE_DOCUMENT") scopesToIndex = Object.entries(fullState).filter(([key]) => key === "header" || key === "document" || key === "auth");
|
|
1189
|
+
else if (operationType === "UPGRADE_DOCUMENT") {
|
|
1190
|
+
const upgradeInput = operation.action.input;
|
|
1191
|
+
const scopesVouchedFor = fullState.__migrated === true || upgradeInput.initialState !== void 0;
|
|
1192
|
+
delete fullState.__migrated;
|
|
1193
|
+
scopesToIndex = scopesVouchedFor ? Object.entries(fullState) : Object.entries(fullState).filter(([key]) => key === "header" || key === "document" || key === "auth");
|
|
1194
|
+
} else {
|
|
1195
|
+
scopesToIndex = [];
|
|
1196
|
+
if (fullState.header !== void 0) scopesToIndex.push(["header", fullState.header]);
|
|
1197
|
+
if (fullState[scope] !== void 0) scopesToIndex.push([scope, fullState[scope]]);
|
|
1198
|
+
else this.logger.warn(`refusing to index ${documentId} scope "${scope}" at operation ${index}: resultingState carries no "${scope}" key; leaving the existing snapshot intact`);
|
|
1199
|
+
}
|
|
1200
|
+
for (const [scopeName, scopeState] of scopesToIndex) {
|
|
1201
|
+
if (typeof scopeState !== "object" || scopeState === null) {
|
|
1202
|
+
this.logger.warn(`refusing to index ${documentId} scope "${scopeName}" at operation ${index}: resultingState carries a non-object "${scopeName}" value; leaving the existing snapshot intact`);
|
|
1203
|
+
continue;
|
|
1204
|
+
}
|
|
1205
|
+
const needsExistingContent = scopeName === "header" && preserveHeaderMeta;
|
|
1206
|
+
const existingSnapshot = await trx.selectFrom("DocumentSnapshot").select([
|
|
1207
|
+
"slug",
|
|
1208
|
+
"name",
|
|
1209
|
+
"isDeleted",
|
|
1210
|
+
"snapshotVersion",
|
|
1211
|
+
"lastOperationOrdinal"
|
|
1212
|
+
]).$if(needsExistingContent, (qb) => qb.select("content")).where("documentId", "=", documentId).where("scope", "=", scopeName).where("branch", "=", branch).executeTakeFirst();
|
|
1213
|
+
if (existingSnapshot !== void 0 && existingSnapshot.lastOperationOrdinal > ordinal) continue;
|
|
1214
|
+
const newState = scopeState;
|
|
1215
|
+
let slug = existingSnapshot?.slug ?? null;
|
|
1216
|
+
let name = existingSnapshot?.name ?? null;
|
|
1217
|
+
if (scopeName === "header") {
|
|
1218
|
+
const headerSlug = newState.slug;
|
|
1219
|
+
const headerName = newState.name;
|
|
1220
|
+
if (typeof headerSlug === "string") slug = headerSlug;
|
|
1221
|
+
if (typeof headerName === "string") name = headerName;
|
|
1222
|
+
if (needsExistingContent && existingSnapshot) {
|
|
1223
|
+
const existingHeader = existingSnapshot.content;
|
|
1224
|
+
if (existingHeader && "meta" in existingHeader) newState.meta = existingHeader.meta;
|
|
1225
|
+
}
|
|
1226
|
+
if (slug && slug !== documentId && !existingSnapshot?.isDeleted) await trx.insertInto("SlugMapping").values({
|
|
1227
|
+
slug,
|
|
1228
|
+
documentId,
|
|
1229
|
+
scope: scopeName,
|
|
1230
|
+
branch
|
|
1231
|
+
}).onConflict((oc) => oc.column("slug").doUpdateSet({
|
|
1232
|
+
documentId,
|
|
1233
|
+
scope: scopeName,
|
|
1234
|
+
branch
|
|
1235
|
+
})).execute();
|
|
1236
|
+
}
|
|
1237
|
+
if (existingSnapshot) await trx.updateTable("DocumentSnapshot").set({
|
|
1238
|
+
lastOperationIndex: index,
|
|
1239
|
+
lastOperationHash: hash,
|
|
1240
|
+
lastOperationOrdinal: ordinal,
|
|
1241
|
+
lastUpdatedAt: /* @__PURE__ */ new Date(),
|
|
1242
|
+
snapshotVersion: existingSnapshot.snapshotVersion + 1,
|
|
1243
|
+
content: newState,
|
|
1244
|
+
slug,
|
|
1245
|
+
name
|
|
1246
|
+
}).where("documentId", "=", documentId).where("scope", "=", scopeName).where("branch", "=", branch).where("lastOperationOrdinal", "<=", ordinal).execute();
|
|
1247
|
+
else {
|
|
1248
|
+
const snapshot = {
|
|
1249
|
+
id: v4(),
|
|
1250
|
+
documentId,
|
|
1251
|
+
slug,
|
|
1252
|
+
name,
|
|
1253
|
+
scope: scopeName,
|
|
1254
|
+
branch,
|
|
1255
|
+
content: newState,
|
|
1256
|
+
documentType,
|
|
1257
|
+
lastOperationIndex: index,
|
|
1258
|
+
lastOperationHash: hash,
|
|
1259
|
+
lastOperationOrdinal: ordinal,
|
|
1260
|
+
identifiers: null,
|
|
1261
|
+
metadata: null,
|
|
1262
|
+
deletedAt: null
|
|
1263
|
+
};
|
|
1264
|
+
await trx.insertInto("DocumentSnapshot").values(snapshot).execute();
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
};
|
|
1269
|
+
if (fenced) {
|
|
1270
|
+
await write(fenced);
|
|
1271
|
+
return;
|
|
1272
|
+
}
|
|
1273
|
+
await this._db.transaction().execute(write);
|
|
1274
|
+
}
|
|
1275
|
+
async exists(documentIds, existence, consistencyToken, signal) {
|
|
1276
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1277
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1278
|
+
if (documentIds.length === 0) return [];
|
|
1279
|
+
const existingIds = existence === "IncludingDeleted" ? await this.idsWithStreams(documentIds, signal) : await this.liveIds(documentIds);
|
|
1280
|
+
return documentIds.map((id) => existingIds.has(id));
|
|
1281
|
+
}
|
|
1282
|
+
async get(documentId, view, consistencyToken, signal) {
|
|
1283
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1284
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1285
|
+
const branch = view?.branch || "main";
|
|
1286
|
+
let scopesToQuery;
|
|
1287
|
+
if (view?.scopes && view.scopes.length > 0) scopesToQuery = [.../* @__PURE__ */ new Set([
|
|
1288
|
+
"header",
|
|
1289
|
+
"document",
|
|
1290
|
+
...view.scopes
|
|
1291
|
+
])];
|
|
1292
|
+
else scopesToQuery = [];
|
|
1293
|
+
let query = this._db.selectFrom("DocumentSnapshot").selectAll().where("documentId", "=", documentId).where("branch", "=", branch);
|
|
1294
|
+
if (this.deletedDocumentRead === "NotFound") query = query.where("isDeleted", "=", false);
|
|
1295
|
+
if (scopesToQuery.length > 0) query = query.where("scope", "in", scopesToQuery);
|
|
1296
|
+
const snapshots = await query.execute();
|
|
1297
|
+
if (snapshots.length === 0) throw new DocumentNotFoundError(documentId, `Document not found: ${documentId}`);
|
|
1298
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1299
|
+
const headerSnapshot = snapshots.find((s) => s.scope === "header");
|
|
1300
|
+
if (!headerSnapshot) throw new Error(`Document header not found: ${documentId}`);
|
|
1301
|
+
const header = headerSnapshot.content;
|
|
1302
|
+
const revisions = await this.operationStore.getRevisions(documentId, branch, signal);
|
|
1303
|
+
header.revision = revisions.revision;
|
|
1304
|
+
header.lastModifiedAtUtcIso = revisions.latestTimestamp;
|
|
1305
|
+
const state = {};
|
|
1306
|
+
for (const snapshot of snapshots) {
|
|
1307
|
+
if (snapshot.scope === "header") continue;
|
|
1308
|
+
state[snapshot.scope] = snapshot.scope === "auth" ? createAuthState(snapshot.content) : snapshot.content;
|
|
1309
|
+
}
|
|
1310
|
+
return {
|
|
1311
|
+
header,
|
|
1312
|
+
state,
|
|
1313
|
+
operations: {},
|
|
1314
|
+
initialState: state,
|
|
1315
|
+
clipboard: []
|
|
1316
|
+
};
|
|
1317
|
+
}
|
|
1318
|
+
async getMany(documentIds, view, consistencyToken, signal) {
|
|
1319
|
+
if (documentIds.length === 0) return [];
|
|
1320
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1321
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1322
|
+
const branch = view?.branch || "main";
|
|
1323
|
+
let scopesToQuery;
|
|
1324
|
+
if (view?.scopes && view.scopes.length > 0) scopesToQuery = [.../* @__PURE__ */ new Set([
|
|
1325
|
+
"header",
|
|
1326
|
+
"document",
|
|
1327
|
+
...view.scopes
|
|
1328
|
+
])];
|
|
1329
|
+
else scopesToQuery = [];
|
|
1330
|
+
let query = this._db.selectFrom("DocumentSnapshot").selectAll().where("documentId", "in", documentIds).where("branch", "=", branch).where("isDeleted", "=", false);
|
|
1331
|
+
if (scopesToQuery.length > 0) query = query.where("scope", "in", scopesToQuery);
|
|
1332
|
+
const allSnapshots = await query.execute();
|
|
1333
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1334
|
+
const snapshotsByDocId = /* @__PURE__ */ new Map();
|
|
1335
|
+
for (const snapshot of allSnapshots) {
|
|
1336
|
+
const existing = snapshotsByDocId.get(snapshot.documentId) || [];
|
|
1337
|
+
existing.push(snapshot);
|
|
1338
|
+
snapshotsByDocId.set(snapshot.documentId, existing);
|
|
1339
|
+
}
|
|
1340
|
+
const foundDocumentIds = [...snapshotsByDocId.keys()];
|
|
1341
|
+
const revisionResults = await Promise.all(foundDocumentIds.map((docId) => this.operationStore.getRevisions(docId, branch, signal)));
|
|
1342
|
+
const revisionsByDocId = /* @__PURE__ */ new Map();
|
|
1343
|
+
for (let i = 0; i < foundDocumentIds.length; i++) revisionsByDocId.set(foundDocumentIds[i], revisionResults[i]);
|
|
1344
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1345
|
+
const documents = [];
|
|
1346
|
+
for (const documentId of documentIds) {
|
|
1347
|
+
const snapshots = snapshotsByDocId.get(documentId);
|
|
1348
|
+
if (!snapshots || snapshots.length === 0) continue;
|
|
1349
|
+
const headerSnapshot = snapshots.find((s) => s.scope === "header");
|
|
1350
|
+
if (!headerSnapshot) continue;
|
|
1351
|
+
const header = headerSnapshot.content;
|
|
1352
|
+
const revisions = revisionsByDocId.get(documentId);
|
|
1353
|
+
if (revisions) {
|
|
1354
|
+
header.revision = revisions.revision;
|
|
1355
|
+
header.lastModifiedAtUtcIso = revisions.latestTimestamp;
|
|
1356
|
+
}
|
|
1357
|
+
const state = {};
|
|
1358
|
+
for (const snapshot of snapshots) {
|
|
1359
|
+
if (snapshot.scope === "header") continue;
|
|
1360
|
+
state[snapshot.scope] = snapshot.scope === "auth" ? createAuthState(snapshot.content) : snapshot.content;
|
|
1361
|
+
}
|
|
1362
|
+
const document = {
|
|
1363
|
+
header,
|
|
1364
|
+
state,
|
|
1365
|
+
operations: {},
|
|
1366
|
+
initialState: state,
|
|
1367
|
+
clipboard: []
|
|
1368
|
+
};
|
|
1369
|
+
documents.push(document);
|
|
1370
|
+
}
|
|
1371
|
+
return documents;
|
|
1372
|
+
}
|
|
1373
|
+
async getByIdOrSlug(identifier, view, consistencyToken, signal) {
|
|
1374
|
+
const documentId = await this.resolveIdOrSlug(identifier, view, consistencyToken, signal);
|
|
1375
|
+
return this.get(documentId, view, void 0, signal);
|
|
1376
|
+
}
|
|
1377
|
+
async findByType(type, view, paging, consistencyToken, signal) {
|
|
1378
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1379
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1380
|
+
const branch = view?.branch || "main";
|
|
1381
|
+
const startIndex = paging?.cursor ? parseInt(paging.cursor) : 0;
|
|
1382
|
+
const limit = paging?.limit || 100;
|
|
1383
|
+
const rows = await this._db.selectFrom("DocumentSnapshot").select("documentId").select((eb) => eb.fn.max("lastUpdatedAt").as("lastUpdatedAt")).where("documentType", "=", type).where("branch", "=", branch).where("isDeleted", "=", false).groupBy("documentId").orderBy("lastUpdatedAt", "desc").orderBy("documentId", "asc").offset(startIndex).limit(limit + 1).execute();
|
|
1384
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1385
|
+
const hasMore = rows.length > limit;
|
|
1386
|
+
const docsToFetch = rows.slice(0, limit).map((row) => row.documentId);
|
|
1387
|
+
const documents = [];
|
|
1388
|
+
for (const documentId of docsToFetch) {
|
|
1389
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1390
|
+
try {
|
|
1391
|
+
const document = await this.get(documentId, view, void 0, signal);
|
|
1392
|
+
documents.push(document);
|
|
1393
|
+
} catch {
|
|
1394
|
+
continue;
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
const nextCursor = hasMore ? String(startIndex + limit) : void 0;
|
|
1398
|
+
return {
|
|
1399
|
+
results: documents,
|
|
1400
|
+
options: paging || {
|
|
1401
|
+
cursor: "0",
|
|
1402
|
+
limit: 100
|
|
1403
|
+
},
|
|
1404
|
+
nextCursor,
|
|
1405
|
+
next: hasMore ? () => this.findByType(type, view, {
|
|
1406
|
+
cursor: nextCursor,
|
|
1407
|
+
limit
|
|
1408
|
+
}, consistencyToken, signal) : void 0
|
|
1409
|
+
};
|
|
1410
|
+
}
|
|
1411
|
+
async resolveSlug(slug, view, consistencyToken, signal) {
|
|
1412
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1413
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1414
|
+
const branch = view?.branch || "main";
|
|
1415
|
+
const mapping = await this._db.selectFrom("SlugMapping").select("documentId").where("slug", "=", slug).where("branch", "=", branch).executeTakeFirst();
|
|
1416
|
+
if (!mapping) return;
|
|
1417
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1418
|
+
if (view?.scopes && view.scopes.length > 0) {
|
|
1419
|
+
if (!await this._db.selectFrom("DocumentSnapshot").select("scope").where("documentId", "=", mapping.documentId).where("branch", "=", branch).where("scope", "in", view.scopes).where("isDeleted", "=", false).executeTakeFirst()) return;
|
|
1420
|
+
}
|
|
1421
|
+
return mapping.documentId;
|
|
1422
|
+
}
|
|
1423
|
+
async resolveSlugs(slugs, view, consistencyToken, signal) {
|
|
1424
|
+
return (await Promise.all(slugs.map((slug) => this.resolveSlug(slug, view, consistencyToken, signal)))).filter((id) => id !== void 0);
|
|
1425
|
+
}
|
|
1426
|
+
async resolveIdOrSlug(identifier, view, consistencyToken, signal) {
|
|
1427
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1428
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1429
|
+
const branch = view?.branch || "main";
|
|
1430
|
+
let idCheckQuery = this._db.selectFrom("DocumentSnapshot").select("documentId").where("documentId", "=", identifier).where("branch", "=", branch);
|
|
1431
|
+
if (this.deletedDocumentRead === "NotFound") idCheckQuery = idCheckQuery.where("isDeleted", "=", false);
|
|
1432
|
+
const idCheckPromise = idCheckQuery.executeTakeFirst();
|
|
1433
|
+
const slugCheckPromise = this._db.selectFrom("SlugMapping").select("documentId").where("slug", "=", identifier).where("branch", "=", branch).executeTakeFirst();
|
|
1434
|
+
const [idMatch, slugMatch] = await Promise.all([idCheckPromise, slugCheckPromise]);
|
|
1435
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1436
|
+
const idMatchDocId = idMatch?.documentId;
|
|
1437
|
+
const slugMatchDocId = slugMatch?.documentId;
|
|
1438
|
+
if (idMatchDocId && slugMatchDocId && idMatchDocId !== slugMatchDocId) throw new Error(`Ambiguous identifier "${identifier}": matches both document ID "${idMatchDocId}" and slug for document ID "${slugMatchDocId}". Please use get() for ID or resolveSlug() + get() for slug to be explicit.`);
|
|
1439
|
+
const resolvedDocumentId = idMatchDocId || slugMatchDocId;
|
|
1440
|
+
if (!resolvedDocumentId) throw new DocumentNotFoundError(identifier, `Document not found: ${identifier}`);
|
|
1441
|
+
return resolvedDocumentId;
|
|
1442
|
+
}
|
|
1443
|
+
/**
|
|
1444
|
+
* Ids that have at least one stream, asked of the operation store rather than
|
|
1445
|
+
* of a table, so the answer holds for any store implementation. A deleted
|
|
1446
|
+
* document keeps its stream, so this is the id-is-taken answer, and it does
|
|
1447
|
+
* not drift when a snapshot row is missing.
|
|
1448
|
+
*
|
|
1449
|
+
* Asked on main because that is the branch a create writes: the store this
|
|
1450
|
+
* question exists to agree with validates the create against main's head.
|
|
1451
|
+
*/
|
|
1452
|
+
async idsWithStreams(documentIds, signal) {
|
|
1453
|
+
const unique = [...new Set(documentIds)];
|
|
1454
|
+
const revisions = await Promise.all(unique.map((documentId) => this.operationStore.getRevisions(documentId, "main", signal)));
|
|
1455
|
+
const taken = /* @__PURE__ */ new Set();
|
|
1456
|
+
for (let i = 0; i < unique.length; i++) if (Object.keys(revisions[i].revision).length > 0) taken.add(unique[i]);
|
|
1457
|
+
return taken;
|
|
1458
|
+
}
|
|
1459
|
+
async liveIds(documentIds) {
|
|
1460
|
+
const rows = await this._db.selectFrom("DocumentSnapshot").select(["documentId"]).where("documentId", "in", documentIds).where("isDeleted", "=", false).distinct().execute();
|
|
1461
|
+
return new Set(rows.map((row) => row.documentId));
|
|
1462
|
+
}
|
|
1463
|
+
};
|
|
1464
|
+
//#endregion
|
|
1465
|
+
//#region src/shared/consistency-tracker.ts
|
|
1466
|
+
/**
|
|
1467
|
+
* Creates a consistency key from documentId, scope, and branch.
|
|
1468
|
+
*/
|
|
1469
|
+
function makeConsistencyKey(documentId, scope, branch) {
|
|
1470
|
+
return `${documentId}:${scope}:${branch}`;
|
|
1471
|
+
}
|
|
1472
|
+
/**
|
|
1473
|
+
* Tracks operation indexes for documents and provides read-after-write consistency guarantees.
|
|
1474
|
+
* Maintains an in-memory map of the latest operation index for each (documentId, scope, branch) tuple.
|
|
1475
|
+
*/
|
|
1476
|
+
var ConsistencyTracker = class {
|
|
1477
|
+
state = /* @__PURE__ */ new Map();
|
|
1478
|
+
waiters = [];
|
|
1479
|
+
update(coordinates) {
|
|
1480
|
+
const deduplicated = this.deduplicateCoordinates(coordinates);
|
|
1481
|
+
for (let i = 0; i < deduplicated.length; i++) {
|
|
1482
|
+
const coord = deduplicated[i];
|
|
1483
|
+
const key = makeConsistencyKey(coord.documentId, coord.scope, coord.branch);
|
|
1484
|
+
const current = this.state.get(key);
|
|
1485
|
+
if (current === void 0 || coord.operationIndex > current) this.state.set(key, coord.operationIndex);
|
|
1486
|
+
}
|
|
1487
|
+
this.checkWaiters();
|
|
1488
|
+
}
|
|
1489
|
+
getLatest(key) {
|
|
1490
|
+
return this.state.get(key);
|
|
1491
|
+
}
|
|
1492
|
+
waitFor(coordinates, timeoutMs, signal) {
|
|
1493
|
+
if (signal?.aborted) return Promise.reject(/* @__PURE__ */ new Error("Operation aborted"));
|
|
1494
|
+
if (this.areCoordinatesSatisfied(coordinates)) return Promise.resolve();
|
|
1495
|
+
return new Promise((resolve, reject) => {
|
|
1496
|
+
const waiter = {
|
|
1497
|
+
coordinates,
|
|
1498
|
+
resolve,
|
|
1499
|
+
reject,
|
|
1500
|
+
signal
|
|
1501
|
+
};
|
|
1502
|
+
if (timeoutMs !== void 0) waiter.timeoutId = setTimeout(() => {
|
|
1503
|
+
this.removeWaiter(waiter);
|
|
1504
|
+
reject(/* @__PURE__ */ new Error(`Consistency wait timed out after ${timeoutMs}ms`));
|
|
1505
|
+
}, timeoutMs);
|
|
1506
|
+
if (signal) {
|
|
1507
|
+
const abortHandler = () => {
|
|
1508
|
+
this.removeWaiter(waiter);
|
|
1509
|
+
reject(/* @__PURE__ */ new Error("Operation aborted"));
|
|
1510
|
+
};
|
|
1511
|
+
signal.addEventListener("abort", abortHandler, { once: true });
|
|
1512
|
+
}
|
|
1513
|
+
this.waiters.push(waiter);
|
|
1514
|
+
});
|
|
1515
|
+
}
|
|
1516
|
+
serialize() {
|
|
1517
|
+
return Array.from(this.state.entries());
|
|
1518
|
+
}
|
|
1519
|
+
hydrate(entries) {
|
|
1520
|
+
this.state.clear();
|
|
1521
|
+
for (const [key, index] of entries) this.state.set(key, index);
|
|
1522
|
+
}
|
|
1523
|
+
deduplicateCoordinates(coordinates) {
|
|
1524
|
+
const map = /* @__PURE__ */ new Map();
|
|
1525
|
+
for (let i = 0; i < coordinates.length; i++) {
|
|
1526
|
+
const coord = coordinates[i];
|
|
1527
|
+
const key = makeConsistencyKey(coord.documentId, coord.scope, coord.branch);
|
|
1528
|
+
const existing = map.get(key);
|
|
1529
|
+
if (!existing || coord.operationIndex > existing.operationIndex) map.set(key, coord);
|
|
1530
|
+
}
|
|
1531
|
+
return Array.from(map.values());
|
|
1532
|
+
}
|
|
1533
|
+
areCoordinatesSatisfied(coordinates) {
|
|
1534
|
+
for (let i = 0; i < coordinates.length; i++) {
|
|
1535
|
+
const coord = coordinates[i];
|
|
1536
|
+
const key = makeConsistencyKey(coord.documentId, coord.scope, coord.branch);
|
|
1537
|
+
const latest = this.state.get(key);
|
|
1538
|
+
if (latest === void 0 || latest < coord.operationIndex) return false;
|
|
1539
|
+
}
|
|
1540
|
+
return true;
|
|
1541
|
+
}
|
|
1542
|
+
checkWaiters() {
|
|
1543
|
+
const satisfiedWaiters = [];
|
|
1544
|
+
const unsatisfiedWaiters = [];
|
|
1545
|
+
for (const waiter of this.waiters) {
|
|
1546
|
+
if (waiter.signal?.aborted) continue;
|
|
1547
|
+
if (this.areCoordinatesSatisfied(waiter.coordinates)) satisfiedWaiters.push(waiter);
|
|
1548
|
+
else unsatisfiedWaiters.push(waiter);
|
|
1549
|
+
}
|
|
1550
|
+
this.waiters = unsatisfiedWaiters;
|
|
1551
|
+
for (const waiter of satisfiedWaiters) {
|
|
1552
|
+
if (waiter.timeoutId !== void 0) clearTimeout(waiter.timeoutId);
|
|
1553
|
+
waiter.resolve();
|
|
1554
|
+
}
|
|
1555
|
+
}
|
|
1556
|
+
removeWaiter(waiter) {
|
|
1557
|
+
const index = this.waiters.indexOf(waiter);
|
|
1558
|
+
if (index !== -1) this.waiters.splice(index, 1);
|
|
1559
|
+
if (waiter.timeoutId !== void 0) clearTimeout(waiter.timeoutId);
|
|
1560
|
+
}
|
|
1561
|
+
};
|
|
1562
|
+
//#endregion
|
|
1563
|
+
//#region src/shared/collect-all-pages.ts
|
|
1564
|
+
/**
|
|
1565
|
+
* Collects all results from a paged result set by following the next() function
|
|
1566
|
+
* until all pages have been fetched.
|
|
1567
|
+
*/
|
|
1568
|
+
async function collectAllPages(firstPage, signal) {
|
|
1569
|
+
const allResults = [...firstPage.results];
|
|
1570
|
+
let currentPage = firstPage;
|
|
1571
|
+
while (currentPage.next) {
|
|
1572
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1573
|
+
currentPage = await currentPage.next();
|
|
1574
|
+
allResults.push(...currentPage.results);
|
|
1575
|
+
}
|
|
1576
|
+
return allResults;
|
|
1577
|
+
}
|
|
1578
|
+
//#endregion
|
|
1579
|
+
//#region src/storage/kysely/document-indexer.ts
|
|
1580
|
+
function isRelationshipAction(actionType) {
|
|
1581
|
+
return actionType === "ADD_RELATIONSHIP" || actionType === "REMOVE_RELATIONSHIP" || actionType === "UPDATE_RELATIONSHIP";
|
|
1582
|
+
}
|
|
1583
|
+
function addRelationshipIds(items) {
|
|
1584
|
+
const ids = [];
|
|
1585
|
+
for (const { operation } of items) {
|
|
1586
|
+
if (operation.action.type !== "ADD_RELATIONSHIP") continue;
|
|
1587
|
+
const { sourceId, targetId } = operation.action.input;
|
|
1588
|
+
if (typeof sourceId === "string") ids.push(sourceId);
|
|
1589
|
+
if (typeof targetId === "string") ids.push(targetId);
|
|
1590
|
+
}
|
|
1591
|
+
return ids;
|
|
1592
|
+
}
|
|
1593
|
+
function touchesPurged(operation, purged) {
|
|
1594
|
+
const input = operation.action.input;
|
|
1595
|
+
return purged.has(input.sourceId) || purged.has(input.targetId);
|
|
1596
|
+
}
|
|
1597
|
+
var KyselyDocumentIndexer = class extends BaseReadModel {
|
|
1598
|
+
static commitsInFenceTransaction = true;
|
|
1599
|
+
_db;
|
|
1600
|
+
constructor(db, operationIndex, writeCache, consistencyTracker, indexing = defaultReadModelIndexingConfig) {
|
|
1601
|
+
super(db, operationIndex, writeCache, consistencyTracker, {
|
|
1602
|
+
readModelId: DOCUMENT_INDEXER_READ_MODEL,
|
|
1603
|
+
rebuildStateOnInit: false,
|
|
1604
|
+
indexing,
|
|
1605
|
+
purgeFence: "locked"
|
|
1606
|
+
});
|
|
1607
|
+
this._db = db;
|
|
1608
|
+
}
|
|
1609
|
+
writesRows(items) {
|
|
1610
|
+
return items.some((item) => isRelationshipAction(item.operation.action.type));
|
|
1611
|
+
}
|
|
1612
|
+
/** Opens no transaction for a batch carrying no relationship operation. */
|
|
1613
|
+
async commitOperations(items, fenced) {
|
|
1614
|
+
const relationshipOps = items.filter((item) => isRelationshipAction(item.operation.action.type));
|
|
1615
|
+
if (relationshipOps.length === 0) return;
|
|
1616
|
+
if (fenced) {
|
|
1617
|
+
await this.writeRelationships(relationshipOps, fenced);
|
|
1618
|
+
return;
|
|
1619
|
+
}
|
|
1620
|
+
await this._db.transaction().execute((trx) => this.writeRelationships(relationshipOps, trx));
|
|
1621
|
+
}
|
|
1622
|
+
async writeRelationships(relationshipOps, trx) {
|
|
1623
|
+
const purged = await findPurged(trx, addRelationshipIds(relationshipOps));
|
|
1624
|
+
for (const item of relationshipOps) {
|
|
1625
|
+
const { operation } = item;
|
|
1626
|
+
const actionType = operation.action.type;
|
|
1627
|
+
if (actionType === "ADD_RELATIONSHIP") {
|
|
1628
|
+
if (touchesPurged(operation, purged)) continue;
|
|
1629
|
+
await this.handleAddRelationship(trx, operation);
|
|
1630
|
+
} else if (actionType === "REMOVE_RELATIONSHIP") await this.handleRemoveRelationship(trx, operation);
|
|
1631
|
+
else if (actionType === "UPDATE_RELATIONSHIP") await this.handleUpdateRelationship(trx, operation);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
async getOutgoing(documentId, types, paging, consistencyToken, signal) {
|
|
1635
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1636
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1637
|
+
const startIndex = paging?.cursor ? parseInt(paging.cursor) : 0;
|
|
1638
|
+
const limit = paging?.limit || 100;
|
|
1639
|
+
let query = this._db.selectFrom("DocumentRelationship").selectAll().where("sourceId", "=", documentId);
|
|
1640
|
+
if (types && types.length > 0) query = query.where("relationshipType", "in", types);
|
|
1641
|
+
const rows = await query.orderBy("createdAt", "asc").orderBy("id", "asc").offset(startIndex).limit(limit + 1).execute();
|
|
1642
|
+
const hasMore = rows.length > limit;
|
|
1643
|
+
const results = hasMore ? rows.slice(0, limit) : rows;
|
|
1644
|
+
const nextCursor = hasMore ? String(startIndex + limit) : void 0;
|
|
1645
|
+
return {
|
|
1646
|
+
results: results.map((row) => ({
|
|
1647
|
+
sourceId: row.sourceId,
|
|
1648
|
+
targetId: row.targetId,
|
|
1649
|
+
relationshipType: row.relationshipType,
|
|
1650
|
+
metadata: row.metadata ? row.metadata : void 0,
|
|
1651
|
+
createdAt: row.createdAt,
|
|
1652
|
+
updatedAt: row.updatedAt
|
|
1653
|
+
})),
|
|
1654
|
+
options: paging || {
|
|
1655
|
+
cursor: "0",
|
|
1656
|
+
limit: 100
|
|
1657
|
+
},
|
|
1658
|
+
nextCursor,
|
|
1659
|
+
next: hasMore ? () => this.getOutgoing(documentId, types, {
|
|
1660
|
+
cursor: nextCursor,
|
|
1661
|
+
limit
|
|
1662
|
+
}, consistencyToken, signal) : void 0
|
|
1663
|
+
};
|
|
1664
|
+
}
|
|
1665
|
+
async getIncoming(documentId, types, paging, consistencyToken, signal) {
|
|
1666
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1667
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1668
|
+
const startIndex = paging?.cursor ? parseInt(paging.cursor) : 0;
|
|
1669
|
+
const limit = paging?.limit || 100;
|
|
1670
|
+
let query = this._db.selectFrom("DocumentRelationship").selectAll().where("targetId", "=", documentId);
|
|
1671
|
+
if (types && types.length > 0) query = query.where("relationshipType", "in", types);
|
|
1672
|
+
const rows = await query.orderBy("createdAt", "asc").orderBy("id", "asc").offset(startIndex).limit(limit + 1).execute();
|
|
1673
|
+
const hasMore = rows.length > limit;
|
|
1674
|
+
const results = hasMore ? rows.slice(0, limit) : rows;
|
|
1675
|
+
const nextCursor = hasMore ? String(startIndex + limit) : void 0;
|
|
1676
|
+
return {
|
|
1677
|
+
results: results.map((row) => ({
|
|
1678
|
+
sourceId: row.sourceId,
|
|
1679
|
+
targetId: row.targetId,
|
|
1680
|
+
relationshipType: row.relationshipType,
|
|
1681
|
+
metadata: row.metadata ? row.metadata : void 0,
|
|
1682
|
+
createdAt: row.createdAt,
|
|
1683
|
+
updatedAt: row.updatedAt
|
|
1684
|
+
})),
|
|
1685
|
+
options: paging || {
|
|
1686
|
+
cursor: "0",
|
|
1687
|
+
limit: 100
|
|
1688
|
+
},
|
|
1689
|
+
nextCursor,
|
|
1690
|
+
next: hasMore ? () => this.getIncoming(documentId, types, {
|
|
1691
|
+
cursor: nextCursor,
|
|
1692
|
+
limit
|
|
1693
|
+
}, consistencyToken, signal) : void 0
|
|
1694
|
+
};
|
|
1695
|
+
}
|
|
1696
|
+
async getOrphanedChildren(parentIds, types, signal) {
|
|
1697
|
+
if (parentIds.length === 0) return [];
|
|
1698
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1699
|
+
let query = this._db.selectFrom("DocumentRelationship as r").select("r.targetId").distinct().where("r.sourceId", "in", parentIds).where((eb) => eb.not(eb.exists(eb.selectFrom("DocumentRelationship as other").select("other.id").whereRef("other.targetId", "=", "r.targetId").where("other.sourceId", "not in", parentIds))));
|
|
1700
|
+
if (types && types.length > 0) query = query.where("r.relationshipType", "in", types);
|
|
1701
|
+
return (await query.execute()).map((row) => row.targetId);
|
|
1702
|
+
}
|
|
1703
|
+
async hasRelationship(sourceId, targetId, types, consistencyToken, signal) {
|
|
1704
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1705
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1706
|
+
let query = this._db.selectFrom("DocumentRelationship").select("id").where("sourceId", "=", sourceId).where("targetId", "=", targetId);
|
|
1707
|
+
if (types && types.length > 0) query = query.where("relationshipType", "in", types);
|
|
1708
|
+
return await query.executeTakeFirst() !== void 0;
|
|
1709
|
+
}
|
|
1710
|
+
async getUndirectedRelationships(a, b, types, paging, consistencyToken, signal) {
|
|
1711
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1712
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1713
|
+
const startIndex = paging?.cursor ? parseInt(paging.cursor) : 0;
|
|
1714
|
+
const limit = paging?.limit || 100;
|
|
1715
|
+
let query = this._db.selectFrom("DocumentRelationship").selectAll().where((eb) => eb.or([eb.and([eb("sourceId", "=", a), eb("targetId", "=", b)]), eb.and([eb("sourceId", "=", b), eb("targetId", "=", a)])]));
|
|
1716
|
+
if (types && types.length > 0) query = query.where("relationshipType", "in", types);
|
|
1717
|
+
const rows = await query.orderBy("createdAt", "asc").orderBy("id", "asc").offset(startIndex).limit(limit + 1).execute();
|
|
1718
|
+
const hasMore = rows.length > limit;
|
|
1719
|
+
const results = hasMore ? rows.slice(0, limit) : rows;
|
|
1720
|
+
const nextCursor = hasMore ? String(startIndex + limit) : void 0;
|
|
1721
|
+
return {
|
|
1722
|
+
results: results.map((row) => ({
|
|
1723
|
+
sourceId: row.sourceId,
|
|
1724
|
+
targetId: row.targetId,
|
|
1725
|
+
relationshipType: row.relationshipType,
|
|
1726
|
+
metadata: row.metadata ? row.metadata : void 0,
|
|
1727
|
+
createdAt: row.createdAt,
|
|
1728
|
+
updatedAt: row.updatedAt
|
|
1729
|
+
})),
|
|
1730
|
+
options: paging || {
|
|
1731
|
+
cursor: "0",
|
|
1732
|
+
limit: 100
|
|
1733
|
+
},
|
|
1734
|
+
nextCursor,
|
|
1735
|
+
next: hasMore ? () => this.getUndirectedRelationships(a, b, types, {
|
|
1736
|
+
cursor: nextCursor,
|
|
1737
|
+
limit
|
|
1738
|
+
}, consistencyToken, signal) : void 0
|
|
1739
|
+
};
|
|
1740
|
+
}
|
|
1741
|
+
async getDirectedRelationships(sourceId, targetId, types, paging, consistencyToken, signal) {
|
|
1742
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1743
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1744
|
+
const startIndex = paging?.cursor ? parseInt(paging.cursor) : 0;
|
|
1745
|
+
const limit = paging?.limit || 100;
|
|
1746
|
+
let query = this._db.selectFrom("DocumentRelationship").selectAll().where("sourceId", "=", sourceId).where("targetId", "=", targetId);
|
|
1747
|
+
if (types && types.length > 0) query = query.where("relationshipType", "in", types);
|
|
1748
|
+
const rows = await query.orderBy("createdAt", "asc").orderBy("id", "asc").offset(startIndex).limit(limit + 1).execute();
|
|
1749
|
+
const hasMore = rows.length > limit;
|
|
1750
|
+
const results = hasMore ? rows.slice(0, limit) : rows;
|
|
1751
|
+
const nextCursor = hasMore ? String(startIndex + limit) : void 0;
|
|
1752
|
+
return {
|
|
1753
|
+
results: results.map((row) => ({
|
|
1754
|
+
sourceId: row.sourceId,
|
|
1755
|
+
targetId: row.targetId,
|
|
1756
|
+
relationshipType: row.relationshipType,
|
|
1757
|
+
metadata: row.metadata ? row.metadata : void 0,
|
|
1758
|
+
createdAt: row.createdAt,
|
|
1759
|
+
updatedAt: row.updatedAt
|
|
1760
|
+
})),
|
|
1761
|
+
options: paging || {
|
|
1762
|
+
cursor: "0",
|
|
1763
|
+
limit: 100
|
|
1764
|
+
},
|
|
1765
|
+
nextCursor,
|
|
1766
|
+
next: hasMore ? () => this.getDirectedRelationships(sourceId, targetId, types, {
|
|
1767
|
+
cursor: nextCursor,
|
|
1768
|
+
limit
|
|
1769
|
+
}, consistencyToken, signal) : void 0
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
async findPath(sourceId, targetId, types, consistencyToken, signal) {
|
|
1773
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1774
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1775
|
+
if (sourceId === targetId) return [sourceId];
|
|
1776
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1777
|
+
const queue = [{
|
|
1778
|
+
id: sourceId,
|
|
1779
|
+
path: [sourceId]
|
|
1780
|
+
}];
|
|
1781
|
+
while (queue.length > 0) {
|
|
1782
|
+
const current = queue.shift();
|
|
1783
|
+
if (current.id === targetId) return current.path;
|
|
1784
|
+
if (visited.has(current.id)) continue;
|
|
1785
|
+
visited.add(current.id);
|
|
1786
|
+
const outgoingRelationships = await collectAllPages(await this.getOutgoing(current.id, types, void 0, consistencyToken, signal), signal);
|
|
1787
|
+
for (const rel of outgoingRelationships) if (!visited.has(rel.targetId)) queue.push({
|
|
1788
|
+
id: rel.targetId,
|
|
1789
|
+
path: [...current.path, rel.targetId]
|
|
1790
|
+
});
|
|
1791
|
+
}
|
|
1792
|
+
return null;
|
|
1793
|
+
}
|
|
1794
|
+
async findAncestors(documentId, types, consistencyToken, signal) {
|
|
1795
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1796
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1797
|
+
const nodes = /* @__PURE__ */ new Set([documentId]);
|
|
1798
|
+
const edges = [];
|
|
1799
|
+
const queue = [documentId];
|
|
1800
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1801
|
+
while (queue.length > 0) {
|
|
1802
|
+
const currentId = queue.shift();
|
|
1803
|
+
if (visited.has(currentId)) continue;
|
|
1804
|
+
visited.add(currentId);
|
|
1805
|
+
const incomingRelationships = await collectAllPages(await this.getIncoming(currentId, types, void 0, consistencyToken, signal), signal);
|
|
1806
|
+
for (const rel of incomingRelationships) {
|
|
1807
|
+
nodes.add(rel.sourceId);
|
|
1808
|
+
edges.push({
|
|
1809
|
+
from: rel.sourceId,
|
|
1810
|
+
to: rel.targetId,
|
|
1811
|
+
type: rel.relationshipType
|
|
1812
|
+
});
|
|
1813
|
+
if (!visited.has(rel.sourceId)) queue.push(rel.sourceId);
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
return {
|
|
1817
|
+
nodes: Array.from(nodes),
|
|
1818
|
+
edges
|
|
1819
|
+
};
|
|
1820
|
+
}
|
|
1821
|
+
async getRelationshipTypes(consistencyToken, signal) {
|
|
1822
|
+
if (consistencyToken) await this.waitForConsistency(consistencyToken, void 0, signal);
|
|
1823
|
+
if (signal?.aborted) throw new Error("Operation aborted");
|
|
1824
|
+
return (await this._db.selectFrom("DocumentRelationship").select("relationshipType").distinct().execute()).map((row) => row.relationshipType);
|
|
1825
|
+
}
|
|
1826
|
+
async handleAddRelationship(trx, operation) {
|
|
1827
|
+
const input = operation.action.input;
|
|
1828
|
+
if (!await trx.selectFrom("Document").select("id").where("id", "=", input.sourceId).executeTakeFirst()) await trx.insertInto("Document").values({ id: input.sourceId }).execute();
|
|
1829
|
+
if (!await trx.selectFrom("Document").select("id").where("id", "=", input.targetId).executeTakeFirst()) await trx.insertInto("Document").values({ id: input.targetId }).execute();
|
|
1830
|
+
if (!await trx.selectFrom("DocumentRelationship").select("id").where("sourceId", "=", input.sourceId).where("targetId", "=", input.targetId).where("relationshipType", "=", input.relationshipType).executeTakeFirst()) {
|
|
1831
|
+
const relationship = {
|
|
1832
|
+
id: v4(),
|
|
1833
|
+
sourceId: input.sourceId,
|
|
1834
|
+
targetId: input.targetId,
|
|
1835
|
+
relationshipType: input.relationshipType,
|
|
1836
|
+
metadata: input.metadata || null
|
|
1837
|
+
};
|
|
1838
|
+
await trx.insertInto("DocumentRelationship").values(relationship).execute();
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
async handleRemoveRelationship(trx, operation) {
|
|
1842
|
+
const input = operation.action.input;
|
|
1843
|
+
await trx.deleteFrom("DocumentRelationship").where("sourceId", "=", input.sourceId).where("targetId", "=", input.targetId).where("relationshipType", "=", input.relationshipType).execute();
|
|
1844
|
+
}
|
|
1845
|
+
async handleUpdateRelationship(trx, operation) {
|
|
1846
|
+
const input = operation.action.input;
|
|
1847
|
+
await trx.updateTable("DocumentRelationship").set({
|
|
1848
|
+
metadata: input.metadata,
|
|
1849
|
+
updatedAt: /* @__PURE__ */ new Date()
|
|
1850
|
+
}).where("sourceId", "=", input.sourceId).where("targetId", "=", input.targetId).where("relationshipType", "=", input.relationshipType).execute();
|
|
1851
|
+
}
|
|
1852
|
+
};
|
|
1853
|
+
//#endregion
|
|
1854
|
+
export { isCatchUpConsumer as S, defaultReadModelIndexingConfig as _, KyselyDocumentView as a, defaultCatchUpConfig as b, ReadModelCoordinator as c, releaseBatch as d, reserveBatch as f, DEFAULT_READ_MODEL_YIELD_DEADLINE_MS as g, DEFAULT_COMMIT_CHUNK_SIZE as h, DeletedDocumentRead as i, indexReserved as l, BaseReadModel as m, ConsistencyTracker as n, DOCUMENT_INDEXER_READ_MODEL as o, spliceReadModel as p, makeConsistencyKey as r, DOCUMENT_VIEW_READ_MODEL as s, KyselyDocumentIndexer as t, indexStage as u, unchunkedReadModelIndexingConfig as v, CatchUpScheduler as x, ContiguousCursor as y };
|
|
1855
|
+
|
|
1856
|
+
//# sourceMappingURL=document-indexer-3fkVWCaN.js.map
|