@nodii/telemetry 0.13.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/worker/checkpoint-store.d.ts +66 -0
  2. package/dist/worker/checkpoint-store.d.ts.map +1 -0
  3. package/dist/worker/checkpoint-store.js +131 -0
  4. package/dist/worker/checkpoint-store.js.map +1 -0
  5. package/dist/worker/dead-letter.d.ts +81 -0
  6. package/dist/worker/dead-letter.d.ts.map +1 -0
  7. package/dist/worker/dead-letter.js +165 -0
  8. package/dist/worker/dead-letter.js.map +1 -0
  9. package/dist/worker/dedupe-store.d.ts +46 -0
  10. package/dist/worker/dedupe-store.d.ts.map +1 -0
  11. package/dist/worker/dedupe-store.js +113 -0
  12. package/dist/worker/dedupe-store.js.map +1 -0
  13. package/dist/worker/index.d.ts +23 -0
  14. package/dist/worker/index.d.ts.map +1 -0
  15. package/dist/worker/index.js +46 -0
  16. package/dist/worker/index.js.map +1 -0
  17. package/dist/worker/observability-hooks.d.ts +99 -0
  18. package/dist/worker/observability-hooks.d.ts.map +1 -0
  19. package/dist/worker/observability-hooks.js +189 -0
  20. package/dist/worker/observability-hooks.js.map +1 -0
  21. package/dist/worker/outbox-dispatcher.d.ts +106 -0
  22. package/dist/worker/outbox-dispatcher.d.ts.map +1 -0
  23. package/dist/worker/outbox-dispatcher.js +285 -0
  24. package/dist/worker/outbox-dispatcher.js.map +1 -0
  25. package/dist/worker/replication-worker.d.ts +170 -0
  26. package/dist/worker/replication-worker.d.ts.map +1 -0
  27. package/dist/worker/replication-worker.js +576 -0
  28. package/dist/worker/replication-worker.js.map +1 -0
  29. package/dist/worker/signal-drain.d.ts +83 -0
  30. package/dist/worker/signal-drain.d.ts.map +1 -0
  31. package/dist/worker/signal-drain.js +131 -0
  32. package/dist/worker/signal-drain.js.map +1 -0
  33. package/dist/worker/single-active-lease.d.ts +180 -0
  34. package/dist/worker/single-active-lease.d.ts.map +1 -0
  35. package/dist/worker/single-active-lease.js +394 -0
  36. package/dist/worker/single-active-lease.js.map +1 -0
  37. package/dist/worker/streams-worker.d.ts +256 -0
  38. package/dist/worker/streams-worker.d.ts.map +1 -0
  39. package/dist/worker/streams-worker.js +621 -0
  40. package/dist/worker/streams-worker.js.map +1 -0
  41. package/dist/worker/sweeper-worker.d.ts +107 -0
  42. package/dist/worker/sweeper-worker.d.ts.map +1 -0
  43. package/dist/worker/sweeper-worker.js +245 -0
  44. package/dist/worker/sweeper-worker.js.map +1 -0
  45. package/dist/worker/tx.d.ts +39 -0
  46. package/dist/worker/tx.d.ts.map +1 -0
  47. package/dist/worker/tx.js +51 -0
  48. package/dist/worker/tx.js.map +1 -0
  49. package/dist/worker/worker-handle.d.ts +94 -0
  50. package/dist/worker/worker-handle.d.ts.map +1 -0
  51. package/dist/worker/worker-handle.js +15 -0
  52. package/dist/worker/worker-handle.js.map +1 -0
  53. package/package.json +6 -1
@@ -0,0 +1,576 @@
1
+ // D402 — ReplicationWorker: the Pattern-3 SUBSCRIBER consume side — the
2
+ // canonical `09-replication-doctrine § 4` sequence every replicating service
3
+ // follows.
4
+ //
5
+ // D402 (LOCKED): "ReplicationWorker — Pattern-3, BOTH sides: producer
6
+ // outbox-drain + subscriber consume + snapshot-bootstrap + HWM stitch + dedupe +
7
+ // last-write-wins + audit." This file is the SUBSCRIBER consume side (the
8
+ // producer side is OutboxDispatcher); it GENERALIZES `ts/replica-consumer/src/`
9
+ // (which this lib does NOT edit — replica-consumer gets re-based onto this in
10
+ // Phase 2).
11
+ //
12
+ // ── Composition (which primitives, how) ──────────────────────────────────────
13
+ // - SingleActiveLease — single-active per (service, source_stream); ttl/2
14
+ // renew inside the lease. `validateFence(token)` guards the projection
15
+ // upsert: a lease-lost subscriber mid-apply has its stale write REJECTED.
16
+ // - DedupeStore — `seen(...)` skips an already-applied (event_id, topic);
17
+ // `record(...)` runs INSIDE the apply tx (atomic with projection + cursor).
18
+ // - CheckpointStore — the HWM cursor; `advance(...)` runs in the SAME apply
19
+ // tx. Also stitches snapshot→live: the bootstrap writes the snapshot HWM as
20
+ // the cursor, then live consume resumes from it (§ 7 stitch).
21
+ // - ObservabilityHooks — lease/lag/hwm/depth metrics, canonical names.
22
+ // - SignalDrain — `stop({timeoutMs})` registerable; bounded drain.
23
+ // - DeadLetterSink — INJECTED (no default); a poison event after retries
24
+ // is dead-lettered + XACK'd.
25
+ // - audit hook — an injected `onApplied(...)` audit emitter fires AFTER
26
+ // a successful commit (the § 4 "audit" step).
27
+ //
28
+ // ── LAST-WRITE-WINS (the replication-specific guard) ─────────────────────────
29
+ // Each event carries `entity_version`. The apply is LWW: an event whose
30
+ // `entity_version` is <= the projection's current `source_entity_version` is a
31
+ // STALE replay and MUST NOT regress the projection. The worker compares
32
+ // against the value the injected `currentVersion(...)` returns (the consumer
33
+ // reads its own projection row) and SKIPS the apply (but STILL records dedupe +
34
+ // advances the cursor, so the stale event is consumed-once and never retried).
35
+ // The CheckpointStore's `high_water_entity_version` also tracks the LWW HWM
36
+ // monotonically (GREATEST) as a secondary guard.
37
+ //
38
+ // ── The consume sequence (09-replication § 4) ────────────────────────────────
39
+ // bootstrap (if cursor absent: snapshot → write snapshot HWM cursor) →
40
+ // live consume: XREADGROUP > → for each (id, event):
41
+ // [1] dedupe.seen? → XACK + skip
42
+ // [2] LWW: event.entity_version <= current? → consume-but-skip-apply
43
+ // [3] apply tx: projection upsert (fence-guarded) + dedupe.record +
44
+ // checkpoint.advance(+hwev) — ONE tx, atomic
45
+ // [4] XACK; audit hook; on error retry-then-dead-letter.
46
+ import { ensurePiiRedaction } from "./dead-letter.js";
47
+ import { hwmGaugeValue, ObservabilityHooks } from "./observability-hooks.js";
48
+ import { FenceCheckUnavailableError, FenceLostError, SingleActiveLease, } from "./single-active-lease.js";
49
+ import { parseAutoClaimEntriesPartitioned, parseStreamEntriesPartitioned, PoisonApplyError, } from "./streams-worker.js";
50
+ const DEFAULT_BATCH = 32;
51
+ const DEFAULT_BLOCK_MS = 2_000;
52
+ const DEFAULT_RETRY_MAX = 3;
53
+ const DEFAULT_STOP_TIMEOUT_MS = 30_000;
54
+ const DEFAULT_RECLAIM_MIN_IDLE_MS = 30_000;
55
+ function toBigint(v) {
56
+ return typeof v === "bigint" ? v : BigInt(v);
57
+ }
58
+ /** Consumer-group name — `${service}:${stream}` (canonical per replica-consumer). */
59
+ export function replicationConsumerGroup(service, stream) {
60
+ return `${service}:${stream}`;
61
+ }
62
+ /**
63
+ * ReplicationWorker — the Pattern-3 subscriber. Composes lease + dedupe +
64
+ * checkpoint + hooks + drain + snapshot-bootstrap + LWW + audit. EXACTLY-ONCE
65
+ * apply, monotonic projection (no LWW regression), atomic cursor+dedupe+upsert.
66
+ */
67
+ export class ReplicationWorker {
68
+ redis;
69
+ sourceStream;
70
+ applyHandler;
71
+ currentVersion;
72
+ begin;
73
+ dedupe;
74
+ checkpoint;
75
+ lease;
76
+ subscription;
77
+ deadLetter;
78
+ snapshot;
79
+ audit;
80
+ retryMaxAttempts;
81
+ pollBatch;
82
+ blockMs;
83
+ reclaimMinIdleMs;
84
+ hooks;
85
+ autoStart;
86
+ group;
87
+ consumer;
88
+ retries = new Map();
89
+ _running = false;
90
+ _stopped = false;
91
+ _inFlight = 0;
92
+ _bootstrapped = false;
93
+ _hwm = {
94
+ lastStreamId: null,
95
+ highWaterEntityVersion: null,
96
+ };
97
+ _lastAppliedAtMs = null;
98
+ groupEnsured = false;
99
+ loop = Promise.resolve();
100
+ constructor(opts) {
101
+ this.redis = opts.redis;
102
+ this.sourceStream = opts.sourceStream;
103
+ this.applyHandler = opts.apply;
104
+ this.currentVersion = opts.currentVersion;
105
+ this.begin = opts.begin;
106
+ this.dedupe = opts.dedupe;
107
+ this.checkpoint = opts.checkpoint;
108
+ this.lease = new SingleActiveLease({
109
+ ...opts.lease,
110
+ onLeaseLost: () => this.hooks.leaseLost(),
111
+ });
112
+ this.subscription = opts.subscription ?? `replication:${opts.sourceStream}`;
113
+ this.deadLetter = ensurePiiRedaction(opts.deadLetter);
114
+ this.snapshot = opts.snapshot;
115
+ this.audit = opts.audit;
116
+ this.retryMaxAttempts = opts.retryMaxAttempts ?? DEFAULT_RETRY_MAX;
117
+ this.pollBatch = opts.pollBatch ?? DEFAULT_BATCH;
118
+ this.blockMs = opts.blockMs ?? DEFAULT_BLOCK_MS;
119
+ this.reclaimMinIdleMs =
120
+ opts.reclaimMinIdleMs ??
121
+ Math.max(opts.lease.ttlMs, DEFAULT_RECLAIM_MIN_IDLE_MS);
122
+ this.autoStart = opts.autoStart !== false;
123
+ this.hooks =
124
+ opts.hooks ??
125
+ new ObservabilityHooks({
126
+ attributes: {
127
+ service: opts.lease.service,
128
+ scope: opts.lease.scope,
129
+ worker: opts.workerLabel ?? "replication",
130
+ },
131
+ });
132
+ this.group = replicationConsumerGroup(opts.lease.service, opts.sourceStream);
133
+ this.consumer = `${opts.lease.service}:${this.lease.ownerId}`;
134
+ }
135
+ get running() {
136
+ return this._running;
137
+ }
138
+ get leaseAcquired() {
139
+ return this.lease.leaseAcquired;
140
+ }
141
+ get lag() {
142
+ if (this._lastAppliedAtMs === null)
143
+ return null;
144
+ return Math.max(0, (Date.now() - this._lastAppliedAtMs) / 1000);
145
+ }
146
+ get currentHwm() {
147
+ return this._hwm;
148
+ }
149
+ /** True once the snapshot bootstrap (if any) has run / been determined absent. */
150
+ get bootstrapped() {
151
+ return this._bootstrapped;
152
+ }
153
+ async ensureGroup() {
154
+ if (this.groupEnsured)
155
+ return;
156
+ try {
157
+ await this.redis.xgroup("CREATE", this.sourceStream, this.group, "$", "MKSTREAM");
158
+ }
159
+ catch (err) {
160
+ const msg = err instanceof Error ? err.message : String(err);
161
+ if (!/BUSYGROUP/i.test(msg))
162
+ throw err;
163
+ }
164
+ this.groupEnsured = true;
165
+ }
166
+ async start() {
167
+ if (this._running)
168
+ return;
169
+ this._stopped = false;
170
+ const acquired = await this.lease.acquire();
171
+ if (acquired)
172
+ this.hooks.leaseAcquired();
173
+ // ensureGroup (XGROUP CREATE) is an idempotent no-op when the group exists —
174
+ // safe regardless of lease ownership. The SNAPSHOT BOOTSTRAP (which WRITES the
175
+ // projection + cursor) runs ONLY when we hold the lease: a non-holder Start()
176
+ // must NOT bootstrap, or two cold instances racing into the shared projection
177
+ // would both snapshot (single-active violation). A non-holder idles; the loop
178
+ // re-acquires + bootstraps (lease-guarded) once it wins. BUG-1: release the
179
+ // lease on a boot-step failure after a successful acquire (never orphan it).
180
+ try {
181
+ await this.ensureGroup();
182
+ if (acquired)
183
+ await this.maybeBootstrap();
184
+ }
185
+ catch (err) {
186
+ if (acquired)
187
+ await this.lease.release();
188
+ throw err;
189
+ }
190
+ this._running = true;
191
+ // The background consume loop. A test may set `autoStart:false` to drive
192
+ // `runOnce()` deterministically without the loop racing the explicit ticks.
193
+ if (this.autoStart)
194
+ this.loop = this.runLoop();
195
+ }
196
+ /**
197
+ * Snapshot-bootstrap + HWM stitch (§ 7): if the durable cursor is ABSENT (cold
198
+ * start) AND a snapshot source is wired, run the snapshot bootstrap inside ONE
199
+ * tx and write the returned high-water stream id as the cursor — live consume
200
+ * then resumes EXACTLY from there (no gap, no double-apply of the snapshot
201
+ * window). If a cursor already exists we skip the snapshot and resume live.
202
+ */
203
+ async maybeBootstrap() {
204
+ if (this._bootstrapped)
205
+ return;
206
+ const existing = await this.begin((tx) => this.checkpoint.get(this.subscription, tx));
207
+ if (existing) {
208
+ // Warm start — resume live from the durable cursor.
209
+ this._hwm = {
210
+ lastStreamId: existing.lastStreamId,
211
+ highWaterEntityVersion: existing.highWaterEntityVersion,
212
+ };
213
+ this._bootstrapped = true;
214
+ return;
215
+ }
216
+ const snapshot = this.snapshot;
217
+ if (!snapshot) {
218
+ // No snapshot wired + no cursor → consume live from `>` (group at `$`).
219
+ this._bootstrapped = true;
220
+ return;
221
+ }
222
+ // Cold start WITH a snapshot — bootstrap + stitch the cursor in ONE tx so
223
+ // the snapshot apply + the cursor write commit together-or-none.
224
+ await this.begin(async (tx) => {
225
+ const { highWaterStreamId } = await snapshot.bootstrap(tx);
226
+ await this.checkpoint.advance({ subscription: this.subscription, lastStreamId: highWaterStreamId }, tx);
227
+ });
228
+ this._hwm = {
229
+ lastStreamId: null,
230
+ highWaterEntityVersion: null,
231
+ };
232
+ const cp = await this.begin((tx) => this.checkpoint.get(this.subscription, tx));
233
+ if (cp) {
234
+ this._hwm = {
235
+ lastStreamId: cp.lastStreamId,
236
+ highWaterEntityVersion: cp.highWaterEntityVersion,
237
+ };
238
+ }
239
+ this._bootstrapped = true;
240
+ }
241
+ async runLoop() {
242
+ while (!this._stopped) {
243
+ if (this.lease.lost)
244
+ break;
245
+ try {
246
+ const processed = await this.pollOnce();
247
+ if (processed === 0) {
248
+ await new Promise((r) => setTimeout(r, 0));
249
+ }
250
+ }
251
+ catch (err) {
252
+ // BUG-2 fix: a poll that throws BECAUSE we're shutting down (stop() set
253
+ // `_stopped`) is a CLEAN cancel, not a redis failure — exit quietly.
254
+ // A genuine poll-level error (XREADGROUP rejected — Redis down) while
255
+ // still running is the Pillar-5 infra-failure case: back off, never crash.
256
+ if (this._stopped)
257
+ break;
258
+ this.hooks.recordDepth(-1, { error: "poll_failed" });
259
+ void err;
260
+ await new Promise((r) => setTimeout(r, 200));
261
+ }
262
+ }
263
+ }
264
+ async pollOnce() {
265
+ let reacquiredThisPoll = false;
266
+ if (!this.lease.leaseAcquired) {
267
+ const got = await this.lease.acquire();
268
+ if (got) {
269
+ reacquiredThisPoll = true;
270
+ this.hooks.leaseAcquired();
271
+ // Re-ensure the consumer group on every re-acquire (a Redis failover may
272
+ // have wiped it — a cached `groupEnsured` would let xreadgroup hit
273
+ // NOGROUP forever).
274
+ this.groupEnsured = false;
275
+ // BUG fix (in-loop re-acquire): release the freshly-taken lease if
276
+ // ensureGroup throws — otherwise the renewal loop holds it INDEFINITELY
277
+ // while the next poll hammers a never-created group (NOGROUP loop).
278
+ try {
279
+ await this.ensureGroup();
280
+ }
281
+ catch (err) {
282
+ await this.lease.release();
283
+ throw err;
284
+ }
285
+ }
286
+ else {
287
+ return 0;
288
+ }
289
+ }
290
+ if (!this._bootstrapped) {
291
+ // Same in-loop rule: a bootstrap failure right after THIS poll re-acquired
292
+ // must release the freshly-taken lease (a lease held from a prior poll is
293
+ // left intact — maybeBootstrap won't run then, `_bootstrapped` is true).
294
+ try {
295
+ await this.maybeBootstrap();
296
+ }
297
+ catch (err) {
298
+ if (reacquiredThisPoll)
299
+ await this.lease.release();
300
+ throw err;
301
+ }
302
+ }
303
+ let processed = 0;
304
+ // [reclaim] CRASH-RECOVERY reaper: XAUTOCLAIM transfers PENDING entries idle
305
+ // ≥ reclaimMinIdleMs from a DEAD predecessor to US (the single-active
306
+ // successor) — without it a crashed worker's un-ACK'd entries are
307
+ // PERMANENTLY ORPHANED (silent loss; the "no-reaper" defect D402 eliminates).
308
+ // The fence guard + LWW still protect against a still-alive predecessor.
309
+ const claimedRaw = await this.redis.xautoclaim(this.sourceStream, this.group, this.consumer, this.reclaimMinIdleMs, "0-0", "COUNT", this.pollBatch);
310
+ const claimed = parseAutoClaimEntriesPartitioned(claimedRaw);
311
+ for (const p of claimed.entries) {
312
+ if (this._stopped || this.lease.lost)
313
+ break;
314
+ await this.applyOne(p.id, p.event);
315
+ processed += 1;
316
+ }
317
+ processed += await this.handleMalformedEntries(claimed.malformed);
318
+ // [retry] Re-read our OWN PEL (id `0`) for NACK'd entries — they must be
319
+ // retried, and a plain `>` read never re-returns a PEL entry.
320
+ const pendingRaw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "STREAMS", this.sourceStream, "0");
321
+ const pending = parseStreamEntriesPartitioned(pendingRaw);
322
+ for (const p of pending.entries) {
323
+ if (this._stopped || this.lease.lost)
324
+ break;
325
+ await this.applyOne(p.id, p.event);
326
+ processed += 1;
327
+ }
328
+ processed += await this.handleMalformedEntries(pending.malformed);
329
+ // [new] Read brand-new entries (`>`, blocking).
330
+ const raw = await this.redis.xreadgroup("GROUP", this.group, this.consumer, "COUNT", this.pollBatch, "BLOCK", this.blockMs, "STREAMS", this.sourceStream, ">");
331
+ try {
332
+ const depth = await this.redis.xlen(this.sourceStream);
333
+ this.hooks.recordDepth(depth);
334
+ }
335
+ catch {
336
+ /* depth advisory */
337
+ }
338
+ const fresh = parseStreamEntriesPartitioned(raw);
339
+ for (const p of fresh.entries) {
340
+ if (this._stopped || this.lease.lost)
341
+ break;
342
+ await this.applyOne(p.id, p.event);
343
+ processed += 1;
344
+ }
345
+ processed += await this.handleMalformedEntries(fresh.malformed);
346
+ return processed;
347
+ }
348
+ /**
349
+ * Dead-letter + XACK every structurally-unparseable (malformed-JSON) entry. A
350
+ * malformed entry is a POISON message (no decodable event) — leaving it in the
351
+ * PEL only re-cycles it through XAUTOCLAIM forever. Route the raw bytes through
352
+ * the (PII-redacting) dead-letter sink, then XACK so it leaves the PEL. If the
353
+ * sink fails, leave the entry in the PEL (recoverable; never silently dropped).
354
+ */
355
+ async handleMalformedEntries(malformed) {
356
+ let handled = 0;
357
+ for (const m of malformed) {
358
+ if (this._stopped || this.lease.lost)
359
+ break;
360
+ try {
361
+ await this.deadLetter.deadLetter({
362
+ sourceStream: this.sourceStream,
363
+ topic: "",
364
+ eventId: "",
365
+ payload: { raw: m.rawPayload },
366
+ errorMessage: "malformed stream entry: event field is not valid JSON",
367
+ retryCount: 0,
368
+ });
369
+ await this.redis.xack(this.sourceStream, this.group, m.id);
370
+ this.retries.delete(m.id);
371
+ handled += 1;
372
+ }
373
+ catch {
374
+ this.hooks.recordDepth(-1, { error: "malformed_dead_letter_failed" });
375
+ }
376
+ }
377
+ return handled;
378
+ }
379
+ /**
380
+ * Apply one replication event: dedupe-skip → LWW check → apply tx (projection
381
+ * + dedupe.record + checkpoint.advance, fence-guarded, ONE tx) → XACK → audit.
382
+ */
383
+ async applyOne(streamId, event) {
384
+ if (this.lease.lost)
385
+ return;
386
+ this._inFlight += 1;
387
+ const token = this.lease.fencingToken;
388
+ try {
389
+ // [1] Pre-tx dedupe probe.
390
+ const alreadySeen = await this.begin((tx) => this.dedupe.seen(event.event_id, event.topic, tx));
391
+ if (alreadySeen) {
392
+ // The event already committed — only the XACK is outstanding. A failing
393
+ // XACK is a TRANSIENT ack-failure, NOT a poison event: swallow it (re-acked
394
+ // next poll). NEVER route a committed event to retry/DLQ over an ack blip.
395
+ await this.ackSwallow(streamId);
396
+ this.retries.delete(streamId);
397
+ return;
398
+ }
399
+ const incomingVersion = toBigint(event.entity_version);
400
+ let applied = false;
401
+ // [2]+[3] Apply tx: LWW check + projection + dedupe + cursor — ONE tx.
402
+ await this.begin(async (tx) => {
403
+ // LWW: read the projection's current version IN the tx so the compare
404
+ // is consistent with the upsert.
405
+ const current = await this.currentVersion(tx, event);
406
+ const isStale = current !== null && incomingVersion <= current;
407
+ if (!isStale) {
408
+ // Apply the projection (fence-guarded BEFORE the write).
409
+ const ctx = {
410
+ tx,
411
+ event,
412
+ streamId,
413
+ validateFence: async () => {
414
+ if (token === null) {
415
+ throw new FenceLostError({
416
+ leaseKey: this.lease.key,
417
+ expectedToken: -1,
418
+ current: null,
419
+ });
420
+ }
421
+ await this.lease.validateFence(token);
422
+ },
423
+ };
424
+ // TAG the apply handler's throws as POISON (vs a TRANSIENT store-down
425
+ // throw of the surrounding tx / currentVersion / dedupe.record /
426
+ // checkpoint.advance). A FenceLostError raised in the handler's
427
+ // validateFence is re-thrown bare so the outer FenceLostError branch
428
+ // still catches it.
429
+ try {
430
+ await this.applyHandler(ctx);
431
+ }
432
+ catch (handlerErr) {
433
+ if (handlerErr instanceof FenceLostError ||
434
+ handlerErr instanceof FenceCheckUnavailableError) {
435
+ throw handlerErr;
436
+ }
437
+ throw new PoisonApplyError(handlerErr);
438
+ }
439
+ applied = true;
440
+ }
441
+ // ALWAYS record dedupe + advance the cursor — even on an LWW-skip the
442
+ // event is consumed-once (it must never be retried) and the cursor must
443
+ // advance past it. The hwev advances monotonically (GREATEST) regardless.
444
+ await this.dedupe.record(event.event_id, event.topic, tx);
445
+ await this.checkpoint.advance({
446
+ subscription: this.subscription,
447
+ lastStreamId: streamId,
448
+ highWaterEntityVersion: incomingVersion,
449
+ }, tx);
450
+ });
451
+ // [4] Commit succeeded → XACK + audit + advance local HWM. The XACK is
452
+ // POST-COMMIT and NOT part of the apply tx: a failing XACK is a
453
+ // transient ack-failure, NOT a poison event — swallow it (re-acked next
454
+ // poll; the dedupe row makes a redelivery a no-op). Do NOT route a
455
+ // successfully-committed event to retry/DLQ over an ack blip.
456
+ await this.ackSwallow(streamId);
457
+ this.retries.delete(streamId);
458
+ const prevHwev = this._hwm.highWaterEntityVersion ?? null;
459
+ this._hwm = {
460
+ lastStreamId: streamId,
461
+ highWaterEntityVersion: prevHwev !== null && prevHwev > incomingVersion
462
+ ? prevHwev
463
+ : incomingVersion,
464
+ };
465
+ this._lastAppliedAtMs = Date.now();
466
+ this.hooks.recordHwm(hwmGaugeValue(streamId));
467
+ if (event.occurred_at_ms !== undefined) {
468
+ this.hooks.recordLag(Math.max(0, (Date.now() - event.occurred_at_ms) / 1000));
469
+ }
470
+ else {
471
+ this.hooks.recordLag(0);
472
+ }
473
+ if (this.audit) {
474
+ try {
475
+ await this.audit({ event, streamId, applied });
476
+ }
477
+ catch {
478
+ // The audit hook is best-effort AFTER commit — a failure must not
479
+ // un-apply the projection (already committed); log-and-continue.
480
+ }
481
+ }
482
+ }
483
+ catch (err) {
484
+ if (err instanceof FenceLostError) {
485
+ this.hooks.leaseLost();
486
+ return;
487
+ }
488
+ // Transient fence-check redis failure: fail closed, leave the event in the
489
+ // PEL (no XACK, no DLQ) to reprocess on recovery — cross-runtime-identical.
490
+ if (err instanceof FenceCheckUnavailableError) {
491
+ this.hooks.recordDepth(-1, { error: "fence_check_unavailable" });
492
+ return;
493
+ }
494
+ // POISON (the injected apply handler rejected on a well-formed event) is the
495
+ // ONLY thing that travels the retry→DLQ path.
496
+ if (err instanceof PoisonApplyError) {
497
+ await this.retryOrDeadLetter(streamId, event, err.cause);
498
+ return;
499
+ }
500
+ // ANY OTHER throw is a TRANSIENT INFRA failure (the pre-tx dedupe probe, the
501
+ // tx open/commit, currentVersion, dedupe.record, or checkpoint.advance threw
502
+ // because the store is unreachable). A healthy event must NOT be
503
+ // dead-lettered over a store outage — leave it in the PEL, no retry-bump, no
504
+ // XACK, no DLQ; retry on recovery (symmetric with the redis-down fence path).
505
+ this.hooks.recordDepth(-1, { error: "transient_infra" });
506
+ return;
507
+ }
508
+ finally {
509
+ this._inFlight -= 1;
510
+ }
511
+ }
512
+ /**
513
+ * XACK an already-committed entry, SWALLOWING a transient ack failure. The
514
+ * XACK is post-commit (or post-dedupe-skip) and NOT part of the apply tx — a
515
+ * failing XACK is a transient redis blip, never a poison event. On failure the
516
+ * entry stays un-acked in the PEL and is re-acked next poll once redis
517
+ * recovers (the durable dedupe row makes the redelivery a no-op). Matches Go's
518
+ * `_, _ = Ack(...)`; a throw here must NEVER reach retry/DLQ.
519
+ */
520
+ async ackSwallow(streamId) {
521
+ try {
522
+ await this.redis.xack(this.sourceStream, this.group, streamId);
523
+ }
524
+ catch {
525
+ this.hooks.recordDepth(-1, { error: "ack_failed" });
526
+ }
527
+ }
528
+ async retryOrDeadLetter(streamId, event, err) {
529
+ const attempt = (this.retries.get(streamId) ?? 0) + 1;
530
+ this.retries.set(streamId, attempt);
531
+ if (attempt < this.retryMaxAttempts)
532
+ return; // stays in the PEL, redelivered.
533
+ try {
534
+ await this.deadLetter.deadLetter({
535
+ sourceStream: this.sourceStream,
536
+ topic: event.topic,
537
+ eventId: event.event_id,
538
+ payload: event,
539
+ errorMessage: err instanceof Error ? err.message : String(err),
540
+ retryCount: attempt,
541
+ });
542
+ await this.redis.xack(this.sourceStream, this.group, streamId);
543
+ this.retries.delete(streamId);
544
+ }
545
+ catch {
546
+ // Dead-letter sink failed — leave in the PEL rather than lose it.
547
+ }
548
+ }
549
+ async runOnce() {
550
+ return this.pollOnce();
551
+ }
552
+ async stop(opts) {
553
+ const timeoutMs = opts?.timeoutMs ?? DEFAULT_STOP_TIMEOUT_MS;
554
+ const start = Date.now();
555
+ this._stopped = true;
556
+ this._running = false;
557
+ try {
558
+ await Promise.race([
559
+ this.loop,
560
+ new Promise((r) => setTimeout(r, timeoutMs)),
561
+ ]);
562
+ }
563
+ catch {
564
+ /* swallow */
565
+ }
566
+ while (this._inFlight > 0 && Date.now() - start < timeoutMs) {
567
+ await new Promise((r) => setTimeout(r, 10));
568
+ }
569
+ const drainedCleanly = this._inFlight === 0;
570
+ await this.lease.release();
571
+ const durationMs = Date.now() - start;
572
+ this.hooks.recordDrainDuration(durationMs);
573
+ return { drainedCleanly, durationMs };
574
+ }
575
+ }
576
+ //# sourceMappingURL=replication-worker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"replication-worker.js","sourceRoot":"","sources":["../../src/worker/replication-worker.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,6EAA6E;AAC7E,WAAW;AACX,EAAE;AACF,sEAAsE;AACtE,iFAAiF;AACjF,0EAA0E;AAC1E,gFAAgF;AAChF,8EAA8E;AAC9E,YAAY;AACZ,EAAE;AACF,gFAAgF;AAChF,6EAA6E;AAC7E,2EAA2E;AAC3E,8EAA8E;AAC9E,mFAAmF;AACnF,gFAAgF;AAChF,iFAAiF;AACjF,gFAAgF;AAChF,kEAAkE;AAClE,yEAAyE;AACzE,4EAA4E;AAC5E,+EAA+E;AAC/E,iCAAiC;AACjC,kFAAkF;AAClF,kDAAkD;AAClD,EAAE;AACF,gFAAgF;AAChF,0EAA0E;AAC1E,iFAAiF;AACjF,0EAA0E;AAC1E,+EAA+E;AAC/E,kFAAkF;AAClF,iFAAiF;AACjF,8EAA8E;AAC9E,mDAAmD;AACnD,EAAE;AACF,gFAAgF;AAChF,yEAAyE;AACzE,uDAAuD;AACvD,qCAAqC;AACrC,yEAAyE;AACzE,wEAAwE;AACxE,qDAAqD;AACrD,6DAA6D;AAG7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAGtD,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC7E,OAAO,EACL,0BAA0B,EAC1B,cAAc,EACd,iBAAiB,GAElB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,gCAAgC,EAChC,6BAA6B,EAC7B,gBAAgB,GAEjB,MAAM,qBAAqB,CAAC;AAiG7B,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAC/B,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,uBAAuB,GAAG,MAAM,CAAC;AACvC,MAAM,2BAA2B,GAAG,MAAM,CAAC;AAE3C,SAAS,QAAQ,CAAC,CAA2B;IAC3C,OAAO,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,wBAAwB,CACtC,OAAe,EACf,MAAc;IAEd,OAAO,GAAG,OAAO,IAAI,MAAM,EAAE,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,iBAAiB;IACX,KAAK,CAAmB;IACxB,YAAY,CAAS;IACrB,YAAY,CAA0B;IACtC,cAAc,CAGH;IACX,KAAK,CAEN;IACC,MAAM,CAAc;IACpB,UAAU,CAAkB;IAC5B,KAAK,CAAoB;IACzB,YAAY,CAAS;IACrB,UAAU,CAAiB;IAC3B,QAAQ,CAAqB;IAC7B,KAAK,CAAwB;IAC7B,gBAAgB,CAAS;IACzB,SAAS,CAAS;IAClB,OAAO,CAAS;IAChB,gBAAgB,CAAS;IACzB,KAAK,CAAqB;IAC1B,SAAS,CAAU;IACnB,KAAK,CAAS;IACd,QAAQ,CAAS;IAEjB,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,QAAQ,GAAG,KAAK,CAAC;IACjB,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACd,aAAa,GAAG,KAAK,CAAC;IACtB,IAAI,GAAc;QACxB,YAAY,EAAE,IAAI;QAClB,sBAAsB,EAAE,IAAI;KAC7B,CAAC;IACM,gBAAgB,GAAkB,IAAI,CAAC;IACvC,YAAY,GAAG,KAAK,CAAC;IACrB,IAAI,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEhD,YAAY,IAA2B;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAiB,CAAC;YACjC,GAAG,IAAI,CAAC,KAAK;YACb,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;SAC1C,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,eAAe,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5E,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,aAAa,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;QAChD,IAAI,CAAC,gBAAgB;YACnB,IAAI,CAAC,gBAAgB;gBACrB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;QAC1D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC;QAC1C,IAAI,CAAC,KAAK;YACR,IAAI,CAAC,KAAK;gBACV,IAAI,kBAAkB,CAAC;oBACrB,UAAU,EAAE;wBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;wBAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;wBACvB,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,aAAa;qBAC1C;iBACF,CAAC,CAAC;QACL,IAAI,CAAC,KAAK,GAAG,wBAAwB,CACnC,IAAI,CAAC,KAAK,CAAC,OAAO,EAClB,IAAI,CAAC,YAAY,CAClB,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAChE,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;IAClC,CAAC;IAED,IAAI,GAAG;QACL,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAED,kFAAkF;IAClF,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAC9B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CACrB,QAAQ,EACR,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,GAAG,EACH,UAAU,CACX,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM,GAAG,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,QAAQ;YAAE,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QACzC,6EAA6E;QAC7E,+EAA+E;QAC/E,8EAA8E;QAC9E,8EAA8E;QAC9E,8EAA8E;QAC9E,4EAA4E;QAC5E,6EAA6E;QAC7E,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,QAAQ;gBAAE,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,QAAQ;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACzC,MAAM,GAAG,CAAC;QACZ,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,yEAAyE;QACzE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IACjD,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO;QAC/B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CACvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAC3C,CAAC;QACF,IAAI,QAAQ,EAAE,CAAC;YACb,oDAAoD;YACpD,IAAI,CAAC,IAAI,GAAG;gBACV,YAAY,EAAE,QAAQ,CAAC,YAAY;gBACnC,sBAAsB,EAAE,QAAQ,CAAC,sBAAsB;aACxD,CAAC;YACF,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,wEAAwE;YACxE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,0EAA0E;QAC1E,iEAAiE;QACjE,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YAC5B,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC3D,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3B,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,iBAAiB,EAAE,EACpE,EAAE,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG;YACV,YAAY,EAAE,IAAI;YAClB,sBAAsB,EAAE,IAAI;SAC7B,CAAC;QACF,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CACjC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC,CAC3C,CAAC;QACF,IAAI,EAAE,EAAE,CAAC;YACP,IAAI,CAAC,IAAI,GAAG;gBACV,YAAY,EAAE,EAAE,CAAC,YAAY;gBAC7B,sBAAsB,EAAE,EAAE,CAAC,sBAAsB;aAClD,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC3B,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACxC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;oBACpB,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,wEAAwE;gBACxE,qEAAqE;gBACrE,sEAAsE;gBACtE,2EAA2E;gBAC3E,IAAI,IAAI,CAAC,QAAQ;oBAAE,MAAM;gBACzB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;gBACrD,KAAK,GAAG,CAAC;gBACT,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ;QACpB,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,GAAG,EAAE,CAAC;gBACR,kBAAkB,GAAG,IAAI,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;gBAC3B,yEAAyE;gBACzE,mEAAmE;gBACnE,oBAAoB;gBACpB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;gBAC1B,mEAAmE;gBACnE,wEAAwE;gBACxE,oEAAoE;gBACpE,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC3B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;oBAC3B,MAAM,GAAG,CAAC;gBACZ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,CAAC;YACX,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,2EAA2E;YAC3E,0EAA0E;YAC1E,yEAAyE;YACzE,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,kBAAkB;oBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QACD,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,6EAA6E;QAC7E,sEAAsE;QACtE,kEAAkE;QAClE,8EAA8E;QAC9E,yEAAyE;QACzE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAC5C,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,gBAAgB,EACrB,KAAK,EACL,OAAO,EACP,IAAI,CAAC,SAAS,CACf,CAAC;QACF,MAAM,OAAO,GAAG,gCAAgC,CAAC,UAAU,CAAC,CAAC;QAC7D,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAGtB,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,SAAS,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAElE,yEAAyE;QACzE,8DAA8D;QAC9D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAC5C,OAAO,EACP,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,IAAI,CAAC,SAAS,EACd,SAAS,EACT,IAAI,CAAC,YAAY,EACjB,GAAG,CACJ,CAAC;QACF,MAAM,OAAO,GAAG,6BAA6B,CAAC,UAAU,CAAC,CAAC;QAC1D,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAGtB,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,SAAS,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAElE,gDAAgD;QAChD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACrC,OAAO,EACP,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,IAAI,CAAC,SAAS,EACd,OAAO,EACP,IAAI,CAAC,OAAO,EACZ,SAAS,EACT,IAAI,CAAC,YAAY,EACjB,GAAG,CACJ,CAAC;QACF,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACvD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,oBAAoB;QACtB,CAAC;QACD,MAAM,KAAK,GAAG,6BAA6B,CAAC,GAAG,CAAC,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAGpB,EAAE,CAAC;YACH,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,SAAS,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,sBAAsB,CAClC,SAAiC;QAEjC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;gBAAE,MAAM;YAC5C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;oBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,KAAK,EAAE,EAAE;oBACT,OAAO,EAAE,EAAE;oBACX,OAAO,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,UAAU,EAAE;oBAC9B,YAAY,EAAE,uDAAuD;oBACrE,UAAU,EAAE,CAAC;iBACd,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC1B,OAAO,IAAI,CAAC,CAAC;YACf,CAAC;YAAC,MAAM,CAAC;gBACP,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,8BAA8B,EAAE,CAAC,CAAC;YACxE,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,QAAQ,CACpB,QAAgB,EAChB,KAAuB;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,OAAO;QAC5B,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC;YACH,2BAA2B;YAC3B,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAC1C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAClD,CAAC;YACF,IAAI,WAAW,EAAE,CAAC;gBAChB,wEAAwE;gBACxE,4EAA4E;gBAC5E,2EAA2E;gBAC3E,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC9B,OAAO;YACT,CAAC;YAED,MAAM,eAAe,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACvD,IAAI,OAAO,GAAG,KAAK,CAAC;YAEpB,uEAAuE;YACvE,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5B,sEAAsE;gBACtE,iCAAiC;gBACjC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,OAAO,KAAK,IAAI,IAAI,eAAe,IAAI,OAAO,CAAC;gBAE/D,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,yDAAyD;oBACzD,MAAM,GAAG,GAA4B;wBACnC,EAAE;wBACF,KAAK;wBACL,QAAQ;wBACR,aAAa,EAAE,KAAK,IAAI,EAAE;4BACxB,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gCACnB,MAAM,IAAI,cAAc,CAAC;oCACvB,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG;oCACxB,aAAa,EAAE,CAAC,CAAC;oCACjB,OAAO,EAAE,IAAI;iCACd,CAAC,CAAC;4BACL,CAAC;4BACD,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;wBACxC,CAAC;qBACF,CAAC;oBACF,sEAAsE;oBACtE,iEAAiE;oBACjE,gEAAgE;oBAChE,qEAAqE;oBACrE,oBAAoB;oBACpB,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;oBAC/B,CAAC;oBAAC,OAAO,UAAU,EAAE,CAAC;wBACpB,IACE,UAAU,YAAY,cAAc;4BACpC,UAAU,YAAY,0BAA0B,EAChD,CAAC;4BACD,MAAM,UAAU,CAAC;wBACnB,CAAC;wBACD,MAAM,IAAI,gBAAgB,CAAC,UAAU,CAAC,CAAC;oBACzC,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;gBACD,sEAAsE;gBACtE,wEAAwE;gBACxE,0EAA0E;gBAC1E,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3B;oBACE,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,YAAY,EAAE,QAAQ;oBACtB,sBAAsB,EAAE,eAAe;iBACxC,EACD,EAAE,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,uEAAuE;YACvE,oEAAoE;YACpE,4EAA4E;YAC5E,uEAAuE;YACvE,kEAAkE;YAClE,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,IAAI,IAAI,CAAC;YAC1D,IAAI,CAAC,IAAI,GAAG;gBACV,YAAY,EAAE,QAAQ;gBACtB,sBAAsB,EACpB,QAAQ,KAAK,IAAI,IAAI,QAAQ,GAAG,eAAe;oBAC7C,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,eAAe;aACtB,CAAC;YACF,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC9C,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;gBACvC,IAAI,CAAC,KAAK,CAAC,SAAS,CAClB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CACxD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;YACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;gBACjD,CAAC;gBAAC,MAAM,CAAC;oBACP,kEAAkE;oBAClE,iEAAiE;gBACnE,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,cAAc,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,2EAA2E;YAC3E,4EAA4E;YAC5E,IAAI,GAAG,YAAY,0BAA0B,EAAE,CAAC;gBAC9C,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;gBACjE,OAAO;YACT,CAAC;YACD,6EAA6E;YAC7E,8CAA8C;YAC9C,IAAI,GAAG,YAAY,gBAAgB,EAAE,CAAC;gBACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;gBACzD,OAAO;YACT,CAAC;YACD,6EAA6E;YAC7E,6EAA6E;YAC7E,iEAAiE;YACjE,6EAA6E;YAC7E,8EAA8E;YAC9E,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACzD,OAAO;QACT,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,UAAU,CAAC,QAAgB;QACvC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACjE,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,QAAgB,EAChB,KAAuB,EACvB,GAAY;QAEZ,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpC,IAAI,OAAO,GAAG,IAAI,CAAC,gBAAgB;YAAE,OAAO,CAAC,iCAAiC;QAC9E,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;gBAC/B,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,KAAK,CAAC,QAAQ;gBACvB,OAAO,EAAE,KAAgC;gBACzC,YAAY,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBAC9D,UAAU,EAAE,OAAO;aACpB,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAC/D,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;QACpE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAwB;QACjC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,uBAAuB,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,IAAI,CAAC,IAAI;gBACT,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;aACnD,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,aAAa;QACf,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,SAAS,EAAE,CAAC;YAC5D,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;QAC3C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,CAAC;IACxC,CAAC;CACF"}