@semics-tech/mongolite 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +80 -21
  2. package/dist/cjs/collection.js +9 -6
  3. package/dist/cjs/cursors/findCursor.js +6 -4
  4. package/dist/cjs/index.js +108 -1
  5. package/dist/cjs/mongo-client.js +18 -1
  6. package/dist/cjs/server.js +164 -0
  7. package/dist/cjs/sync/commands.js +126 -0
  8. package/dist/cjs/sync/diff.js +174 -0
  9. package/dist/cjs/sync/httpSink.js +159 -0
  10. package/dist/cjs/sync/index.js +40 -0
  11. package/dist/cjs/sync/mongoExecutor.js +306 -0
  12. package/dist/cjs/sync/mongoSink.js +161 -0
  13. package/dist/cjs/sync/outbox.js +358 -0
  14. package/dist/cjs/sync/protocol.js +177 -0
  15. package/dist/cjs/sync/replicator.js +747 -0
  16. package/dist/cjs/sync/shadow.js +178 -0
  17. package/dist/cjs/sync/types.js +2 -0
  18. package/dist/cloudflare.d.ts +1 -1
  19. package/dist/cloudflare.js.map +1 -1
  20. package/dist/collection.js +9 -6
  21. package/dist/collection.js.map +1 -1
  22. package/dist/cursors/findCursor.d.ts +2 -1
  23. package/dist/cursors/findCursor.js +6 -4
  24. package/dist/cursors/findCursor.js.map +1 -1
  25. package/dist/index.d.ts +68 -2
  26. package/dist/index.js +96 -1
  27. package/dist/index.js.map +1 -1
  28. package/dist/mongo-client.d.ts +13 -0
  29. package/dist/mongo-client.js +18 -1
  30. package/dist/mongo-client.js.map +1 -1
  31. package/dist/server.d.ts +97 -0
  32. package/dist/server.js +158 -0
  33. package/dist/server.js.map +1 -0
  34. package/dist/sync/commands.d.ts +49 -0
  35. package/dist/sync/commands.js +118 -0
  36. package/dist/sync/commands.js.map +1 -0
  37. package/dist/sync/diff.d.ts +55 -0
  38. package/dist/sync/diff.js +169 -0
  39. package/dist/sync/diff.js.map +1 -0
  40. package/dist/sync/httpSink.d.ts +93 -0
  41. package/dist/sync/httpSink.js +155 -0
  42. package/dist/sync/httpSink.js.map +1 -0
  43. package/dist/sync/index.d.ts +17 -0
  44. package/dist/sync/index.js +10 -0
  45. package/dist/sync/index.js.map +1 -0
  46. package/dist/sync/mongoExecutor.d.ts +37 -0
  47. package/dist/sync/mongoExecutor.js +302 -0
  48. package/dist/sync/mongoExecutor.js.map +1 -0
  49. package/dist/sync/mongoSink.d.ts +109 -0
  50. package/dist/sync/mongoSink.js +123 -0
  51. package/dist/sync/mongoSink.js.map +1 -0
  52. package/dist/sync/outbox.d.ts +120 -0
  53. package/dist/sync/outbox.js +355 -0
  54. package/dist/sync/outbox.js.map +1 -0
  55. package/dist/sync/protocol.d.ts +93 -0
  56. package/dist/sync/protocol.js +168 -0
  57. package/dist/sync/protocol.js.map +1 -0
  58. package/dist/sync/replicator.d.ts +177 -0
  59. package/dist/sync/replicator.js +744 -0
  60. package/dist/sync/replicator.js.map +1 -0
  61. package/dist/sync/shadow.d.ts +54 -0
  62. package/dist/sync/shadow.js +173 -0
  63. package/dist/sync/shadow.js.map +1 -0
  64. package/dist/sync/types.d.ts +388 -0
  65. package/dist/sync/types.js +2 -0
  66. package/dist/sync/types.js.map +1 -0
  67. package/package.json +16 -2
@@ -0,0 +1,744 @@
1
+ /**
2
+ * Drains the {@link SyncOutbox} into an upstream {@link SyncSink}.
3
+ *
4
+ * ## Delivery guarantees
5
+ *
6
+ * Delivery is **at-least-once**, and every operation the replicator emits is
7
+ * idempotent (`replaceOne` with `upsert`, or `deleteOne` by `_id`). Replaying a
8
+ * batch therefore converges on the same upstream state, which is what makes it
9
+ * safe to acknowledge *after* the upstream write rather than before: a crash in
10
+ * the window between the two costs a duplicate write, never a lost one.
11
+ *
12
+ * ## Ordering
13
+ *
14
+ * Changes are read in outbox order and coalesced per document, so the newest
15
+ * revision wins. Within a batch, operations touch distinct documents and are
16
+ * applied unordered for throughput; across batches, order is preserved.
17
+ */
18
+ import { EventEmitter } from 'events';
19
+ import { applyDiff, computeDiff, isEmptyDiff } from './diff.js';
20
+ import { SyncOutbox } from './outbox.js';
21
+ import { SyncShadow, projectForComparison } from './shadow.js';
22
+ const DEFAULTS = {
23
+ name: 'default',
24
+ batchSize: 500,
25
+ pollIntervalMs: 250,
26
+ retryDelayMs: 500,
27
+ maxRetryDelayMs: 30000,
28
+ maxRetries: Number.POSITIVE_INFINITY,
29
+ idMapping: 'auto',
30
+ initial: 'backfill',
31
+ maxOutboxSize: 100000,
32
+ overflowStrategy: 'compact',
33
+ collectionScanIntervalMs: 5000,
34
+ unref: false,
35
+ versioning: true,
36
+ verbose: false,
37
+ };
38
+ /** Rows scanned per query during the initial backfill. */
39
+ const BACKFILL_CHUNK_SIZE = 500;
40
+ export class SyncReplicator extends EventEmitter {
41
+ constructor(db, sink, options = {}) {
42
+ super();
43
+ this.db = db;
44
+ this.sink = sink;
45
+ this.running = false;
46
+ this.connected = false;
47
+ this.loop = null;
48
+ this.checkpoint = 0;
49
+ this.backfilled = [];
50
+ this.watched = [];
51
+ this.lastCollectionScan = 0;
52
+ this.appliedCount = 0;
53
+ this.retryCount = 0;
54
+ this.deadLetterCount = 0;
55
+ this.conflictCount = 0;
56
+ /** Resolves the current sleep early when a local write arrives or `stop()` is called. */
57
+ this.wake = null;
58
+ this.opts = { ...DEFAULTS, ...stripUndefined(options) };
59
+ this.outbox = new SyncOutbox(db);
60
+ this.shadow = new SyncShadow(db, this.opts.name);
61
+ }
62
+ /** The outbox backing this replicator — for dead-letter inspection and manual maintenance. */
63
+ get log() {
64
+ return this.outbox;
65
+ }
66
+ /** The shadow store holding last-known upstream state — for diagnostics and reseeding. */
67
+ get baseline() {
68
+ return this.shadow;
69
+ }
70
+ /**
71
+ * Installs capture triggers, backfills existing documents if this replicator
72
+ * has never run, and starts streaming changes upstream.
73
+ *
74
+ * Resolves once replication is *running*, not once it has caught up — use
75
+ * {@link SyncReplicator.waitForDrain} for that.
76
+ */
77
+ async start() {
78
+ if (this.running)
79
+ return;
80
+ this.running = true;
81
+ await this.outbox.ensureSchema();
82
+ if (this.opts.versioning)
83
+ await this.shadow.ensureSchema();
84
+ const state = await this.outbox.loadState(this.opts.name);
85
+ this.checkpoint = state.checkpoint;
86
+ this.backfilled = state.backfilled;
87
+ await this.refreshCollections();
88
+ this.loop = this.run().catch((err) => {
89
+ this.fail(err);
90
+ });
91
+ }
92
+ /**
93
+ * Stops replication. Capture triggers stay installed, so changes made while
94
+ * stopped are still recorded and picked up on the next `start()`.
95
+ *
96
+ * @param options.flush Drain everything pending before stopping. Defaults to `false`.
97
+ */
98
+ async stop(options = {}) {
99
+ if (!this.running)
100
+ return;
101
+ if (options.flush) {
102
+ await this.waitForDrain();
103
+ }
104
+ this.running = false;
105
+ this.wake?.();
106
+ if (this.loop) {
107
+ await this.loop.catch(() => undefined);
108
+ this.loop = null;
109
+ }
110
+ if (this.sink.close) {
111
+ try {
112
+ await this.sink.close();
113
+ }
114
+ catch (err) {
115
+ this.log_(`error closing sink: ${describeError(err)}`);
116
+ }
117
+ }
118
+ this.connected = false;
119
+ // Unblock anyone parked in waitForDrain() rather than leaving them hanging.
120
+ this.emit('drained', { checkpoint: this.checkpoint });
121
+ this.emit('stopped');
122
+ }
123
+ /**
124
+ * Nudges the replicator to check the outbox now instead of waiting for the
125
+ * next poll. Called by the client after local writes so replication latency
126
+ * is bounded by the upstream round trip, not the poll interval.
127
+ */
128
+ notify() {
129
+ this.wake?.();
130
+ }
131
+ /**
132
+ * Resolves once every change written *before this call* has reached the
133
+ * upstream. Rejects if replication stops with an error.
134
+ *
135
+ * The target is pinned to the outbox position at call time rather than
136
+ * "outbox is empty" — otherwise a concurrent writer could keep the outbox
137
+ * non-empty forever, and, worse, a `drained` event from a scan that began
138
+ * before the caller's own write would resolve it too early.
139
+ */
140
+ async waitForDrain() {
141
+ if (!this.running)
142
+ return;
143
+ const target = await this.outbox.currentSequence();
144
+ if (this.checkpoint >= target)
145
+ return;
146
+ return new Promise((resolve, reject) => {
147
+ const cleanup = () => {
148
+ this.removeListener('drained', check);
149
+ this.removeListener('batch', check);
150
+ this.removeListener('stopped', check);
151
+ this.removeListener('error', onError);
152
+ };
153
+ const check = () => {
154
+ if (!this.running || this.checkpoint >= target) {
155
+ cleanup();
156
+ resolve();
157
+ }
158
+ };
159
+ const onError = (err) => {
160
+ cleanup();
161
+ reject(err);
162
+ };
163
+ this.on('drained', check);
164
+ this.on('batch', check);
165
+ this.on('stopped', check);
166
+ this.on('error', onError);
167
+ this.wake?.();
168
+ });
169
+ }
170
+ /** Current replication progress. */
171
+ async status() {
172
+ return {
173
+ name: this.opts.name,
174
+ running: this.running,
175
+ connected: this.connected,
176
+ checkpoint: this.checkpoint,
177
+ pending: await this.pending(),
178
+ applied: this.appliedCount,
179
+ retries: this.retryCount,
180
+ deadLettered: this.deadLetterCount,
181
+ conflicts: this.conflictCount,
182
+ collections: [...this.watched],
183
+ lastError: this.lastError,
184
+ };
185
+ }
186
+ /** Changes permanently rejected upstream. */
187
+ async deadLetters(limit = 100) {
188
+ return this.outbox.listDeadLetters(this.opts.name, limit);
189
+ }
190
+ // ---------------------------------------------------------------- internals
191
+ async pending() {
192
+ return this.outbox.pendingCount(this.checkpoint, this.collectionFilter());
193
+ }
194
+ /** `null` means "every collection"; an array restricts the outbox query. */
195
+ collectionFilter() {
196
+ return this.opts.collections || this.opts.exclude ? this.watched : null;
197
+ }
198
+ /**
199
+ * Discovers collections to watch and installs their triggers.
200
+ *
201
+ * Re-run periodically so collections created after `start()` join replication
202
+ * without a restart.
203
+ */
204
+ async refreshCollections() {
205
+ const existing = await this.outbox.listCollections();
206
+ const exclude = new Set(this.opts.exclude ?? []);
207
+ const selected = (this.opts.collections ?? existing).filter((name) => !exclude.has(name) && existing.includes(name));
208
+ // A table dropped and recreated between scans has lost its triggers.
209
+ await this.outbox.refreshTriggerCache();
210
+ for (const collection of selected) {
211
+ await this.outbox.ensureTriggers(collection);
212
+ }
213
+ this.watched = selected;
214
+ this.lastCollectionScan = Date.now();
215
+ if (this.opts.initial === 'backfill') {
216
+ const pendingBackfill = selected.filter((name) => !this.backfilled.includes(name));
217
+ if (pendingBackfill.length > 0) {
218
+ await this.backfill(pendingBackfill);
219
+ }
220
+ }
221
+ else if (selected.some((name) => !this.backfilled.includes(name))) {
222
+ // Record them as done so switching to 'backfill' later doesn't replay history.
223
+ this.backfilled = [...new Set([...this.backfilled, ...selected])];
224
+ await this.outbox.markBackfilled(this.opts.name, this.backfilled);
225
+ }
226
+ }
227
+ /**
228
+ * Seeds the upstream with documents that existed before replication started.
229
+ *
230
+ * The checkpoint is pinned *before* the scan begins, so a write racing the
231
+ * scan is either seen by the scan, or captured in the outbox above the
232
+ * checkpoint, or both — and "both" is harmless because upserts are idempotent.
233
+ */
234
+ async backfill(collections) {
235
+ await this.ensureConnected();
236
+ for (const collection of collections) {
237
+ let count = 0;
238
+ for await (const chunk of this.outbox.scanCollection(collection, BACKFILL_CHUNK_SIZE)) {
239
+ if (!this.running)
240
+ return;
241
+ const records = chunk
242
+ // Unreadable JSON — skip, the row is already broken locally.
243
+ .filter((row) => row.document !== null)
244
+ .map((row) => ({
245
+ id: 0,
246
+ collection,
247
+ documentId: row.documentId,
248
+ operation: 'insert',
249
+ document: row.document,
250
+ capturedAt: Date.now(),
251
+ }));
252
+ const operations = await this.buildOperations(records);
253
+ if (operations.length > 0) {
254
+ const outcome = await this.applyWithRetry(operations);
255
+ if (outcome === null)
256
+ return;
257
+ for (const failure of outcome.failures) {
258
+ const op = operations[failure.index];
259
+ if (!op)
260
+ continue;
261
+ await this.recordDeadLetter({
262
+ id: 0,
263
+ collection: op.sourceCollection,
264
+ documentId: op.documentId,
265
+ operation: 'insert',
266
+ document: op.document ?? null,
267
+ capturedAt: Date.now(),
268
+ }, failure.message);
269
+ }
270
+ // A document that already exists upstream is not ours to overwrite — the
271
+ // backfill adopts the server's version instead.
272
+ const conflicted = new Set(outcome.conflicts.map((conflict) => conflict.index));
273
+ for (const conflict of outcome.conflicts) {
274
+ await this.resolveConflict(operations[conflict.index], conflict);
275
+ }
276
+ await this.commitShadows(operations, outcome.failures, conflicted);
277
+ const applied = operations.length - outcome.failures.length - conflicted.size;
278
+ this.appliedCount += applied;
279
+ count += applied;
280
+ }
281
+ }
282
+ this.backfilled = [...new Set([...this.backfilled, collection])];
283
+ await this.outbox.markBackfilled(this.opts.name, this.backfilled);
284
+ this.log_(`backfilled ${count} document(s) from "${collection}"`);
285
+ }
286
+ }
287
+ /** The replication loop: read, coalesce, apply, checkpoint, repeat. */
288
+ async run() {
289
+ await this.ensureConnected();
290
+ while (this.running) {
291
+ let moved = false;
292
+ try {
293
+ moved = await this.drainOnce();
294
+ }
295
+ catch (err) {
296
+ if (!this.running)
297
+ return;
298
+ throw err;
299
+ }
300
+ if (!this.running)
301
+ return;
302
+ if (!moved) {
303
+ this.emit('drained', { checkpoint: this.checkpoint });
304
+ await this.sleep(this.opts.pollIntervalMs);
305
+ if (this.running &&
306
+ Date.now() - this.lastCollectionScan >= this.opts.collectionScanIntervalMs) {
307
+ await this.refreshCollections();
308
+ }
309
+ }
310
+ }
311
+ }
312
+ /**
313
+ * Applies one batch.
314
+ * @returns `true` if there was work to do, `false` if the outbox was empty.
315
+ */
316
+ async drainOnce() {
317
+ // Read the tail position first: if the batch comes back empty we can safely
318
+ // fast-forward to here without stepping over a row that arrived meanwhile.
319
+ const tail = await this.outbox.currentSequence();
320
+ const records = await this.outbox.readBatch(this.checkpoint, this.opts.batchSize, this.collectionFilter());
321
+ if (records.length === 0) {
322
+ if (tail > this.checkpoint) {
323
+ // Only rows for collections we don't replicate. Skip past them so they
324
+ // become prunable instead of accumulating forever.
325
+ await this.advanceTo(tail);
326
+ }
327
+ await this.enforceOutboxLimit();
328
+ return false;
329
+ }
330
+ const highestId = records[records.length - 1].id;
331
+ const { latest, poison } = this.coalesce(records);
332
+ for (const record of poison) {
333
+ await this.recordDeadLetter(record, 'stored document JSON could not be parsed');
334
+ }
335
+ const operations = await this.buildOperations(latest);
336
+ if (operations.length > 0) {
337
+ const outcome = await this.applyWithRetry(operations);
338
+ if (outcome === null)
339
+ return false; // Stopped mid-batch — leave the checkpoint alone.
340
+ for (const failure of outcome.failures) {
341
+ const op = operations[failure.index];
342
+ if (!op)
343
+ continue;
344
+ await this.recordDeadLetter({
345
+ id: op.outboxId,
346
+ collection: op.sourceCollection,
347
+ documentId: op.documentId,
348
+ operation: op.type === 'delete' ? 'delete' : 'update',
349
+ document: op.document ?? null,
350
+ capturedAt: Date.now(),
351
+ }, failure.message);
352
+ }
353
+ const conflicted = new Set(outcome.conflicts.map((conflict) => conflict.index));
354
+ for (const conflict of outcome.conflicts) {
355
+ await this.resolveConflict(operations[conflict.index], conflict);
356
+ }
357
+ // Roll the shadow forward for everything that actually landed, so the next
358
+ // local edit diffs against the state we just wrote rather than a stale one.
359
+ await this.commitShadows(operations, outcome.failures, conflicted);
360
+ const applied = operations.length - outcome.failures.length - conflicted.size;
361
+ this.appliedCount += applied;
362
+ // Checkpoint before announcing, so a `batch` listener that inspects
363
+ // progress sees the position the batch actually reached.
364
+ await this.advanceTo(highestId);
365
+ this.emit('batch', {
366
+ applied,
367
+ deadLettered: outcome.failures.length,
368
+ checkpoint: highestId,
369
+ });
370
+ return true;
371
+ }
372
+ await this.advanceTo(highestId);
373
+ return true;
374
+ }
375
+ /** Acknowledges up to `id` and prunes rows no replicator needs any more. */
376
+ async advanceTo(id) {
377
+ this.checkpoint = id;
378
+ await this.outbox.saveCheckpoint(this.opts.name, id);
379
+ await this.outbox.prune();
380
+ }
381
+ /**
382
+ * Collapses a run of changes into one operation per document.
383
+ *
384
+ * Records are in ascending outbox order, so the last one seen for a document
385
+ * is its current state — an upsert of the newest revision, or a delete.
386
+ */
387
+ coalesce(records) {
388
+ const latest = new Map();
389
+ const poison = [];
390
+ for (const record of records) {
391
+ latest.set(`${record.collection}${record.documentId}`, record);
392
+ }
393
+ const kept = [];
394
+ for (const record of latest.values()) {
395
+ if (record.operation !== 'delete' && record.document === null) {
396
+ poison.push(record);
397
+ continue;
398
+ }
399
+ kept.push(record);
400
+ }
401
+ return { latest: kept, poison };
402
+ }
403
+ /**
404
+ * Turns coalesced records into upstream operations, attaching the base version and
405
+ * minimal diff from the shadow.
406
+ *
407
+ * Shadows load per collection in one query rather than per document, so a batch of
408
+ * 500 changes costs one extra read per collection, not 500.
409
+ */
410
+ async buildOperations(records) {
411
+ const operations = [];
412
+ if (records.length === 0)
413
+ return operations;
414
+ if (!this.opts.versioning) {
415
+ for (const record of records) {
416
+ const op = this.toOperation(record);
417
+ if (op)
418
+ operations.push(op);
419
+ }
420
+ return operations;
421
+ }
422
+ const byCollection = new Map();
423
+ for (const record of records) {
424
+ const bucket = byCollection.get(record.collection);
425
+ if (bucket)
426
+ bucket.push(record);
427
+ else
428
+ byCollection.set(record.collection, [record]);
429
+ }
430
+ for (const [collection, bucket] of byCollection) {
431
+ const shadows = await this.shadow.load(collection, bucket.map((record) => record.documentId));
432
+ for (const record of bucket) {
433
+ const op = this.toOperation(record);
434
+ if (!op)
435
+ continue;
436
+ const entry = shadows.get(record.documentId);
437
+ // No shadow means the document has never been upstream, so the write must
438
+ // create it rather than claim to replace a revision it never read.
439
+ op.baseVersion = entry ? entry.baseVersion : null;
440
+ if (op.type === 'upsert' && entry && op.document) {
441
+ const diff = computeDiff(entry.projection, op.document);
442
+ // Nothing actually changed relative to upstream — skip rather than burn a
443
+ // version bump on a no-op write.
444
+ if (isEmptyDiff(diff))
445
+ continue;
446
+ op.diff = diff;
447
+ }
448
+ operations.push(op);
449
+ }
450
+ }
451
+ return operations;
452
+ }
453
+ /**
454
+ * Records the new upstream state for operations that landed.
455
+ *
456
+ * The post-write state is computed by applying the diff to the previous *server*
457
+ * document rather than by re-reading it — one fewer round trip, and it keeps the BSON
458
+ * values of untouched fields exactly as they were.
459
+ */
460
+ async commitShadows(operations, failures, conflicted) {
461
+ if (!this.opts.versioning)
462
+ return;
463
+ const failed = new Set(failures.map((failure) => failure.index));
464
+ for (const [index, op] of operations.entries()) {
465
+ if (failed.has(index) || conflicted.has(index))
466
+ continue;
467
+ if (op.type === 'delete') {
468
+ await this.shadow.remove(op.sourceCollection, op.documentId);
469
+ continue;
470
+ }
471
+ const document = op.document ?? {};
472
+ if (op.baseVersion === null || op.baseVersion === undefined) {
473
+ // Freshly created upstream: the server holds what we sent, at version 1.
474
+ await this.shadow.put(op.sourceCollection, op.documentId, 1, document);
475
+ continue;
476
+ }
477
+ const previous = await this.shadow.load(op.sourceCollection, [op.documentId]);
478
+ const entry = previous.get(op.documentId);
479
+ const next = entry && op.diff ? applyDiff(entry.serverDocument, op.diff) : { ...document };
480
+ await this.shadow.put(op.sourceCollection, op.documentId, op.baseVersion + 1, next);
481
+ }
482
+ }
483
+ /**
484
+ * Reconciles a push that lost a race.
485
+ *
486
+ * The upstream state goes into the shadow first, so whatever happens next — a retry,
487
+ * or the application's own decision — is made against current reality rather than
488
+ * against the version that already lost.
489
+ */
490
+ async resolveConflict(op, conflict) {
491
+ if (!op)
492
+ return;
493
+ this.conflictCount += 1;
494
+ const serverDocument = conflict.serverDocument ?? (await this.fetchUpstream(op.collection, op.documentId));
495
+ const serverVersion = conflict.serverVersion ?? readVersionField(serverDocument);
496
+ const context = {
497
+ collection: op.collection,
498
+ sourceCollection: op.sourceCollection,
499
+ documentId: op.documentId,
500
+ operation: op.type,
501
+ reason: conflict.reason,
502
+ localDocument: op.document ?? null,
503
+ serverDocument: serverDocument ?? null,
504
+ baseVersion: op.baseVersion ?? null,
505
+ serverVersion,
506
+ };
507
+ const resolution = this.opts.onConflict
508
+ ? this.opts.onConflict(context)
509
+ : 'server';
510
+ if (serverDocument) {
511
+ await this.shadow.put(op.sourceCollection, op.documentId, serverVersion, serverDocument);
512
+ }
513
+ else {
514
+ // Gone upstream — drop the shadow so a later local write recreates it.
515
+ await this.shadow.remove(op.sourceCollection, op.documentId);
516
+ }
517
+ if (resolution === 'server') {
518
+ // Adopting the upstream version locally is what makes "server wins" durable.
519
+ // Without it the local row keeps the value that just lost, and the very next
520
+ // local edit pushes it straight back up — the conflict would only ever be
521
+ // deferred, never resolved.
522
+ await this.adoptUpstream(op.sourceCollection, op.documentId, serverDocument ?? null);
523
+ }
524
+ if (resolution === 'local') {
525
+ // Re-queue so the next drain diffs against the refreshed shadow and wins the
526
+ // second time. Deliberately not retried inline: that could livelock against a
527
+ // writer actively holding the document.
528
+ await this.outbox.requeue(op.sourceCollection, op.documentId, op.document ?? null);
529
+ }
530
+ this.log_(`conflict on ${op.sourceCollection}/${op.documentId} (${conflict.reason}) → ${resolution}`);
531
+ this.emit('conflict', { ...context, resolution });
532
+ }
533
+ /**
534
+ * Writes the upstream version of a document into the local collection.
535
+ *
536
+ * The write trips the capture triggers and lands back in the outbox, which would
537
+ * normally mean pushing it straight back up. It does not, because the local row now
538
+ * matches the shadow exactly: the diff comes out empty and the echo is dropped in
539
+ * {@link SyncReplicator.buildOperations}. That is why the local copy is written from
540
+ * the same projection the diff compares against.
541
+ */
542
+ async adoptUpstream(collection, documentId, serverDocument) {
543
+ const table = `"${collection.replace(/"/g, '""')}"`;
544
+ if (!serverDocument) {
545
+ await this.db.run(`DELETE FROM ${table} WHERE _id = ?`, [documentId]);
546
+ return;
547
+ }
548
+ // `_id` lives in its own column and is re-attached on read.
549
+ const rest = projectForComparison(serverDocument);
550
+ delete rest._id;
551
+ await this.db.run(`INSERT INTO ${table} (_id, data) VALUES (?, ?)
552
+ ON CONFLICT(_id) DO UPDATE SET data = excluded.data`, [documentId, JSON.stringify(rest)]);
553
+ }
554
+ async fetchUpstream(collection, documentId) {
555
+ if (!this.sink.fetch)
556
+ return undefined;
557
+ try {
558
+ const documents = await this.sink.fetch(collection, [documentId]);
559
+ return documents[0];
560
+ }
561
+ catch (err) {
562
+ this.log_(`could not re-read ${collection}/${documentId}: ${describeError(err)}`);
563
+ return undefined;
564
+ }
565
+ }
566
+ /** Maps an outbox record to an upstream operation, applying rename and transform. */
567
+ toOperation(record) {
568
+ const target = this.opts.collectionMap?.[record.collection] ?? record.collection;
569
+ if (record.operation === 'delete') {
570
+ return {
571
+ collection: target,
572
+ sourceCollection: record.collection,
573
+ documentId: record.documentId,
574
+ type: 'delete',
575
+ outboxId: record.id,
576
+ };
577
+ }
578
+ let document = record.document;
579
+ if (this.opts.transform) {
580
+ const transformed = this.opts.transform(document, {
581
+ collection: target,
582
+ sourceCollection: record.collection,
583
+ operation: record.operation,
584
+ });
585
+ // `null` means "don't replicate this one" — acknowledged, not retried.
586
+ if (transformed === null)
587
+ return null;
588
+ document = transformed;
589
+ }
590
+ return {
591
+ collection: target,
592
+ sourceCollection: record.collection,
593
+ documentId: record.documentId,
594
+ type: 'upsert',
595
+ document,
596
+ outboxId: record.id,
597
+ };
598
+ }
599
+ /**
600
+ * Applies a batch, retrying transient failures with exponential backoff and
601
+ * jitter. Returns the operations the upstream rejected permanently, or `null`
602
+ * if the replicator was stopped before the batch landed.
603
+ *
604
+ * The checkpoint is never advanced while this is retrying, so an upstream
605
+ * outage simply parks the batch — the outbox absorbs the backlog.
606
+ */
607
+ async applyWithRetry(operations) {
608
+ let attempt = 0;
609
+ let delay = this.opts.retryDelayMs;
610
+ for (;;) {
611
+ // Stopping mid-retry must not look like success: returning `null` keeps
612
+ // the caller from checkpointing work that never reached the upstream.
613
+ if (!this.running)
614
+ return null;
615
+ try {
616
+ await this.ensureConnected();
617
+ const result = await this.sink.apply(operations);
618
+ this.lastError = undefined;
619
+ return { failures: result.failures ?? [], conflicts: result.conflicts ?? [] };
620
+ }
621
+ catch (err) {
622
+ attempt += 1;
623
+ this.retryCount += 1;
624
+ this.connected = false;
625
+ this.lastError = describeError(err);
626
+ if (attempt > this.opts.maxRetries) {
627
+ throw err instanceof Error ? err : new Error(this.lastError);
628
+ }
629
+ // Jitter keeps several replicators from stampeding a recovering upstream.
630
+ const jitter = 0.85 + Math.random() * 0.3;
631
+ const retryInMs = Math.min(delay * jitter, this.opts.maxRetryDelayMs);
632
+ delay = Math.min(delay * 2, this.opts.maxRetryDelayMs);
633
+ this.emit('retry', {
634
+ attempt,
635
+ retryInMs,
636
+ error: err instanceof Error ? err : new Error(this.lastError),
637
+ });
638
+ this.log_(`batch failed (attempt ${attempt}), retrying in ${Math.round(retryInMs)}ms`);
639
+ // An outage is exactly when the backlog builds, and this loop is where
640
+ // the replicator spends that time — so the size check belongs here too,
641
+ // not only on the idle path.
642
+ await this.enforceOutboxLimit();
643
+ await this.sleep(retryInMs);
644
+ }
645
+ }
646
+ }
647
+ async ensureConnected() {
648
+ if (this.connected)
649
+ return;
650
+ if (this.sink.connect) {
651
+ await this.sink.connect();
652
+ }
653
+ this.connected = true;
654
+ this.emit('connected');
655
+ }
656
+ async recordDeadLetter(record, error) {
657
+ this.deadLetterCount += 1;
658
+ await this.outbox.deadLetter(this.opts.name, {
659
+ collection: record.collection,
660
+ documentId: record.documentId,
661
+ operation: record.operation,
662
+ document: record.document,
663
+ }, error);
664
+ this.emit('deadLetter', {
665
+ id: record.id,
666
+ collection: record.collection,
667
+ documentId: record.documentId,
668
+ operation: record.operation,
669
+ document: record.document,
670
+ error,
671
+ failedAt: Date.now(),
672
+ });
673
+ }
674
+ /**
675
+ * Keeps the outbox from growing without bound during a long outage.
676
+ * Compaction is lossless here because replication is full-document
677
+ * last-write-wins: only superseded revisions are dropped.
678
+ */
679
+ async enforceOutboxLimit() {
680
+ const pending = await this.pending();
681
+ if (pending <= this.opts.maxOutboxSize)
682
+ return;
683
+ const compacted = this.opts.overflowStrategy === 'compact' ? await this.outbox.compact() : 0;
684
+ this.emit('overflow', { pending, limit: this.opts.maxOutboxSize, compacted });
685
+ this.log_(`outbox at ${pending} rows (limit ${this.opts.maxOutboxSize}); compacted ${compacted}`);
686
+ }
687
+ /** Sleeps, but returns early when `notify()` or `stop()` is called. */
688
+ sleep(ms) {
689
+ return new Promise((resolve) => {
690
+ let settled = false;
691
+ const finish = () => {
692
+ if (settled)
693
+ return;
694
+ settled = true;
695
+ clearTimeout(timer);
696
+ this.wake = null;
697
+ resolve();
698
+ };
699
+ const timer = setTimeout(finish, ms);
700
+ // A running replicator holds the process open by default, the same way an
701
+ // open server does: exiting with changes still queued would silently lose
702
+ // them. `unref: true` opts out for short-lived scripts.
703
+ if (this.opts.unref)
704
+ timer.unref?.();
705
+ this.wake = finish;
706
+ });
707
+ }
708
+ fail(err) {
709
+ this.running = false;
710
+ this.connected = false;
711
+ this.lastError = describeError(err);
712
+ const error = err instanceof Error ? err : new Error(this.lastError);
713
+ // Never throw from the loop's catch: an unhandled 'error' event would crash
714
+ // the host process for what is a recoverable, reportable condition.
715
+ if (this.listenerCount('error') > 0) {
716
+ this.emit('error', error);
717
+ }
718
+ else {
719
+ console.error(`[mongolite:sync:${this.opts.name}] replication stopped:`, error);
720
+ }
721
+ }
722
+ log_(message) {
723
+ if (this.opts.verbose) {
724
+ console.log(`[mongolite:sync:${this.opts.name}] ${message}`);
725
+ }
726
+ }
727
+ }
728
+ /** Reads `_v` off an upstream document, tolerating its absence. */
729
+ function readVersionField(document) {
730
+ if (!document)
731
+ return null;
732
+ const raw = document._v;
733
+ return typeof raw === 'number' ? raw : null;
734
+ }
735
+ function describeError(err) {
736
+ if (err instanceof Error)
737
+ return err.message;
738
+ return String(err);
739
+ }
740
+ /** Drops explicitly-undefined keys so they don't clobber defaults in a spread. */
741
+ function stripUndefined(value) {
742
+ return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
743
+ }
744
+ //# sourceMappingURL=replicator.js.map