loro-repo 0.18.4 → 0.18.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -134,14 +134,35 @@ The schema (created on first use) is:
134
134
  ```
135
135
  docs (doc_id PK, snapshot, updated_at, snapshot_digest)
136
136
  doc_updates (id, doc_id, update_data, created_at)
137
+ flock_docs (flock_doc_id PK, snapshot, updated_at, snapshot_digest)
138
+ flock_doc_updates (id, flock_doc_id, update_data, created_at)
137
139
  meta_snapshot (id=1, snapshot, updated_at)
138
140
  meta_updates (id, update_data, created_at)
139
141
  remote_cursors (stream_url PK, next_offset, server_lower_bound_version, updated_at_ms)
140
142
  ```
141
143
 
142
- `loadDoc` / `loadMeta` opportunistically compact replayed updates into a fresh
143
- snapshot inside a single transaction, mirroring the filesystem adapter's
144
- behaviour but without any file churn.
144
+ Post-sync Flock document persistence is version-vector based: no-op syncs do
145
+ not write, and changed state is appended as a JSON delta rather than a full
146
+ Flock file. Local changes that arrive while sync persistence is in flight are
147
+ queued behind that write and flushed from the last durable version. This keeps
148
+ storage growth proportional to new CRDT operations while preserving restart
149
+ convergence. Once a cached Flock document has accumulated 102400 persisted
150
+ delta bytes, the repo asks adapters that support compaction to consolidate
151
+ those updates. Configure this bound with
152
+ `flockDocCompactionByteThreshold` or set it to `0` to disable compaction. The
153
+ threshold-crossing delta save queues a coalesced per-document compaction rather
154
+ than awaiting the full snapshot export, so that sync can return once its delta
155
+ is durable. The adapter compaction starts on a later event-loop turn while
156
+ remaining serialized with later document persistence, and `flush()` /
157
+ `destroy()` drain the queued work.
158
+
159
+ `loadDoc` / `loadFlockDoc` / `loadMeta` opportunistically compact replayed
160
+ updates into a fresh snapshot inside a single transaction, mirroring the
161
+ filesystem adapter's behaviour but without any file churn. Existing databases
162
+ need no migration for delta-aware post-sync persistence; previously amplified
163
+ rows are compacted the next time that Flock document is loaded. Compaction is
164
+ driven by actual delta volume, not by time or sync count, so no-op syncs never
165
+ cause full snapshot exports.
145
166
 
146
167
  ### Backups in WAL mode
147
168
 
package/dist/index.cjs CHANGED
@@ -365,6 +365,7 @@ function yieldToMacroTask() {
365
365
  return new Promise((resolve) => setTimeout(resolve, 0));
366
366
  }
367
367
  const YIELD_BUDGET_MS = 5;
368
+ const DEFAULT_COMPACTION_BYTE_THRESHOLD = 100 * 1024;
368
369
  const nowMs = typeof globalThis.performance?.now === "function" ? globalThis.performance.now.bind(globalThis.performance) : Date.now;
369
370
  function stableVersion(vector) {
370
371
  const sorted = {};
@@ -389,6 +390,7 @@ var FlockDocManager = class {
389
390
  ensureFlockDocExists;
390
391
  retentionMs;
391
392
  persistDebounceMs;
393
+ compactionByteThreshold;
392
394
  flocks = /* @__PURE__ */ new Map();
393
395
  subscriptions = /* @__PURE__ */ new Map();
394
396
  persistedVersions = /* @__PURE__ */ new Map();
@@ -396,6 +398,8 @@ var FlockDocManager = class {
396
398
  dirtyFlockDocs = /* @__PURE__ */ new Set();
397
399
  existenceMaterialized = /* @__PURE__ */ new Set();
398
400
  docsWithVisibleContent = /* @__PURE__ */ new Set();
401
+ updateBytesSinceCompaction = /* @__PURE__ */ new Map();
402
+ queuedCompactions = /* @__PURE__ */ new Set();
399
403
  debounceTimer;
400
404
  flushScheduled = false;
401
405
  operationPromise = Promise.resolve();
@@ -406,6 +410,8 @@ var FlockDocManager = class {
406
410
  this.retentionMs = options.retentionMs;
407
411
  const configured = options.persistDebounceMs;
408
412
  this.persistDebounceMs = typeof configured === "number" && Number.isFinite(configured) && configured >= 0 ? configured : 0;
413
+ const configuredCompactionThreshold = options.compactionByteThreshold;
414
+ this.compactionByteThreshold = typeof configuredCompactionThreshold === "number" && Number.isFinite(configuredCompactionThreshold) && configuredCompactionThreshold >= 0 ? Math.floor(configuredCompactionThreshold) : DEFAULT_COMPACTION_BYTE_THRESHOLD;
409
415
  }
410
416
  async openFlockDoc(flockDocId) {
411
417
  return this.ensureFlockDoc(flockDocId);
@@ -459,16 +465,26 @@ var FlockDocManager = class {
459
465
  throw error;
460
466
  }
461
467
  }
468
+ async persistFlockDocChanges(flockDocId, flock) {
469
+ await this.enqueueOperation(async () => {
470
+ if (this.flocks.get(flockDocId) !== flock) return;
471
+ await this.persistFlockDocUpdate(flockDocId, flock);
472
+ });
473
+ }
462
474
  async unloadFlockDoc(flockDocId) {
463
- const flock = this.flocks.get(flockDocId);
464
- if (!flock) return;
465
- await this.persistFlockDocUpdate(flockDocId, flock);
466
- this.subscriptions.get(flockDocId)?.();
467
- this.subscriptions.delete(flockDocId);
468
- this.flocks.delete(flockDocId);
469
- this.persistedVersions.delete(flockDocId);
470
- this.existenceMaterialized.delete(flockDocId);
471
- this.docsWithVisibleContent.delete(flockDocId);
475
+ await this.enqueueOperation(async () => {
476
+ const flock = this.flocks.get(flockDocId);
477
+ if (!flock) return;
478
+ await this.persistFlockDocUpdate(flockDocId, flock);
479
+ this.subscriptions.get(flockDocId)?.();
480
+ this.subscriptions.delete(flockDocId);
481
+ this.flocks.delete(flockDocId);
482
+ this.persistedVersions.delete(flockDocId);
483
+ this.dirtyFlockDocs.delete(flockDocId);
484
+ this.existenceMaterialized.delete(flockDocId);
485
+ this.docsWithVisibleContent.delete(flockDocId);
486
+ this.updateBytesSinceCompaction.delete(flockDocId);
487
+ });
472
488
  }
473
489
  async dropFlockDoc(flockDocId) {
474
490
  await this.enqueueOperation(async () => {
@@ -479,6 +495,7 @@ var FlockDocManager = class {
479
495
  this.dirtyFlockDocs.delete(flockDocId);
480
496
  this.existenceMaterialized.delete(flockDocId);
481
497
  this.docsWithVisibleContent.delete(flockDocId);
498
+ this.updateBytesSinceCompaction.delete(flockDocId);
482
499
  await this.storage?.deleteFlockDoc?.(flockDocId);
483
500
  });
484
501
  }
@@ -487,6 +504,7 @@ var FlockDocManager = class {
487
504
  this.flushScheduled = false;
488
505
  while (true) {
489
506
  await this.enqueueOperation(() => this.drainDirtyFlockDocs());
507
+ await this.operationPromise;
490
508
  if (this.dirtyFlockDocs.size === 0 && !this.flushScheduled) break;
491
509
  this.flushScheduled = false;
492
510
  }
@@ -507,6 +525,8 @@ var FlockDocManager = class {
507
525
  this.dirtyFlockDocs.clear();
508
526
  this.existenceMaterialized.clear();
509
527
  this.docsWithVisibleContent.clear();
528
+ this.updateBytesSinceCompaction.clear();
529
+ this.queuedCompactions.clear();
510
530
  }
511
531
  }
512
532
  async loadOrCreateFlockDoc(flockDocId) {
@@ -555,22 +575,23 @@ var FlockDocManager = class {
555
575
  }
556
576
  if (versionEquals(previousVersion, nextVersion)) {
557
577
  await this.materializeExistence(flockDocId, flock);
558
- this.dirtyFlockDocs.delete(flockDocId);
578
+ this.clearDirtyIfStable(flockDocId, nextVersion, flock);
559
579
  return;
560
580
  }
561
581
  const bundle = await exportJsonUpdate(flock, previousVersion);
562
582
  if (!hasBundleEntries(bundle)) {
563
583
  await this.materializeExistence(flockDocId, flock);
564
584
  this.persistedVersions.set(flockDocId, nextVersion);
565
- this.dirtyFlockDocs.delete(flockDocId);
585
+ this.clearDirtyIfStable(flockDocId, nextVersion, flock);
566
586
  return;
567
587
  }
588
+ const update = require_flock_snapshot.encodeMetaJsonUpdate(bundle);
568
589
  this.persistedVersions.set(flockDocId, nextVersion);
569
590
  try {
570
591
  await this.storage.save({
571
592
  type: "flock-doc-update",
572
593
  flockDocId,
573
- update: require_flock_snapshot.encodeMetaJsonUpdate(bundle)
594
+ update
574
595
  });
575
596
  await this.materializeExistence(flockDocId, flock);
576
597
  this.clearDirtyIfStable(flockDocId, nextVersion, flock);
@@ -578,6 +599,31 @@ var FlockDocManager = class {
578
599
  this.persistedVersions.set(flockDocId, previousVersion);
579
600
  throw error;
580
601
  }
602
+ this.scheduleFlockDocCompaction(flockDocId, flock, update.byteLength);
603
+ }
604
+ scheduleFlockDocCompaction(flockDocId, flock, updateBytes) {
605
+ if (this.compactionByteThreshold === 0 || !this.storage?.compactFlockDoc) return;
606
+ const total = (this.updateBytesSinceCompaction.get(flockDocId) ?? 0) + updateBytes;
607
+ this.updateBytesSinceCompaction.set(flockDocId, total);
608
+ if (total < this.compactionByteThreshold) return;
609
+ this.queueFlockDocCompaction(flockDocId, flock);
610
+ }
611
+ queueFlockDocCompaction(flockDocId, flock) {
612
+ if (this.queuedCompactions.has(flockDocId)) return;
613
+ const storage = this.storage;
614
+ if (!storage?.compactFlockDoc) return;
615
+ this.queuedCompactions.add(flockDocId);
616
+ this.enqueueOperation(async () => {
617
+ try {
618
+ await yieldToMacroTask();
619
+ if (this.flocks.get(flockDocId) !== flock) return;
620
+ await storage.compactFlockDoc?.(flockDocId);
621
+ if (this.flocks.has(flockDocId)) this.updateBytesSinceCompaction.set(flockDocId, 0);
622
+ else this.updateBytesSinceCompaction.delete(flockDocId);
623
+ } finally {
624
+ this.queuedCompactions.delete(flockDocId);
625
+ }
626
+ }).catch(require_logging.logAsyncError(`Flock document "${flockDocId}" compaction`));
581
627
  }
582
628
  async materializeExistence(flockDocId, flock) {
583
629
  if (this.existenceMaterialized.has(flockDocId)) return;
@@ -1499,7 +1545,7 @@ var SyncRunner = class {
1499
1545
  } finally {
1500
1546
  this.eventBus.popEventBy();
1501
1547
  }
1502
- await this.flockDocManager.persistFlockDoc(flockDocId, flock);
1548
+ await this.flockDocManager.persistFlockDocChanges(flockDocId, flock);
1503
1549
  }
1504
1550
  }
1505
1551
  async buildSyncPlan(options, buildOptions = {}) {
@@ -1979,7 +2025,8 @@ var LoroRepo = class LoroRepo {
1979
2025
  storage: this.storage,
1980
2026
  ensureFlockDocExists: (flockDocId, expiresAtMs) => this.flockDocRegistry.ensureExists(flockDocId, expiresAtMs),
1981
2027
  retentionMs: getFlockDocRetentionMs,
1982
- persistDebounceMs: options.docPersistDebounceMs
2028
+ persistDebounceMs: options.docPersistDebounceMs,
2029
+ compactionByteThreshold: options.flockDocCompactionByteThreshold
1983
2030
  });
1984
2031
  if (this.storage) this.metaPersister = new MetaPersister({
1985
2032
  getMetaFlock,