loro-repo 0.18.4 → 0.18.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/dist/index.cjs +61 -14
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +61 -14
- package/dist/index.js.map +1 -1
- package/dist/storage/filesystem.cjs +3 -0
- package/dist/storage/filesystem.cjs.map +1 -1
- package/dist/storage/filesystem.d.cts +1 -0
- package/dist/storage/filesystem.d.ts +1 -0
- package/dist/storage/filesystem.js +3 -0
- package/dist/storage/filesystem.js.map +1 -1
- package/dist/storage/indexeddb.cjs +3 -0
- package/dist/storage/indexeddb.cjs.map +1 -1
- package/dist/storage/indexeddb.d.cts +1 -0
- package/dist/storage/indexeddb.d.ts +1 -0
- package/dist/storage/indexeddb.js +3 -0
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/sqlite.cjs +4 -0
- package/dist/storage/sqlite.cjs.map +1 -1
- package/dist/storage/sqlite.d.cts +1 -0
- package/dist/storage/sqlite.d.ts +1 -0
- package/dist/storage/sqlite.js +4 -0
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types.d.cts +13 -0
- package/dist/types.d.ts +13 -0
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -134,14 +134,35 @@ The schema (created on first use) is:
|
|
|
134
134
|
```
|
|
135
135
|
docs (doc_id PK, snapshot, updated_at, snapshot_digest)
|
|
136
136
|
doc_updates (id, doc_id, update_data, created_at)
|
|
137
|
+
flock_docs (flock_doc_id PK, snapshot, updated_at, snapshot_digest)
|
|
138
|
+
flock_doc_updates (id, flock_doc_id, update_data, created_at)
|
|
137
139
|
meta_snapshot (id=1, snapshot, updated_at)
|
|
138
140
|
meta_updates (id, update_data, created_at)
|
|
139
141
|
remote_cursors (stream_url PK, next_offset, server_lower_bound_version, updated_at_ms)
|
|
140
142
|
```
|
|
141
143
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
144
|
+
Post-sync Flock document persistence is version-vector based: no-op syncs do
|
|
145
|
+
not write, and changed state is appended as a JSON delta rather than a full
|
|
146
|
+
Flock file. Local changes that arrive while sync persistence is in flight are
|
|
147
|
+
queued behind that write and flushed from the last durable version. This keeps
|
|
148
|
+
storage growth proportional to new CRDT operations while preserving restart
|
|
149
|
+
convergence. Once a cached Flock document has accumulated 102400 persisted
|
|
150
|
+
delta bytes, the repo asks adapters that support compaction to consolidate
|
|
151
|
+
those updates. Configure this bound with
|
|
152
|
+
`flockDocCompactionByteThreshold` or set it to `0` to disable compaction. The
|
|
153
|
+
threshold-crossing delta save queues a coalesced per-document compaction rather
|
|
154
|
+
than awaiting the full snapshot export, so that sync can return once its delta
|
|
155
|
+
is durable. The adapter compaction starts on a later event-loop turn while
|
|
156
|
+
remaining serialized with later document persistence, and `flush()` /
|
|
157
|
+
`destroy()` drain the queued work.
|
|
158
|
+
|
|
159
|
+
`loadDoc` / `loadFlockDoc` / `loadMeta` opportunistically compact replayed
|
|
160
|
+
updates into a fresh snapshot inside a single transaction, mirroring the
|
|
161
|
+
filesystem adapter's behaviour but without any file churn. Existing databases
|
|
162
|
+
need no migration for delta-aware post-sync persistence; previously amplified
|
|
163
|
+
rows are compacted the next time that Flock document is loaded. Compaction is
|
|
164
|
+
driven by actual delta volume, not by time or sync count, so no-op syncs never
|
|
165
|
+
cause full snapshot exports.
|
|
145
166
|
|
|
146
167
|
### Backups in WAL mode
|
|
147
168
|
|
package/dist/index.cjs
CHANGED
|
@@ -365,6 +365,7 @@ function yieldToMacroTask() {
|
|
|
365
365
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
366
366
|
}
|
|
367
367
|
const YIELD_BUDGET_MS = 5;
|
|
368
|
+
const DEFAULT_COMPACTION_BYTE_THRESHOLD = 100 * 1024;
|
|
368
369
|
const nowMs = typeof globalThis.performance?.now === "function" ? globalThis.performance.now.bind(globalThis.performance) : Date.now;
|
|
369
370
|
function stableVersion(vector) {
|
|
370
371
|
const sorted = {};
|
|
@@ -389,6 +390,7 @@ var FlockDocManager = class {
|
|
|
389
390
|
ensureFlockDocExists;
|
|
390
391
|
retentionMs;
|
|
391
392
|
persistDebounceMs;
|
|
393
|
+
compactionByteThreshold;
|
|
392
394
|
flocks = /* @__PURE__ */ new Map();
|
|
393
395
|
subscriptions = /* @__PURE__ */ new Map();
|
|
394
396
|
persistedVersions = /* @__PURE__ */ new Map();
|
|
@@ -396,6 +398,8 @@ var FlockDocManager = class {
|
|
|
396
398
|
dirtyFlockDocs = /* @__PURE__ */ new Set();
|
|
397
399
|
existenceMaterialized = /* @__PURE__ */ new Set();
|
|
398
400
|
docsWithVisibleContent = /* @__PURE__ */ new Set();
|
|
401
|
+
updateBytesSinceCompaction = /* @__PURE__ */ new Map();
|
|
402
|
+
queuedCompactions = /* @__PURE__ */ new Set();
|
|
399
403
|
debounceTimer;
|
|
400
404
|
flushScheduled = false;
|
|
401
405
|
operationPromise = Promise.resolve();
|
|
@@ -406,6 +410,8 @@ var FlockDocManager = class {
|
|
|
406
410
|
this.retentionMs = options.retentionMs;
|
|
407
411
|
const configured = options.persistDebounceMs;
|
|
408
412
|
this.persistDebounceMs = typeof configured === "number" && Number.isFinite(configured) && configured >= 0 ? configured : 0;
|
|
413
|
+
const configuredCompactionThreshold = options.compactionByteThreshold;
|
|
414
|
+
this.compactionByteThreshold = typeof configuredCompactionThreshold === "number" && Number.isFinite(configuredCompactionThreshold) && configuredCompactionThreshold >= 0 ? Math.floor(configuredCompactionThreshold) : DEFAULT_COMPACTION_BYTE_THRESHOLD;
|
|
409
415
|
}
|
|
410
416
|
async openFlockDoc(flockDocId) {
|
|
411
417
|
return this.ensureFlockDoc(flockDocId);
|
|
@@ -459,16 +465,26 @@ var FlockDocManager = class {
|
|
|
459
465
|
throw error;
|
|
460
466
|
}
|
|
461
467
|
}
|
|
468
|
+
async persistFlockDocChanges(flockDocId, flock) {
|
|
469
|
+
await this.enqueueOperation(async () => {
|
|
470
|
+
if (this.flocks.get(flockDocId) !== flock) return;
|
|
471
|
+
await this.persistFlockDocUpdate(flockDocId, flock);
|
|
472
|
+
});
|
|
473
|
+
}
|
|
462
474
|
async unloadFlockDoc(flockDocId) {
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
475
|
+
await this.enqueueOperation(async () => {
|
|
476
|
+
const flock = this.flocks.get(flockDocId);
|
|
477
|
+
if (!flock) return;
|
|
478
|
+
await this.persistFlockDocUpdate(flockDocId, flock);
|
|
479
|
+
this.subscriptions.get(flockDocId)?.();
|
|
480
|
+
this.subscriptions.delete(flockDocId);
|
|
481
|
+
this.flocks.delete(flockDocId);
|
|
482
|
+
this.persistedVersions.delete(flockDocId);
|
|
483
|
+
this.dirtyFlockDocs.delete(flockDocId);
|
|
484
|
+
this.existenceMaterialized.delete(flockDocId);
|
|
485
|
+
this.docsWithVisibleContent.delete(flockDocId);
|
|
486
|
+
this.updateBytesSinceCompaction.delete(flockDocId);
|
|
487
|
+
});
|
|
472
488
|
}
|
|
473
489
|
async dropFlockDoc(flockDocId) {
|
|
474
490
|
await this.enqueueOperation(async () => {
|
|
@@ -479,6 +495,7 @@ var FlockDocManager = class {
|
|
|
479
495
|
this.dirtyFlockDocs.delete(flockDocId);
|
|
480
496
|
this.existenceMaterialized.delete(flockDocId);
|
|
481
497
|
this.docsWithVisibleContent.delete(flockDocId);
|
|
498
|
+
this.updateBytesSinceCompaction.delete(flockDocId);
|
|
482
499
|
await this.storage?.deleteFlockDoc?.(flockDocId);
|
|
483
500
|
});
|
|
484
501
|
}
|
|
@@ -487,6 +504,7 @@ var FlockDocManager = class {
|
|
|
487
504
|
this.flushScheduled = false;
|
|
488
505
|
while (true) {
|
|
489
506
|
await this.enqueueOperation(() => this.drainDirtyFlockDocs());
|
|
507
|
+
await this.operationPromise;
|
|
490
508
|
if (this.dirtyFlockDocs.size === 0 && !this.flushScheduled) break;
|
|
491
509
|
this.flushScheduled = false;
|
|
492
510
|
}
|
|
@@ -507,6 +525,8 @@ var FlockDocManager = class {
|
|
|
507
525
|
this.dirtyFlockDocs.clear();
|
|
508
526
|
this.existenceMaterialized.clear();
|
|
509
527
|
this.docsWithVisibleContent.clear();
|
|
528
|
+
this.updateBytesSinceCompaction.clear();
|
|
529
|
+
this.queuedCompactions.clear();
|
|
510
530
|
}
|
|
511
531
|
}
|
|
512
532
|
async loadOrCreateFlockDoc(flockDocId) {
|
|
@@ -555,22 +575,23 @@ var FlockDocManager = class {
|
|
|
555
575
|
}
|
|
556
576
|
if (versionEquals(previousVersion, nextVersion)) {
|
|
557
577
|
await this.materializeExistence(flockDocId, flock);
|
|
558
|
-
this.
|
|
578
|
+
this.clearDirtyIfStable(flockDocId, nextVersion, flock);
|
|
559
579
|
return;
|
|
560
580
|
}
|
|
561
581
|
const bundle = await exportJsonUpdate(flock, previousVersion);
|
|
562
582
|
if (!hasBundleEntries(bundle)) {
|
|
563
583
|
await this.materializeExistence(flockDocId, flock);
|
|
564
584
|
this.persistedVersions.set(flockDocId, nextVersion);
|
|
565
|
-
this.
|
|
585
|
+
this.clearDirtyIfStable(flockDocId, nextVersion, flock);
|
|
566
586
|
return;
|
|
567
587
|
}
|
|
588
|
+
const update = require_flock_snapshot.encodeMetaJsonUpdate(bundle);
|
|
568
589
|
this.persistedVersions.set(flockDocId, nextVersion);
|
|
569
590
|
try {
|
|
570
591
|
await this.storage.save({
|
|
571
592
|
type: "flock-doc-update",
|
|
572
593
|
flockDocId,
|
|
573
|
-
update
|
|
594
|
+
update
|
|
574
595
|
});
|
|
575
596
|
await this.materializeExistence(flockDocId, flock);
|
|
576
597
|
this.clearDirtyIfStable(flockDocId, nextVersion, flock);
|
|
@@ -578,6 +599,31 @@ var FlockDocManager = class {
|
|
|
578
599
|
this.persistedVersions.set(flockDocId, previousVersion);
|
|
579
600
|
throw error;
|
|
580
601
|
}
|
|
602
|
+
this.scheduleFlockDocCompaction(flockDocId, flock, update.byteLength);
|
|
603
|
+
}
|
|
604
|
+
scheduleFlockDocCompaction(flockDocId, flock, updateBytes) {
|
|
605
|
+
if (this.compactionByteThreshold === 0 || !this.storage?.compactFlockDoc) return;
|
|
606
|
+
const total = (this.updateBytesSinceCompaction.get(flockDocId) ?? 0) + updateBytes;
|
|
607
|
+
this.updateBytesSinceCompaction.set(flockDocId, total);
|
|
608
|
+
if (total < this.compactionByteThreshold) return;
|
|
609
|
+
this.queueFlockDocCompaction(flockDocId, flock);
|
|
610
|
+
}
|
|
611
|
+
queueFlockDocCompaction(flockDocId, flock) {
|
|
612
|
+
if (this.queuedCompactions.has(flockDocId)) return;
|
|
613
|
+
const storage = this.storage;
|
|
614
|
+
if (!storage?.compactFlockDoc) return;
|
|
615
|
+
this.queuedCompactions.add(flockDocId);
|
|
616
|
+
this.enqueueOperation(async () => {
|
|
617
|
+
try {
|
|
618
|
+
await yieldToMacroTask();
|
|
619
|
+
if (this.flocks.get(flockDocId) !== flock) return;
|
|
620
|
+
await storage.compactFlockDoc?.(flockDocId);
|
|
621
|
+
if (this.flocks.has(flockDocId)) this.updateBytesSinceCompaction.set(flockDocId, 0);
|
|
622
|
+
else this.updateBytesSinceCompaction.delete(flockDocId);
|
|
623
|
+
} finally {
|
|
624
|
+
this.queuedCompactions.delete(flockDocId);
|
|
625
|
+
}
|
|
626
|
+
}).catch(require_logging.logAsyncError(`Flock document "${flockDocId}" compaction`));
|
|
581
627
|
}
|
|
582
628
|
async materializeExistence(flockDocId, flock) {
|
|
583
629
|
if (this.existenceMaterialized.has(flockDocId)) return;
|
|
@@ -1499,7 +1545,7 @@ var SyncRunner = class {
|
|
|
1499
1545
|
} finally {
|
|
1500
1546
|
this.eventBus.popEventBy();
|
|
1501
1547
|
}
|
|
1502
|
-
await this.flockDocManager.
|
|
1548
|
+
await this.flockDocManager.persistFlockDocChanges(flockDocId, flock);
|
|
1503
1549
|
}
|
|
1504
1550
|
}
|
|
1505
1551
|
async buildSyncPlan(options, buildOptions = {}) {
|
|
@@ -1979,7 +2025,8 @@ var LoroRepo = class LoroRepo {
|
|
|
1979
2025
|
storage: this.storage,
|
|
1980
2026
|
ensureFlockDocExists: (flockDocId, expiresAtMs) => this.flockDocRegistry.ensureExists(flockDocId, expiresAtMs),
|
|
1981
2027
|
retentionMs: getFlockDocRetentionMs,
|
|
1982
|
-
persistDebounceMs: options.docPersistDebounceMs
|
|
2028
|
+
persistDebounceMs: options.docPersistDebounceMs,
|
|
2029
|
+
compactionByteThreshold: options.flockDocCompactionByteThreshold
|
|
1983
2030
|
});
|
|
1984
2031
|
if (this.storage) this.metaPersister = new MetaPersister({
|
|
1985
2032
|
getMetaFlock,
|