loro-repo 0.18.3 → 0.18.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -134,14 +134,35 @@ The schema (created on first use) is:
134
134
  ```
135
135
  docs (doc_id PK, snapshot, updated_at, snapshot_digest)
136
136
  doc_updates (id, doc_id, update_data, created_at)
137
+ flock_docs (flock_doc_id PK, snapshot, updated_at, snapshot_digest)
138
+ flock_doc_updates (id, flock_doc_id, update_data, created_at)
137
139
  meta_snapshot (id=1, snapshot, updated_at)
138
140
  meta_updates (id, update_data, created_at)
139
141
  remote_cursors (stream_url PK, next_offset, server_lower_bound_version, updated_at_ms)
140
142
  ```
141
143
 
142
- `loadDoc` / `loadMeta` opportunistically compact replayed updates into a fresh
143
- snapshot inside a single transaction, mirroring the filesystem adapter's
144
- behaviour but without any file churn.
144
+ Post-sync Flock document persistence is version-vector based: no-op syncs do
145
+ not write, and changed state is appended as a JSON delta rather than a full
146
+ Flock file. Local changes that arrive while sync persistence is in flight are
147
+ queued behind that write and flushed from the last durable version. This keeps
148
+ storage growth proportional to new CRDT operations while preserving restart
149
+ convergence. Once a cached Flock document has accumulated 102400 persisted
150
+ delta bytes, the repo asks adapters that support compaction to consolidate
151
+ those updates. Configure this bound with
152
+ `flockDocCompactionByteThreshold` or set it to `0` to disable compaction. The
153
+ threshold-crossing delta save queues a coalesced per-document compaction rather
154
+ than awaiting the full snapshot export, so that sync can return once its delta
155
+ is durable. The adapter compaction starts on a later event-loop turn while
156
+ remaining serialized with later document persistence, and `flush()` /
157
+ `destroy()` drain the queued work.
158
+
159
+ `loadDoc` / `loadFlockDoc` / `loadMeta` opportunistically compact replayed
160
+ updates into a fresh snapshot inside a single transaction, mirroring the
161
+ filesystem adapter's behaviour but without any file churn. Existing databases
162
+ need no migration for delta-aware post-sync persistence; previously amplified
163
+ rows are compacted the next time that Flock document is loaded. Compaction is
164
+ driven by actual delta volume, not by time or sync count, so no-op syncs never
165
+ cause full snapshot exports.
145
166
 
146
167
  ### Backups in WAL mode
147
168
 
@@ -81,6 +81,25 @@ function mergeMetaSnapshot(flock, bytes) {
81
81
  };
82
82
  }
83
83
 
84
+ //#endregion
85
+ //#region src/internal/flock-snapshot.ts
86
+ const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;
87
+ async function exportFlockFileSnapshot(flock, options = {}) {
88
+ const exportIncremental = flock.exportFileV2Incremental;
89
+ if (typeof exportIncremental === "function") return exportIncremental.call(flock, {
90
+ budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,
91
+ signal: options.signal
92
+ });
93
+ return flock.exportFile();
94
+ }
95
+ function hasVisibleRows(flock) {
96
+ const isEmpty = flock.isEmpty;
97
+ if (typeof isEmpty === "function") return !isEmpty.call(flock);
98
+ const rows = flock.scan({});
99
+ for (const row of rows) if (row.value !== void 0) return true;
100
+ return false;
101
+ }
102
+
84
103
  //#endregion
85
104
  Object.defineProperty(exports, 'encodeMetaJsonUpdate', {
86
105
  enumerable: true,
@@ -88,6 +107,18 @@ Object.defineProperty(exports, 'encodeMetaJsonUpdate', {
88
107
  return encodeMetaJsonUpdate;
89
108
  }
90
109
  });
110
+ Object.defineProperty(exports, 'exportFlockFileSnapshot', {
111
+ enumerable: true,
112
+ get: function () {
113
+ return exportFlockFileSnapshot;
114
+ }
115
+ });
116
+ Object.defineProperty(exports, 'hasVisibleRows', {
117
+ enumerable: true,
118
+ get: function () {
119
+ return hasVisibleRows;
120
+ }
121
+ });
91
122
  Object.defineProperty(exports, 'hydrateMetaSnapshots', {
92
123
  enumerable: true,
93
124
  get: function () {
@@ -100,4 +131,4 @@ Object.defineProperty(exports, 'isFlockFile', {
100
131
  return isFlockFile;
101
132
  }
102
133
  });
103
- //# sourceMappingURL=meta-snapshot.cjs.map
134
+ //# sourceMappingURL=flock-snapshot.cjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"flock-snapshot.cjs","names":["flock: Flock | undefined","Flock"],"sources":["../src/storage/meta-snapshot.ts","../src/internal/flock-snapshot.ts"],"sourcesContent":["import { Flock, type ExportBundle } from \"@loro-dev/flock-wasm\";\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\nconst FLOCK_FILE_MAGIC_LENGTH = 4;\n// Flock file snapshots are prefixed with a 4-byte magic header. The legacy v1\n// B+Tree page format uses \"FLOK\"; flock-wasm >=0.4.0 writes the canonical v2\n// format with \"FLK2\". `Flock.fromFile`/`importFile` read both, so we just need\n// to recognize either magic to distinguish a binary snapshot from a JSON bundle.\nconst FLOCK_FILE_MAGIC_V1 = [0x46, 0x4c, 0x4f, 0x4b]; // \"FLOK\"\nconst FLOCK_FILE_MAGIC_V2 = [0x46, 0x4c, 0x4b, 0x32]; // \"FLK2\"\n\ntype SnapshotFormat = \"file\" | \"json\";\n\ninterface MergeResult {\n flock: Flock;\n format: SnapshotFormat;\n}\n\nexport interface HydrateMetaSnapshotResult {\n flock: Flock | undefined;\n loadedJsonUpdate: boolean;\n}\n\nfunction matchesMagic(bytes: Uint8Array, magic: readonly number[]): boolean {\n for (let i = 0; i < magic.length; i += 1) {\n if (bytes[i] !== magic[i]) return false;\n }\n return true;\n}\n\nexport function isFlockFile(bytes: Uint8Array): boolean {\n return (\n bytes.length >= FLOCK_FILE_MAGIC_LENGTH &&\n (matchesMagic(bytes, FLOCK_FILE_MAGIC_V1) ||\n matchesMagic(bytes, FLOCK_FILE_MAGIC_V2))\n );\n}\n\nexport function encodeMetaJsonUpdate(bundle: ExportBundle): Uint8Array {\n return textEncoder.encode(JSON.stringify(bundle));\n}\n\nexport function hydrateMetaSnapshots(\n legacy: Uint8Array | undefined,\n updates: readonly Uint8Array[],\n): HydrateMetaSnapshotResult {\n let loadedJsonUpdate = false;\n let baseUpdateIndex = -1;\n let flock: Flock | undefined;\n\n if (legacy) {\n const merged = mergeMetaSnapshot(flock, legacy);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n } else {\n baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));\n if (baseUpdateIndex >= 0) {\n flock = Flock.fromFile(updates[baseUpdateIndex] as Uint8Array);\n }\n }\n\n for (let index = 0; index < updates.length; index += 1) {\n if (index === baseUpdateIndex) continue;\n const merged = mergeMetaSnapshot(flock, updates[index] as Uint8Array);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n }\n\n return { flock, loadedJsonUpdate };\n}\n\nfunction mergeMetaSnapshot(\n flock: Flock | undefined,\n bytes: Uint8Array,\n): MergeResult {\n if (isFlockFile(bytes)) {\n if (flock) {\n flock.importFile(bytes);\n return { flock, format: \"file\" };\n }\n return { flock: Flock.fromFile(bytes), format: \"file\" };\n }\n\n const bundle = JSON.parse(textDecoder.decode(bytes)) as ExportBundle;\n if (flock) {\n flock.importJson(bundle);\n return { flock, format: \"json\" };\n }\n const hydrated = new Flock();\n hydrated.importJson(bundle);\n return { flock: hydrated, format: \"json\" };\n}\n","import type { Flock } from \"@loro-dev/flock-wasm\";\n\nexport const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;\n\nexport interface ExportFlockFileSnapshotOptions {\n readonly budgetMs?: number;\n readonly signal?: AbortSignal;\n}\n\ntype FlockWithIncrementalFileExport = Flock & {\n exportFileV2Incremental?: (\n options?: ExportFlockFileSnapshotOptions,\n ) => Promise<Uint8Array>;\n};\n\ntype FlockWithIsEmpty = Flock & {\n isEmpty?: () => boolean;\n};\n\nexport async function exportFlockFileSnapshot(\n flock: Flock,\n options: ExportFlockFileSnapshotOptions = {},\n): Promise<Uint8Array> {\n const exportIncremental = (flock as FlockWithIncrementalFileExport)\n .exportFileV2Incremental;\n if (typeof exportIncremental === \"function\") {\n return exportIncremental.call(flock, {\n budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,\n signal: options.signal,\n });\n }\n return flock.exportFile();\n}\n\nexport function hasVisibleRows(flock: Flock): boolean {\n const isEmpty = (flock as FlockWithIsEmpty).isEmpty;\n if (typeof isEmpty === \"function\") {\n return !isEmpty.call(flock);\n }\n\n const rows = flock.scan({});\n for (const row of rows) {\n if (row.value !== undefined) return true;\n }\n return false;\n}\n"],"mappings":";;;;;AAEA,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,0BAA0B;AAKhC,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AACpD,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AAcpD,SAAS,aAAa,OAAmB,OAAmC;AAC1E,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,EACrC,KAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAEpC,QAAO;;AAGT,SAAgB,YAAY,OAA4B;AACtD,QACE,MAAM,UAAU,4BACf,aAAa,OAAO,oBAAoB,IACvC,aAAa,OAAO,oBAAoB;;AAI9C,SAAgB,qBAAqB,QAAkC;AACrE,QAAO,YAAY,OAAO,KAAK,UAAU,OAAO,CAAC;;AAGnD,SAAgB,qBACd,QACA,SAC2B;CAC3B,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,IAAIA;AAEJ,KAAI,QAAQ;EACV,MAAM,SAAS,kBAAkB,OAAO,OAAO;AAC/C,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;QAEhB;AACL,oBAAkB,QAAQ,WAAW,UAAU,YAAY,MAAM,CAAC;AAClE,MAAI,mBAAmB,EACrB,SAAQC,4BAAM,SAAS,QAAQ,iBAA+B;;AAIlE,MAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;AACtD,MAAI,UAAU,gBAAiB;EAC/B,MAAM,SAAS,kBAAkB,OAAO,QAAQ,OAAqB;AACrE,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;;AAIvB,QAAO;EAAE;EAAO;EAAkB;;AAGpC,SAAS,kBACP,OACA,OACa;AACb,KAAI,YAAY,MAAM,EAAE;AACtB,MAAI,OAAO;AACT,SAAM,WAAW,MAAM;AACvB,UAAO;IAAE;IAAO,QAAQ;IAAQ;;AAElC,SAAO;GAAE,OAAOA,4BAAM,SAAS,MAAM;GAAE,QAAQ;GAAQ;;CAGzD,MAAM,SAAS,KAAK,MAAM,YAAY,OAAO,MAAM,CAAC;AACpD,KAAI,OAAO;AACT,QAAM,WAAW,OAAO;AACxB,SAAO;GAAE;GAAO,QAAQ;GAAQ;;CAElC,MAAM,WAAW,IAAIA,6BAAO;AAC5B,UAAS,WAAW,OAAO;AAC3B,QAAO;EAAE,OAAO;EAAU,QAAQ;EAAQ;;;;;AC7F5C,MAAa,sCAAsC;AAiBnD,eAAsB,wBACpB,OACA,UAA0C,EAAE,EACvB;CACrB,MAAM,oBAAqB,MACxB;AACH,KAAI,OAAO,sBAAsB,WAC/B,QAAO,kBAAkB,KAAK,OAAO;EACnC,UAAU,QAAQ,YAAY;EAC9B,QAAQ,QAAQ;EACjB,CAAC;AAEJ,QAAO,MAAM,YAAY;;AAG3B,SAAgB,eAAe,OAAuB;CACpD,MAAM,UAAW,MAA2B;AAC5C,KAAI,OAAO,YAAY,WACrB,QAAO,CAAC,QAAQ,KAAK,MAAM;CAG7B,MAAM,OAAO,MAAM,KAAK,EAAE,CAAC;AAC3B,MAAK,MAAM,OAAO,KAChB,KAAI,IAAI,UAAU,OAAW,QAAO;AAEtC,QAAO"}
@@ -80,5 +80,24 @@ function mergeMetaSnapshot(flock, bytes) {
80
80
  }
81
81
 
82
82
  //#endregion
83
- export { hydrateMetaSnapshots as n, isFlockFile as r, encodeMetaJsonUpdate as t };
84
- //# sourceMappingURL=meta-snapshot.js.map
83
+ //#region src/internal/flock-snapshot.ts
84
+ const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;
85
+ async function exportFlockFileSnapshot(flock, options = {}) {
86
+ const exportIncremental = flock.exportFileV2Incremental;
87
+ if (typeof exportIncremental === "function") return exportIncremental.call(flock, {
88
+ budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,
89
+ signal: options.signal
90
+ });
91
+ return flock.exportFile();
92
+ }
93
+ function hasVisibleRows(flock) {
94
+ const isEmpty = flock.isEmpty;
95
+ if (typeof isEmpty === "function") return !isEmpty.call(flock);
96
+ const rows = flock.scan({});
97
+ for (const row of rows) if (row.value !== void 0) return true;
98
+ return false;
99
+ }
100
+
101
+ //#endregion
102
+ export { isFlockFile as a, hydrateMetaSnapshots as i, hasVisibleRows as n, encodeMetaJsonUpdate as r, exportFlockFileSnapshot as t };
103
+ //# sourceMappingURL=flock-snapshot.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"flock-snapshot.js","names":["flock: Flock | undefined"],"sources":["../src/storage/meta-snapshot.ts","../src/internal/flock-snapshot.ts"],"sourcesContent":["import { Flock, type ExportBundle } from \"@loro-dev/flock-wasm\";\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\nconst FLOCK_FILE_MAGIC_LENGTH = 4;\n// Flock file snapshots are prefixed with a 4-byte magic header. The legacy v1\n// B+Tree page format uses \"FLOK\"; flock-wasm >=0.4.0 writes the canonical v2\n// format with \"FLK2\". `Flock.fromFile`/`importFile` read both, so we just need\n// to recognize either magic to distinguish a binary snapshot from a JSON bundle.\nconst FLOCK_FILE_MAGIC_V1 = [0x46, 0x4c, 0x4f, 0x4b]; // \"FLOK\"\nconst FLOCK_FILE_MAGIC_V2 = [0x46, 0x4c, 0x4b, 0x32]; // \"FLK2\"\n\ntype SnapshotFormat = \"file\" | \"json\";\n\ninterface MergeResult {\n flock: Flock;\n format: SnapshotFormat;\n}\n\nexport interface HydrateMetaSnapshotResult {\n flock: Flock | undefined;\n loadedJsonUpdate: boolean;\n}\n\nfunction matchesMagic(bytes: Uint8Array, magic: readonly number[]): boolean {\n for (let i = 0; i < magic.length; i += 1) {\n if (bytes[i] !== magic[i]) return false;\n }\n return true;\n}\n\nexport function isFlockFile(bytes: Uint8Array): boolean {\n return (\n bytes.length >= FLOCK_FILE_MAGIC_LENGTH &&\n (matchesMagic(bytes, FLOCK_FILE_MAGIC_V1) ||\n matchesMagic(bytes, FLOCK_FILE_MAGIC_V2))\n );\n}\n\nexport function encodeMetaJsonUpdate(bundle: ExportBundle): Uint8Array {\n return textEncoder.encode(JSON.stringify(bundle));\n}\n\nexport function hydrateMetaSnapshots(\n legacy: Uint8Array | undefined,\n updates: readonly Uint8Array[],\n): HydrateMetaSnapshotResult {\n let loadedJsonUpdate = false;\n let baseUpdateIndex = -1;\n let flock: Flock | undefined;\n\n if (legacy) {\n const merged = mergeMetaSnapshot(flock, legacy);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n } else {\n baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));\n if (baseUpdateIndex >= 0) {\n flock = Flock.fromFile(updates[baseUpdateIndex] as Uint8Array);\n }\n }\n\n for (let index = 0; index < updates.length; index += 1) {\n if (index === baseUpdateIndex) continue;\n const merged = mergeMetaSnapshot(flock, updates[index] as Uint8Array);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n }\n\n return { flock, loadedJsonUpdate };\n}\n\nfunction mergeMetaSnapshot(\n flock: Flock | undefined,\n bytes: Uint8Array,\n): MergeResult {\n if (isFlockFile(bytes)) {\n if (flock) {\n flock.importFile(bytes);\n return { flock, format: \"file\" };\n }\n return { flock: Flock.fromFile(bytes), format: \"file\" };\n }\n\n const bundle = JSON.parse(textDecoder.decode(bytes)) as ExportBundle;\n if (flock) {\n flock.importJson(bundle);\n return { flock, format: \"json\" };\n }\n const hydrated = new Flock();\n hydrated.importJson(bundle);\n return { flock: hydrated, format: \"json\" };\n}\n","import type { Flock } from \"@loro-dev/flock-wasm\";\n\nexport const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;\n\nexport interface ExportFlockFileSnapshotOptions {\n readonly budgetMs?: number;\n readonly signal?: AbortSignal;\n}\n\ntype FlockWithIncrementalFileExport = Flock & {\n exportFileV2Incremental?: (\n options?: ExportFlockFileSnapshotOptions,\n ) => Promise<Uint8Array>;\n};\n\ntype FlockWithIsEmpty = Flock & {\n isEmpty?: () => boolean;\n};\n\nexport async function exportFlockFileSnapshot(\n flock: Flock,\n options: ExportFlockFileSnapshotOptions = {},\n): Promise<Uint8Array> {\n const exportIncremental = (flock as FlockWithIncrementalFileExport)\n .exportFileV2Incremental;\n if (typeof exportIncremental === \"function\") {\n return exportIncremental.call(flock, {\n budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,\n signal: options.signal,\n });\n }\n return flock.exportFile();\n}\n\nexport function hasVisibleRows(flock: Flock): boolean {\n const isEmpty = (flock as FlockWithIsEmpty).isEmpty;\n if (typeof isEmpty === \"function\") {\n return !isEmpty.call(flock);\n }\n\n const rows = flock.scan({});\n for (const row of rows) {\n if (row.value !== undefined) return true;\n }\n return false;\n}\n"],"mappings":";;;AAEA,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,0BAA0B;AAKhC,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AACpD,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AAcpD,SAAS,aAAa,OAAmB,OAAmC;AAC1E,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,EACrC,KAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAEpC,QAAO;;AAGT,SAAgB,YAAY,OAA4B;AACtD,QACE,MAAM,UAAU,4BACf,aAAa,OAAO,oBAAoB,IACvC,aAAa,OAAO,oBAAoB;;AAI9C,SAAgB,qBAAqB,QAAkC;AACrE,QAAO,YAAY,OAAO,KAAK,UAAU,OAAO,CAAC;;AAGnD,SAAgB,qBACd,QACA,SAC2B;CAC3B,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,IAAIA;AAEJ,KAAI,QAAQ;EACV,MAAM,SAAS,kBAAkB,OAAO,OAAO;AAC/C,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;QAEhB;AACL,oBAAkB,QAAQ,WAAW,UAAU,YAAY,MAAM,CAAC;AAClE,MAAI,mBAAmB,EACrB,SAAQ,MAAM,SAAS,QAAQ,iBAA+B;;AAIlE,MAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;AACtD,MAAI,UAAU,gBAAiB;EAC/B,MAAM,SAAS,kBAAkB,OAAO,QAAQ,OAAqB;AACrE,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;;AAIvB,QAAO;EAAE;EAAO;EAAkB;;AAGpC,SAAS,kBACP,OACA,OACa;AACb,KAAI,YAAY,MAAM,EAAE;AACtB,MAAI,OAAO;AACT,SAAM,WAAW,MAAM;AACvB,UAAO;IAAE;IAAO,QAAQ;IAAQ;;AAElC,SAAO;GAAE,OAAO,MAAM,SAAS,MAAM;GAAE,QAAQ;GAAQ;;CAGzD,MAAM,SAAS,KAAK,MAAM,YAAY,OAAO,MAAM,CAAC;AACpD,KAAI,OAAO;AACT,QAAM,WAAW,OAAO;AACxB,SAAO;GAAE;GAAO,QAAQ;GAAQ;;CAElC,MAAM,WAAW,IAAI,OAAO;AAC5B,UAAS,WAAW,OAAO;AAC3B,QAAO;EAAE,OAAO;EAAU,QAAQ;EAAQ;;;;;AC7F5C,MAAa,sCAAsC;AAiBnD,eAAsB,wBACpB,OACA,UAA0C,EAAE,EACvB;CACrB,MAAM,oBAAqB,MACxB;AACH,KAAI,OAAO,sBAAsB,WAC/B,QAAO,kBAAkB,KAAK,OAAO;EACnC,UAAU,QAAQ,YAAY;EAC9B,QAAQ,QAAQ;EACjB,CAAC;AAEJ,QAAO,MAAM,YAAY;;AAG3B,SAAgB,eAAe,OAAuB;CACpD,MAAM,UAAW,MAA2B;AAC5C,KAAI,OAAO,YAAY,WACrB,QAAO,CAAC,QAAQ,KAAK,MAAM;CAG7B,MAAM,OAAO,MAAM,KAAK,EAAE,CAAC;AAC3B,MAAK,MAAM,OAAO,KAChB,KAAI,IAAI,UAAU,OAAW,QAAO;AAEtC,QAAO"}
package/dist/index.cjs CHANGED
@@ -1,6 +1,6 @@
1
1
  const require_chunk = require('./chunk.cjs');
2
2
  const require_logging = require('./logging.cjs');
3
- const require_meta_snapshot = require('./meta-snapshot.cjs');
3
+ const require_flock_snapshot = require('./flock-snapshot.cjs');
4
4
  let __loro_dev_flock_wasm = require("@loro-dev/flock-wasm");
5
5
  __loro_dev_flock_wasm = require_chunk.__toESM(__loro_dev_flock_wasm);
6
6
  let loro_crdt = require("loro-crdt");
@@ -365,6 +365,7 @@ function yieldToMacroTask() {
365
365
  return new Promise((resolve) => setTimeout(resolve, 0));
366
366
  }
367
367
  const YIELD_BUDGET_MS = 5;
368
+ const DEFAULT_COMPACTION_BYTE_THRESHOLD = 100 * 1024;
368
369
  const nowMs = typeof globalThis.performance?.now === "function" ? globalThis.performance.now.bind(globalThis.performance) : Date.now;
369
370
  function stableVersion(vector) {
370
371
  const sorted = {};
@@ -377,11 +378,6 @@ function versionEquals(a, b) {
377
378
  function hasEmptyVersion(vector) {
378
379
  return Object.keys(vector).length === 0;
379
380
  }
380
- function hasVisibleRows(flock) {
381
- const rows = flock.scan({});
382
- for (const row of rows) if (row.value !== void 0) return true;
383
- return false;
384
- }
385
381
  async function exportJsonUpdate(flock, from) {
386
382
  if (hasEmptyVersion(from)) return flock.exportJson();
387
383
  return flock.exportJson({ from });
@@ -394,6 +390,7 @@ var FlockDocManager = class {
394
390
  ensureFlockDocExists;
395
391
  retentionMs;
396
392
  persistDebounceMs;
393
+ compactionByteThreshold;
397
394
  flocks = /* @__PURE__ */ new Map();
398
395
  subscriptions = /* @__PURE__ */ new Map();
399
396
  persistedVersions = /* @__PURE__ */ new Map();
@@ -401,6 +398,8 @@ var FlockDocManager = class {
401
398
  dirtyFlockDocs = /* @__PURE__ */ new Set();
402
399
  existenceMaterialized = /* @__PURE__ */ new Set();
403
400
  docsWithVisibleContent = /* @__PURE__ */ new Set();
401
+ updateBytesSinceCompaction = /* @__PURE__ */ new Map();
402
+ queuedCompactions = /* @__PURE__ */ new Set();
404
403
  debounceTimer;
405
404
  flushScheduled = false;
406
405
  operationPromise = Promise.resolve();
@@ -411,15 +410,17 @@ var FlockDocManager = class {
411
410
  this.retentionMs = options.retentionMs;
412
411
  const configured = options.persistDebounceMs;
413
412
  this.persistDebounceMs = typeof configured === "number" && Number.isFinite(configured) && configured >= 0 ? configured : 0;
413
+ const configuredCompactionThreshold = options.compactionByteThreshold;
414
+ this.compactionByteThreshold = typeof configuredCompactionThreshold === "number" && Number.isFinite(configuredCompactionThreshold) && configuredCompactionThreshold >= 0 ? Math.floor(configuredCompactionThreshold) : DEFAULT_COMPACTION_BYTE_THRESHOLD;
414
415
  }
415
416
  async openFlockDoc(flockDocId) {
416
417
  return this.ensureFlockDoc(flockDocId);
417
418
  }
418
419
  async openDetachedFlockDoc(flockDocId) {
419
420
  const cached = this.flocks.get(flockDocId);
420
- if (cached) return __loro_dev_flock_wasm.Flock.fromFile(cached.exportFile());
421
+ if (cached) return __loro_dev_flock_wasm.Flock.fromFile(await require_flock_snapshot.exportFlockFileSnapshot(cached));
421
422
  const stored = await this.storage?.loadFlockDoc?.(flockDocId);
422
- if (stored) return __loro_dev_flock_wasm.Flock.fromFile(stored.exportFile());
423
+ if (stored) return __loro_dev_flock_wasm.Flock.fromFile(await require_flock_snapshot.exportFlockFileSnapshot(stored));
423
424
  return new __loro_dev_flock_wasm.Flock();
424
425
  }
425
426
  async ensureFlockDoc(flockDocId) {
@@ -441,16 +442,15 @@ var FlockDocManager = class {
441
442
  }
442
443
  async persistFlockDoc(flockDocId, flock) {
443
444
  const previousVersion = this.persistedVersions.get(flockDocId);
444
- const snapshot = flock.exportFile();
445
- const nextVersion = flock.version();
446
445
  const versionAtCapture = flock.version();
447
446
  if (!this.storage) {
448
- this.persistedVersions.set(flockDocId, nextVersion);
447
+ this.persistedVersions.set(flockDocId, versionAtCapture);
449
448
  await this.materializeExistence(flockDocId, flock);
450
449
  this.clearDirtyIfStable(flockDocId, versionAtCapture, flock);
451
450
  return;
452
451
  }
453
- this.persistedVersions.set(flockDocId, nextVersion);
452
+ const snapshot = await require_flock_snapshot.exportFlockFileSnapshot(flock);
453
+ this.persistedVersions.set(flockDocId, versionAtCapture);
454
454
  try {
455
455
  await this.storage.save({
456
456
  type: "flock-doc-snapshot",
@@ -465,16 +465,26 @@ var FlockDocManager = class {
465
465
  throw error;
466
466
  }
467
467
  }
468
+ async persistFlockDocChanges(flockDocId, flock) {
469
+ await this.enqueueOperation(async () => {
470
+ if (this.flocks.get(flockDocId) !== flock) return;
471
+ await this.persistFlockDocUpdate(flockDocId, flock);
472
+ });
473
+ }
468
474
  async unloadFlockDoc(flockDocId) {
469
- const flock = this.flocks.get(flockDocId);
470
- if (!flock) return;
471
- await this.persistFlockDocUpdate(flockDocId, flock);
472
- this.subscriptions.get(flockDocId)?.();
473
- this.subscriptions.delete(flockDocId);
474
- this.flocks.delete(flockDocId);
475
- this.persistedVersions.delete(flockDocId);
476
- this.existenceMaterialized.delete(flockDocId);
477
- this.docsWithVisibleContent.delete(flockDocId);
475
+ await this.enqueueOperation(async () => {
476
+ const flock = this.flocks.get(flockDocId);
477
+ if (!flock) return;
478
+ await this.persistFlockDocUpdate(flockDocId, flock);
479
+ this.subscriptions.get(flockDocId)?.();
480
+ this.subscriptions.delete(flockDocId);
481
+ this.flocks.delete(flockDocId);
482
+ this.persistedVersions.delete(flockDocId);
483
+ this.dirtyFlockDocs.delete(flockDocId);
484
+ this.existenceMaterialized.delete(flockDocId);
485
+ this.docsWithVisibleContent.delete(flockDocId);
486
+ this.updateBytesSinceCompaction.delete(flockDocId);
487
+ });
478
488
  }
479
489
  async dropFlockDoc(flockDocId) {
480
490
  await this.enqueueOperation(async () => {
@@ -485,6 +495,7 @@ var FlockDocManager = class {
485
495
  this.dirtyFlockDocs.delete(flockDocId);
486
496
  this.existenceMaterialized.delete(flockDocId);
487
497
  this.docsWithVisibleContent.delete(flockDocId);
498
+ this.updateBytesSinceCompaction.delete(flockDocId);
488
499
  await this.storage?.deleteFlockDoc?.(flockDocId);
489
500
  });
490
501
  }
@@ -493,6 +504,7 @@ var FlockDocManager = class {
493
504
  this.flushScheduled = false;
494
505
  while (true) {
495
506
  await this.enqueueOperation(() => this.drainDirtyFlockDocs());
507
+ await this.operationPromise;
496
508
  if (this.dirtyFlockDocs.size === 0 && !this.flushScheduled) break;
497
509
  this.flushScheduled = false;
498
510
  }
@@ -513,6 +525,8 @@ var FlockDocManager = class {
513
525
  this.dirtyFlockDocs.clear();
514
526
  this.existenceMaterialized.clear();
515
527
  this.docsWithVisibleContent.clear();
528
+ this.updateBytesSinceCompaction.clear();
529
+ this.queuedCompactions.clear();
516
530
  }
517
531
  }
518
532
  async loadOrCreateFlockDoc(flockDocId) {
@@ -530,7 +544,7 @@ var FlockDocManager = class {
530
544
  registerFlockDoc(flockDocId, flock) {
531
545
  this.flocks.set(flockDocId, flock);
532
546
  this.persistedVersions.set(flockDocId, flock.version());
533
- if (hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
547
+ if (require_flock_snapshot.hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
534
548
  this.ensureSubscription(flockDocId, flock);
535
549
  }
536
550
  ensureSubscription(flockDocId, flock) {
@@ -557,28 +571,27 @@ var FlockDocManager = class {
557
571
  }
558
572
  if (!previousVersion) {
559
573
  await this.persistFlockDoc(flockDocId, flock);
560
- this.persistedVersions.set(flockDocId, nextVersion);
561
- this.clearDirtyIfStable(flockDocId, nextVersion, flock);
562
574
  return;
563
575
  }
564
576
  if (versionEquals(previousVersion, nextVersion)) {
565
577
  await this.materializeExistence(flockDocId, flock);
566
- this.dirtyFlockDocs.delete(flockDocId);
578
+ this.clearDirtyIfStable(flockDocId, nextVersion, flock);
567
579
  return;
568
580
  }
569
581
  const bundle = await exportJsonUpdate(flock, previousVersion);
570
582
  if (!hasBundleEntries(bundle)) {
571
583
  await this.materializeExistence(flockDocId, flock);
572
584
  this.persistedVersions.set(flockDocId, nextVersion);
573
- this.dirtyFlockDocs.delete(flockDocId);
585
+ this.clearDirtyIfStable(flockDocId, nextVersion, flock);
574
586
  return;
575
587
  }
588
+ const update = require_flock_snapshot.encodeMetaJsonUpdate(bundle);
576
589
  this.persistedVersions.set(flockDocId, nextVersion);
577
590
  try {
578
591
  await this.storage.save({
579
592
  type: "flock-doc-update",
580
593
  flockDocId,
581
- update: require_meta_snapshot.encodeMetaJsonUpdate(bundle)
594
+ update
582
595
  });
583
596
  await this.materializeExistence(flockDocId, flock);
584
597
  this.clearDirtyIfStable(flockDocId, nextVersion, flock);
@@ -586,11 +599,36 @@ var FlockDocManager = class {
586
599
  this.persistedVersions.set(flockDocId, previousVersion);
587
600
  throw error;
588
601
  }
602
+ this.scheduleFlockDocCompaction(flockDocId, flock, update.byteLength);
603
+ }
604
+ scheduleFlockDocCompaction(flockDocId, flock, updateBytes) {
605
+ if (this.compactionByteThreshold === 0 || !this.storage?.compactFlockDoc) return;
606
+ const total = (this.updateBytesSinceCompaction.get(flockDocId) ?? 0) + updateBytes;
607
+ this.updateBytesSinceCompaction.set(flockDocId, total);
608
+ if (total < this.compactionByteThreshold) return;
609
+ this.queueFlockDocCompaction(flockDocId, flock);
610
+ }
611
+ queueFlockDocCompaction(flockDocId, flock) {
612
+ if (this.queuedCompactions.has(flockDocId)) return;
613
+ const storage = this.storage;
614
+ if (!storage?.compactFlockDoc) return;
615
+ this.queuedCompactions.add(flockDocId);
616
+ this.enqueueOperation(async () => {
617
+ try {
618
+ await yieldToMacroTask();
619
+ if (this.flocks.get(flockDocId) !== flock) return;
620
+ await storage.compactFlockDoc?.(flockDocId);
621
+ if (this.flocks.has(flockDocId)) this.updateBytesSinceCompaction.set(flockDocId, 0);
622
+ else this.updateBytesSinceCompaction.delete(flockDocId);
623
+ } finally {
624
+ this.queuedCompactions.delete(flockDocId);
625
+ }
626
+ }).catch(require_logging.logAsyncError(`Flock document "${flockDocId}" compaction`));
589
627
  }
590
628
  async materializeExistence(flockDocId, flock) {
591
629
  if (this.existenceMaterialized.has(flockDocId)) return;
592
630
  if (!this.docsWithVisibleContent.has(flockDocId)) {
593
- if (!hasVisibleRows(flock)) return;
631
+ if (!require_flock_snapshot.hasVisibleRows(flock)) return;
594
632
  this.docsWithVisibleContent.add(flockDocId);
595
633
  }
596
634
  await this.ensureFlockDocExists(flockDocId, this.computeExpiresAtMs(flockDocId));
@@ -607,7 +645,7 @@ var FlockDocManager = class {
607
645
  onFlockDocEvent(flockDocId) {
608
646
  const flock = this.flocks.get(flockDocId);
609
647
  if (!flock) return;
610
- if (!this.docsWithVisibleContent.has(flockDocId) && hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
648
+ if (!this.docsWithVisibleContent.has(flockDocId) && require_flock_snapshot.hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
611
649
  this.dirtyFlockDocs.add(flockDocId);
612
650
  this.scheduleDirtyFlush();
613
651
  }
@@ -1507,7 +1545,7 @@ var SyncRunner = class {
1507
1545
  } finally {
1508
1546
  this.eventBus.popEventBy();
1509
1547
  }
1510
- await this.flockDocManager.persistFlockDoc(flockDocId, flock);
1548
+ await this.flockDocManager.persistFlockDocChanges(flockDocId, flock);
1511
1549
  }
1512
1550
  }
1513
1551
  async buildSyncPlan(options, buildOptions = {}) {
@@ -1855,7 +1893,7 @@ var MetaPersister = class {
1855
1893
  return;
1856
1894
  }
1857
1895
  if (!previousVersion || this.isEmptyVersion(previousVersion)) {
1858
- const snapshot = flock.exportFile();
1896
+ const snapshot = await require_flock_snapshot.exportFlockFileSnapshot(flock);
1859
1897
  await this.storage.save({
1860
1898
  type: "meta-snapshot",
1861
1899
  snapshot
@@ -1863,7 +1901,7 @@ var MetaPersister = class {
1863
1901
  this.incrementalUpdatesSinceCompaction = 0;
1864
1902
  this.incrementalUpdateBytesSinceCompaction = 0;
1865
1903
  } else {
1866
- const update = require_meta_snapshot.encodeMetaJsonUpdate(flock.exportJson(previousVersion));
1904
+ const update = require_flock_snapshot.encodeMetaJsonUpdate(flock.exportJson(previousVersion));
1867
1905
  await this.storage.save({
1868
1906
  type: "meta-update",
1869
1907
  update
@@ -1987,7 +2025,8 @@ var LoroRepo = class LoroRepo {
1987
2025
  storage: this.storage,
1988
2026
  ensureFlockDocExists: (flockDocId, expiresAtMs) => this.flockDocRegistry.ensureExists(flockDocId, expiresAtMs),
1989
2027
  retentionMs: getFlockDocRetentionMs,
1990
- persistDebounceMs: options.docPersistDebounceMs
2028
+ persistDebounceMs: options.docPersistDebounceMs,
2029
+ compactionByteThreshold: options.flockDocCompactionByteThreshold
1991
2030
  });
1992
2031
  if (this.storage) this.metaPersister = new MetaPersister({
1993
2032
  getMetaFlock,