loro-repo 0.18.3 → 0.18.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/dist/{meta-snapshot.cjs → flock-snapshot.cjs} +32 -1
- package/dist/flock-snapshot.cjs.map +1 -0
- package/dist/{meta-snapshot.js → flock-snapshot.js} +21 -2
- package/dist/flock-snapshot.js.map +1 -0
- package/dist/index.cjs +72 -33
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +68 -29
- package/dist/index.js.map +1 -1
- package/dist/storage/filesystem.cjs +8 -5
- package/dist/storage/filesystem.cjs.map +1 -1
- package/dist/storage/filesystem.d.cts +1 -0
- package/dist/storage/filesystem.d.ts +1 -0
- package/dist/storage/filesystem.js +6 -3
- package/dist/storage/filesystem.js.map +1 -1
- package/dist/storage/indexeddb.cjs +102 -50
- package/dist/storage/indexeddb.cjs.map +1 -1
- package/dist/storage/indexeddb.d.cts +4 -1
- package/dist/storage/indexeddb.d.ts +4 -1
- package/dist/storage/indexeddb.js +100 -48
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/sqlite.cjs +72 -45
- package/dist/storage/sqlite.cjs.map +1 -1
- package/dist/storage/sqlite.d.cts +3 -0
- package/dist/storage/sqlite.d.ts +3 -0
- package/dist/storage/sqlite.js +69 -42
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/types.d.cts +13 -0
- package/dist/types.d.ts +13 -0
- package/package.json +4 -3
- package/dist/meta-snapshot.cjs.map +0 -1
- package/dist/meta-snapshot.js.map +0 -1
package/README.md
CHANGED
|
@@ -134,14 +134,35 @@ The schema (created on first use) is:
|
|
|
134
134
|
```
|
|
135
135
|
docs (doc_id PK, snapshot, updated_at, snapshot_digest)
|
|
136
136
|
doc_updates (id, doc_id, update_data, created_at)
|
|
137
|
+
flock_docs (flock_doc_id PK, snapshot, updated_at, snapshot_digest)
|
|
138
|
+
flock_doc_updates (id, flock_doc_id, update_data, created_at)
|
|
137
139
|
meta_snapshot (id=1, snapshot, updated_at)
|
|
138
140
|
meta_updates (id, update_data, created_at)
|
|
139
141
|
remote_cursors (stream_url PK, next_offset, server_lower_bound_version, updated_at_ms)
|
|
140
142
|
```
|
|
141
143
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
144
|
+
Post-sync Flock document persistence is version-vector based: no-op syncs do
|
|
145
|
+
not write, and changed state is appended as a JSON delta rather than a full
|
|
146
|
+
Flock file. Local changes that arrive while sync persistence is in flight are
|
|
147
|
+
queued behind that write and flushed from the last durable version. This keeps
|
|
148
|
+
storage growth proportional to new CRDT operations while preserving restart
|
|
149
|
+
convergence. Once a cached Flock document has accumulated 102400 persisted
|
|
150
|
+
delta bytes, the repo asks adapters that support compaction to consolidate
|
|
151
|
+
those updates. Configure this bound with
|
|
152
|
+
`flockDocCompactionByteThreshold` or set it to `0` to disable compaction. The
|
|
153
|
+
threshold-crossing delta save queues a coalesced per-document compaction rather
|
|
154
|
+
than awaiting the full snapshot export, so that sync can return once its delta
|
|
155
|
+
is durable. The adapter compaction starts on a later event-loop turn while
|
|
156
|
+
remaining serialized with later document persistence, and `flush()` /
|
|
157
|
+
`destroy()` drain the queued work.
|
|
158
|
+
|
|
159
|
+
`loadDoc` / `loadFlockDoc` / `loadMeta` opportunistically compact replayed
|
|
160
|
+
updates into a fresh snapshot inside a single transaction, mirroring the
|
|
161
|
+
filesystem adapter's behaviour but without any file churn. Existing databases
|
|
162
|
+
need no migration for delta-aware post-sync persistence; previously amplified
|
|
163
|
+
rows are compacted the next time that Flock document is loaded. Compaction is
|
|
164
|
+
driven by actual delta volume, not by time or sync count, so no-op syncs never
|
|
165
|
+
cause full snapshot exports.
|
|
145
166
|
|
|
146
167
|
### Backups in WAL mode
|
|
147
168
|
|
|
@@ -81,6 +81,25 @@ function mergeMetaSnapshot(flock, bytes) {
|
|
|
81
81
|
};
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
+
//#endregion
|
|
85
|
+
//#region src/internal/flock-snapshot.ts
|
|
86
|
+
const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;
|
|
87
|
+
async function exportFlockFileSnapshot(flock, options = {}) {
|
|
88
|
+
const exportIncremental = flock.exportFileV2Incremental;
|
|
89
|
+
if (typeof exportIncremental === "function") return exportIncremental.call(flock, {
|
|
90
|
+
budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,
|
|
91
|
+
signal: options.signal
|
|
92
|
+
});
|
|
93
|
+
return flock.exportFile();
|
|
94
|
+
}
|
|
95
|
+
function hasVisibleRows(flock) {
|
|
96
|
+
const isEmpty = flock.isEmpty;
|
|
97
|
+
if (typeof isEmpty === "function") return !isEmpty.call(flock);
|
|
98
|
+
const rows = flock.scan({});
|
|
99
|
+
for (const row of rows) if (row.value !== void 0) return true;
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
|
|
84
103
|
//#endregion
|
|
85
104
|
Object.defineProperty(exports, 'encodeMetaJsonUpdate', {
|
|
86
105
|
enumerable: true,
|
|
@@ -88,6 +107,18 @@ Object.defineProperty(exports, 'encodeMetaJsonUpdate', {
|
|
|
88
107
|
return encodeMetaJsonUpdate;
|
|
89
108
|
}
|
|
90
109
|
});
|
|
110
|
+
Object.defineProperty(exports, 'exportFlockFileSnapshot', {
|
|
111
|
+
enumerable: true,
|
|
112
|
+
get: function () {
|
|
113
|
+
return exportFlockFileSnapshot;
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
Object.defineProperty(exports, 'hasVisibleRows', {
|
|
117
|
+
enumerable: true,
|
|
118
|
+
get: function () {
|
|
119
|
+
return hasVisibleRows;
|
|
120
|
+
}
|
|
121
|
+
});
|
|
91
122
|
Object.defineProperty(exports, 'hydrateMetaSnapshots', {
|
|
92
123
|
enumerable: true,
|
|
93
124
|
get: function () {
|
|
@@ -100,4 +131,4 @@ Object.defineProperty(exports, 'isFlockFile', {
|
|
|
100
131
|
return isFlockFile;
|
|
101
132
|
}
|
|
102
133
|
});
|
|
103
|
-
//# sourceMappingURL=
|
|
134
|
+
//# sourceMappingURL=flock-snapshot.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flock-snapshot.cjs","names":["flock: Flock | undefined","Flock"],"sources":["../src/storage/meta-snapshot.ts","../src/internal/flock-snapshot.ts"],"sourcesContent":["import { Flock, type ExportBundle } from \"@loro-dev/flock-wasm\";\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\nconst FLOCK_FILE_MAGIC_LENGTH = 4;\n// Flock file snapshots are prefixed with a 4-byte magic header. The legacy v1\n// B+Tree page format uses \"FLOK\"; flock-wasm >=0.4.0 writes the canonical v2\n// format with \"FLK2\". `Flock.fromFile`/`importFile` read both, so we just need\n// to recognize either magic to distinguish a binary snapshot from a JSON bundle.\nconst FLOCK_FILE_MAGIC_V1 = [0x46, 0x4c, 0x4f, 0x4b]; // \"FLOK\"\nconst FLOCK_FILE_MAGIC_V2 = [0x46, 0x4c, 0x4b, 0x32]; // \"FLK2\"\n\ntype SnapshotFormat = \"file\" | \"json\";\n\ninterface MergeResult {\n flock: Flock;\n format: SnapshotFormat;\n}\n\nexport interface HydrateMetaSnapshotResult {\n flock: Flock | undefined;\n loadedJsonUpdate: boolean;\n}\n\nfunction matchesMagic(bytes: Uint8Array, magic: readonly number[]): boolean {\n for (let i = 0; i < magic.length; i += 1) {\n if (bytes[i] !== magic[i]) return false;\n }\n return true;\n}\n\nexport function isFlockFile(bytes: Uint8Array): boolean {\n return (\n bytes.length >= FLOCK_FILE_MAGIC_LENGTH &&\n (matchesMagic(bytes, FLOCK_FILE_MAGIC_V1) ||\n matchesMagic(bytes, FLOCK_FILE_MAGIC_V2))\n );\n}\n\nexport function encodeMetaJsonUpdate(bundle: ExportBundle): Uint8Array {\n return textEncoder.encode(JSON.stringify(bundle));\n}\n\nexport function hydrateMetaSnapshots(\n legacy: Uint8Array | undefined,\n updates: readonly Uint8Array[],\n): HydrateMetaSnapshotResult {\n let loadedJsonUpdate = false;\n let baseUpdateIndex = -1;\n let flock: Flock | undefined;\n\n if (legacy) {\n const merged = mergeMetaSnapshot(flock, legacy);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n } else {\n baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));\n if (baseUpdateIndex >= 0) {\n flock = Flock.fromFile(updates[baseUpdateIndex] as Uint8Array);\n }\n }\n\n for (let index = 0; index < updates.length; index += 1) {\n if (index === baseUpdateIndex) continue;\n const merged = mergeMetaSnapshot(flock, updates[index] as Uint8Array);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n }\n\n return { flock, loadedJsonUpdate };\n}\n\nfunction mergeMetaSnapshot(\n flock: Flock | undefined,\n bytes: Uint8Array,\n): MergeResult {\n if (isFlockFile(bytes)) {\n if (flock) {\n flock.importFile(bytes);\n return { flock, format: \"file\" };\n }\n return { flock: Flock.fromFile(bytes), format: \"file\" };\n }\n\n const bundle = JSON.parse(textDecoder.decode(bytes)) as ExportBundle;\n if (flock) {\n flock.importJson(bundle);\n return { flock, format: \"json\" };\n }\n const hydrated = new Flock();\n hydrated.importJson(bundle);\n return { flock: hydrated, format: \"json\" };\n}\n","import type { Flock } from \"@loro-dev/flock-wasm\";\n\nexport const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;\n\nexport interface ExportFlockFileSnapshotOptions {\n readonly budgetMs?: number;\n readonly signal?: AbortSignal;\n}\n\ntype FlockWithIncrementalFileExport = Flock & {\n exportFileV2Incremental?: (\n options?: ExportFlockFileSnapshotOptions,\n ) => Promise<Uint8Array>;\n};\n\ntype FlockWithIsEmpty = Flock & {\n isEmpty?: () => boolean;\n};\n\nexport async function exportFlockFileSnapshot(\n flock: Flock,\n options: ExportFlockFileSnapshotOptions = {},\n): Promise<Uint8Array> {\n const exportIncremental = (flock as FlockWithIncrementalFileExport)\n .exportFileV2Incremental;\n if (typeof exportIncremental === \"function\") {\n return exportIncremental.call(flock, {\n budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,\n signal: options.signal,\n });\n }\n return flock.exportFile();\n}\n\nexport function hasVisibleRows(flock: Flock): boolean {\n const isEmpty = (flock as FlockWithIsEmpty).isEmpty;\n if (typeof isEmpty === \"function\") {\n return !isEmpty.call(flock);\n }\n\n const rows = flock.scan({});\n for (const row of rows) {\n if (row.value !== undefined) return true;\n }\n return false;\n}\n"],"mappings":";;;;;AAEA,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,0BAA0B;AAKhC,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AACpD,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AAcpD,SAAS,aAAa,OAAmB,OAAmC;AAC1E,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,EACrC,KAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAEpC,QAAO;;AAGT,SAAgB,YAAY,OAA4B;AACtD,QACE,MAAM,UAAU,4BACf,aAAa,OAAO,oBAAoB,IACvC,aAAa,OAAO,oBAAoB;;AAI9C,SAAgB,qBAAqB,QAAkC;AACrE,QAAO,YAAY,OAAO,KAAK,UAAU,OAAO,CAAC;;AAGnD,SAAgB,qBACd,QACA,SAC2B;CAC3B,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,IAAIA;AAEJ,KAAI,QAAQ;EACV,MAAM,SAAS,kBAAkB,OAAO,OAAO;AAC/C,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;QAEhB;AACL,oBAAkB,QAAQ,WAAW,UAAU,YAAY,MAAM,CAAC;AAClE,MAAI,mBAAmB,EACrB,SAAQC,4BAAM,SAAS,QAAQ,iBAA+B;;AAIlE,MAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;AACtD,MAAI,UAAU,gBAAiB;EAC/B,MAAM,SAAS,kBAAkB,OAAO,QAAQ,OAAqB;AACrE,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;;AAIvB,QAAO;EAAE;EAAO;EAAkB;;AAGpC,SAAS,kBACP,OACA,OACa;AACb,KAAI,YAAY,MAAM,EAAE;AACtB,MAAI,OAAO;AACT,SAAM,WAAW,MAAM;AACvB,UAAO;IAAE;IAAO,QAAQ;IAAQ;;AAElC,SAAO;GAAE,OAAOA,4BAAM,SAAS,MAAM;GAAE,QAAQ;GAAQ;;CAGzD,MAAM,SAAS,KAAK,MAAM,YAAY,OAAO,MAAM,CAAC;AACpD,KAAI,OAAO;AACT,QAAM,WAAW,OAAO;AACxB,SAAO;GAAE;GAAO,QAAQ;GAAQ;;CAElC,MAAM,WAAW,IAAIA,6BAAO;AAC5B,UAAS,WAAW,OAAO;AAC3B,QAAO;EAAE,OAAO;EAAU,QAAQ;EAAQ;;;;;AC7F5C,MAAa,sCAAsC;AAiBnD,eAAsB,wBACpB,OACA,UAA0C,EAAE,EACvB;CACrB,MAAM,oBAAqB,MACxB;AACH,KAAI,OAAO,sBAAsB,WAC/B,QAAO,kBAAkB,KAAK,OAAO;EACnC,UAAU,QAAQ,YAAY;EAC9B,QAAQ,QAAQ;EACjB,CAAC;AAEJ,QAAO,MAAM,YAAY;;AAG3B,SAAgB,eAAe,OAAuB;CACpD,MAAM,UAAW,MAA2B;AAC5C,KAAI,OAAO,YAAY,WACrB,QAAO,CAAC,QAAQ,KAAK,MAAM;CAG7B,MAAM,OAAO,MAAM,KAAK,EAAE,CAAC;AAC3B,MAAK,MAAM,OAAO,KAChB,KAAI,IAAI,UAAU,OAAW,QAAO;AAEtC,QAAO"}
|
|
@@ -80,5 +80,24 @@ function mergeMetaSnapshot(flock, bytes) {
|
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
//#endregion
|
|
83
|
-
|
|
84
|
-
|
|
83
|
+
//#region src/internal/flock-snapshot.ts
|
|
84
|
+
const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;
|
|
85
|
+
async function exportFlockFileSnapshot(flock, options = {}) {
|
|
86
|
+
const exportIncremental = flock.exportFileV2Incremental;
|
|
87
|
+
if (typeof exportIncremental === "function") return exportIncremental.call(flock, {
|
|
88
|
+
budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,
|
|
89
|
+
signal: options.signal
|
|
90
|
+
});
|
|
91
|
+
return flock.exportFile();
|
|
92
|
+
}
|
|
93
|
+
function hasVisibleRows(flock) {
|
|
94
|
+
const isEmpty = flock.isEmpty;
|
|
95
|
+
if (typeof isEmpty === "function") return !isEmpty.call(flock);
|
|
96
|
+
const rows = flock.scan({});
|
|
97
|
+
for (const row of rows) if (row.value !== void 0) return true;
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
//#endregion
|
|
102
|
+
export { isFlockFile as a, hydrateMetaSnapshots as i, hasVisibleRows as n, encodeMetaJsonUpdate as r, exportFlockFileSnapshot as t };
|
|
103
|
+
//# sourceMappingURL=flock-snapshot.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flock-snapshot.js","names":["flock: Flock | undefined"],"sources":["../src/storage/meta-snapshot.ts","../src/internal/flock-snapshot.ts"],"sourcesContent":["import { Flock, type ExportBundle } from \"@loro-dev/flock-wasm\";\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\nconst FLOCK_FILE_MAGIC_LENGTH = 4;\n// Flock file snapshots are prefixed with a 4-byte magic header. The legacy v1\n// B+Tree page format uses \"FLOK\"; flock-wasm >=0.4.0 writes the canonical v2\n// format with \"FLK2\". `Flock.fromFile`/`importFile` read both, so we just need\n// to recognize either magic to distinguish a binary snapshot from a JSON bundle.\nconst FLOCK_FILE_MAGIC_V1 = [0x46, 0x4c, 0x4f, 0x4b]; // \"FLOK\"\nconst FLOCK_FILE_MAGIC_V2 = [0x46, 0x4c, 0x4b, 0x32]; // \"FLK2\"\n\ntype SnapshotFormat = \"file\" | \"json\";\n\ninterface MergeResult {\n flock: Flock;\n format: SnapshotFormat;\n}\n\nexport interface HydrateMetaSnapshotResult {\n flock: Flock | undefined;\n loadedJsonUpdate: boolean;\n}\n\nfunction matchesMagic(bytes: Uint8Array, magic: readonly number[]): boolean {\n for (let i = 0; i < magic.length; i += 1) {\n if (bytes[i] !== magic[i]) return false;\n }\n return true;\n}\n\nexport function isFlockFile(bytes: Uint8Array): boolean {\n return (\n bytes.length >= FLOCK_FILE_MAGIC_LENGTH &&\n (matchesMagic(bytes, FLOCK_FILE_MAGIC_V1) ||\n matchesMagic(bytes, FLOCK_FILE_MAGIC_V2))\n );\n}\n\nexport function encodeMetaJsonUpdate(bundle: ExportBundle): Uint8Array {\n return textEncoder.encode(JSON.stringify(bundle));\n}\n\nexport function hydrateMetaSnapshots(\n legacy: Uint8Array | undefined,\n updates: readonly Uint8Array[],\n): HydrateMetaSnapshotResult {\n let loadedJsonUpdate = false;\n let baseUpdateIndex = -1;\n let flock: Flock | undefined;\n\n if (legacy) {\n const merged = mergeMetaSnapshot(flock, legacy);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n } else {\n baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));\n if (baseUpdateIndex >= 0) {\n flock = Flock.fromFile(updates[baseUpdateIndex] as Uint8Array);\n }\n }\n\n for (let index = 0; index < updates.length; index += 1) {\n if (index === baseUpdateIndex) continue;\n const merged = mergeMetaSnapshot(flock, updates[index] as Uint8Array);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n }\n\n return { flock, loadedJsonUpdate };\n}\n\nfunction mergeMetaSnapshot(\n flock: Flock | undefined,\n bytes: Uint8Array,\n): MergeResult {\n if (isFlockFile(bytes)) {\n if (flock) {\n flock.importFile(bytes);\n return { flock, format: \"file\" };\n }\n return { flock: Flock.fromFile(bytes), format: \"file\" };\n }\n\n const bundle = JSON.parse(textDecoder.decode(bytes)) as ExportBundle;\n if (flock) {\n flock.importJson(bundle);\n return { flock, format: \"json\" };\n }\n const hydrated = new Flock();\n hydrated.importJson(bundle);\n return { flock: hydrated, format: \"json\" };\n}\n","import type { Flock } from \"@loro-dev/flock-wasm\";\n\nexport const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;\n\nexport interface ExportFlockFileSnapshotOptions {\n readonly budgetMs?: number;\n readonly signal?: AbortSignal;\n}\n\ntype FlockWithIncrementalFileExport = Flock & {\n exportFileV2Incremental?: (\n options?: ExportFlockFileSnapshotOptions,\n ) => Promise<Uint8Array>;\n};\n\ntype FlockWithIsEmpty = Flock & {\n isEmpty?: () => boolean;\n};\n\nexport async function exportFlockFileSnapshot(\n flock: Flock,\n options: ExportFlockFileSnapshotOptions = {},\n): Promise<Uint8Array> {\n const exportIncremental = (flock as FlockWithIncrementalFileExport)\n .exportFileV2Incremental;\n if (typeof exportIncremental === \"function\") {\n return exportIncremental.call(flock, {\n budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,\n signal: options.signal,\n });\n }\n return flock.exportFile();\n}\n\nexport function hasVisibleRows(flock: Flock): boolean {\n const isEmpty = (flock as FlockWithIsEmpty).isEmpty;\n if (typeof isEmpty === \"function\") {\n return !isEmpty.call(flock);\n }\n\n const rows = flock.scan({});\n for (const row of rows) {\n if (row.value !== undefined) return true;\n }\n return false;\n}\n"],"mappings":";;;AAEA,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,0BAA0B;AAKhC,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AACpD,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AAcpD,SAAS,aAAa,OAAmB,OAAmC;AAC1E,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,EACrC,KAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAEpC,QAAO;;AAGT,SAAgB,YAAY,OAA4B;AACtD,QACE,MAAM,UAAU,4BACf,aAAa,OAAO,oBAAoB,IACvC,aAAa,OAAO,oBAAoB;;AAI9C,SAAgB,qBAAqB,QAAkC;AACrE,QAAO,YAAY,OAAO,KAAK,UAAU,OAAO,CAAC;;AAGnD,SAAgB,qBACd,QACA,SAC2B;CAC3B,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,IAAIA;AAEJ,KAAI,QAAQ;EACV,MAAM,SAAS,kBAAkB,OAAO,OAAO;AAC/C,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;QAEhB;AACL,oBAAkB,QAAQ,WAAW,UAAU,YAAY,MAAM,CAAC;AAClE,MAAI,mBAAmB,EACrB,SAAQ,MAAM,SAAS,QAAQ,iBAA+B;;AAIlE,MAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;AACtD,MAAI,UAAU,gBAAiB;EAC/B,MAAM,SAAS,kBAAkB,OAAO,QAAQ,OAAqB;AACrE,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;;AAIvB,QAAO;EAAE;EAAO;EAAkB;;AAGpC,SAAS,kBACP,OACA,OACa;AACb,KAAI,YAAY,MAAM,EAAE;AACtB,MAAI,OAAO;AACT,SAAM,WAAW,MAAM;AACvB,UAAO;IAAE;IAAO,QAAQ;IAAQ;;AAElC,SAAO;GAAE,OAAO,MAAM,SAAS,MAAM;GAAE,QAAQ;GAAQ;;CAGzD,MAAM,SAAS,KAAK,MAAM,YAAY,OAAO,MAAM,CAAC;AACpD,KAAI,OAAO;AACT,QAAM,WAAW,OAAO;AACxB,SAAO;GAAE;GAAO,QAAQ;GAAQ;;CAElC,MAAM,WAAW,IAAI,OAAO;AAC5B,UAAS,WAAW,OAAO;AAC3B,QAAO;EAAE,OAAO;EAAU,QAAQ;EAAQ;;;;;AC7F5C,MAAa,sCAAsC;AAiBnD,eAAsB,wBACpB,OACA,UAA0C,EAAE,EACvB;CACrB,MAAM,oBAAqB,MACxB;AACH,KAAI,OAAO,sBAAsB,WAC/B,QAAO,kBAAkB,KAAK,OAAO;EACnC,UAAU,QAAQ,YAAY;EAC9B,QAAQ,QAAQ;EACjB,CAAC;AAEJ,QAAO,MAAM,YAAY;;AAG3B,SAAgB,eAAe,OAAuB;CACpD,MAAM,UAAW,MAA2B;AAC5C,KAAI,OAAO,YAAY,WACrB,QAAO,CAAC,QAAQ,KAAK,MAAM;CAG7B,MAAM,OAAO,MAAM,KAAK,EAAE,CAAC;AAC3B,MAAK,MAAM,OAAO,KAChB,KAAI,IAAI,UAAU,OAAW,QAAO;AAEtC,QAAO"}
|
package/dist/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const require_chunk = require('./chunk.cjs');
|
|
2
2
|
const require_logging = require('./logging.cjs');
|
|
3
|
-
const
|
|
3
|
+
const require_flock_snapshot = require('./flock-snapshot.cjs');
|
|
4
4
|
let __loro_dev_flock_wasm = require("@loro-dev/flock-wasm");
|
|
5
5
|
__loro_dev_flock_wasm = require_chunk.__toESM(__loro_dev_flock_wasm);
|
|
6
6
|
let loro_crdt = require("loro-crdt");
|
|
@@ -365,6 +365,7 @@ function yieldToMacroTask() {
|
|
|
365
365
|
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
366
366
|
}
|
|
367
367
|
const YIELD_BUDGET_MS = 5;
|
|
368
|
+
const DEFAULT_COMPACTION_BYTE_THRESHOLD = 100 * 1024;
|
|
368
369
|
const nowMs = typeof globalThis.performance?.now === "function" ? globalThis.performance.now.bind(globalThis.performance) : Date.now;
|
|
369
370
|
function stableVersion(vector) {
|
|
370
371
|
const sorted = {};
|
|
@@ -377,11 +378,6 @@ function versionEquals(a, b) {
|
|
|
377
378
|
function hasEmptyVersion(vector) {
|
|
378
379
|
return Object.keys(vector).length === 0;
|
|
379
380
|
}
|
|
380
|
-
function hasVisibleRows(flock) {
|
|
381
|
-
const rows = flock.scan({});
|
|
382
|
-
for (const row of rows) if (row.value !== void 0) return true;
|
|
383
|
-
return false;
|
|
384
|
-
}
|
|
385
381
|
async function exportJsonUpdate(flock, from) {
|
|
386
382
|
if (hasEmptyVersion(from)) return flock.exportJson();
|
|
387
383
|
return flock.exportJson({ from });
|
|
@@ -394,6 +390,7 @@ var FlockDocManager = class {
|
|
|
394
390
|
ensureFlockDocExists;
|
|
395
391
|
retentionMs;
|
|
396
392
|
persistDebounceMs;
|
|
393
|
+
compactionByteThreshold;
|
|
397
394
|
flocks = /* @__PURE__ */ new Map();
|
|
398
395
|
subscriptions = /* @__PURE__ */ new Map();
|
|
399
396
|
persistedVersions = /* @__PURE__ */ new Map();
|
|
@@ -401,6 +398,8 @@ var FlockDocManager = class {
|
|
|
401
398
|
dirtyFlockDocs = /* @__PURE__ */ new Set();
|
|
402
399
|
existenceMaterialized = /* @__PURE__ */ new Set();
|
|
403
400
|
docsWithVisibleContent = /* @__PURE__ */ new Set();
|
|
401
|
+
updateBytesSinceCompaction = /* @__PURE__ */ new Map();
|
|
402
|
+
queuedCompactions = /* @__PURE__ */ new Set();
|
|
404
403
|
debounceTimer;
|
|
405
404
|
flushScheduled = false;
|
|
406
405
|
operationPromise = Promise.resolve();
|
|
@@ -411,15 +410,17 @@ var FlockDocManager = class {
|
|
|
411
410
|
this.retentionMs = options.retentionMs;
|
|
412
411
|
const configured = options.persistDebounceMs;
|
|
413
412
|
this.persistDebounceMs = typeof configured === "number" && Number.isFinite(configured) && configured >= 0 ? configured : 0;
|
|
413
|
+
const configuredCompactionThreshold = options.compactionByteThreshold;
|
|
414
|
+
this.compactionByteThreshold = typeof configuredCompactionThreshold === "number" && Number.isFinite(configuredCompactionThreshold) && configuredCompactionThreshold >= 0 ? Math.floor(configuredCompactionThreshold) : DEFAULT_COMPACTION_BYTE_THRESHOLD;
|
|
414
415
|
}
|
|
415
416
|
async openFlockDoc(flockDocId) {
|
|
416
417
|
return this.ensureFlockDoc(flockDocId);
|
|
417
418
|
}
|
|
418
419
|
async openDetachedFlockDoc(flockDocId) {
|
|
419
420
|
const cached = this.flocks.get(flockDocId);
|
|
420
|
-
if (cached) return __loro_dev_flock_wasm.Flock.fromFile(
|
|
421
|
+
if (cached) return __loro_dev_flock_wasm.Flock.fromFile(await require_flock_snapshot.exportFlockFileSnapshot(cached));
|
|
421
422
|
const stored = await this.storage?.loadFlockDoc?.(flockDocId);
|
|
422
|
-
if (stored) return __loro_dev_flock_wasm.Flock.fromFile(
|
|
423
|
+
if (stored) return __loro_dev_flock_wasm.Flock.fromFile(await require_flock_snapshot.exportFlockFileSnapshot(stored));
|
|
423
424
|
return new __loro_dev_flock_wasm.Flock();
|
|
424
425
|
}
|
|
425
426
|
async ensureFlockDoc(flockDocId) {
|
|
@@ -441,16 +442,15 @@ var FlockDocManager = class {
|
|
|
441
442
|
}
|
|
442
443
|
async persistFlockDoc(flockDocId, flock) {
|
|
443
444
|
const previousVersion = this.persistedVersions.get(flockDocId);
|
|
444
|
-
const snapshot = flock.exportFile();
|
|
445
|
-
const nextVersion = flock.version();
|
|
446
445
|
const versionAtCapture = flock.version();
|
|
447
446
|
if (!this.storage) {
|
|
448
|
-
this.persistedVersions.set(flockDocId,
|
|
447
|
+
this.persistedVersions.set(flockDocId, versionAtCapture);
|
|
449
448
|
await this.materializeExistence(flockDocId, flock);
|
|
450
449
|
this.clearDirtyIfStable(flockDocId, versionAtCapture, flock);
|
|
451
450
|
return;
|
|
452
451
|
}
|
|
453
|
-
|
|
452
|
+
const snapshot = await require_flock_snapshot.exportFlockFileSnapshot(flock);
|
|
453
|
+
this.persistedVersions.set(flockDocId, versionAtCapture);
|
|
454
454
|
try {
|
|
455
455
|
await this.storage.save({
|
|
456
456
|
type: "flock-doc-snapshot",
|
|
@@ -465,16 +465,26 @@ var FlockDocManager = class {
|
|
|
465
465
|
throw error;
|
|
466
466
|
}
|
|
467
467
|
}
|
|
468
|
+
async persistFlockDocChanges(flockDocId, flock) {
|
|
469
|
+
await this.enqueueOperation(async () => {
|
|
470
|
+
if (this.flocks.get(flockDocId) !== flock) return;
|
|
471
|
+
await this.persistFlockDocUpdate(flockDocId, flock);
|
|
472
|
+
});
|
|
473
|
+
}
|
|
468
474
|
async unloadFlockDoc(flockDocId) {
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
475
|
+
await this.enqueueOperation(async () => {
|
|
476
|
+
const flock = this.flocks.get(flockDocId);
|
|
477
|
+
if (!flock) return;
|
|
478
|
+
await this.persistFlockDocUpdate(flockDocId, flock);
|
|
479
|
+
this.subscriptions.get(flockDocId)?.();
|
|
480
|
+
this.subscriptions.delete(flockDocId);
|
|
481
|
+
this.flocks.delete(flockDocId);
|
|
482
|
+
this.persistedVersions.delete(flockDocId);
|
|
483
|
+
this.dirtyFlockDocs.delete(flockDocId);
|
|
484
|
+
this.existenceMaterialized.delete(flockDocId);
|
|
485
|
+
this.docsWithVisibleContent.delete(flockDocId);
|
|
486
|
+
this.updateBytesSinceCompaction.delete(flockDocId);
|
|
487
|
+
});
|
|
478
488
|
}
|
|
479
489
|
async dropFlockDoc(flockDocId) {
|
|
480
490
|
await this.enqueueOperation(async () => {
|
|
@@ -485,6 +495,7 @@ var FlockDocManager = class {
|
|
|
485
495
|
this.dirtyFlockDocs.delete(flockDocId);
|
|
486
496
|
this.existenceMaterialized.delete(flockDocId);
|
|
487
497
|
this.docsWithVisibleContent.delete(flockDocId);
|
|
498
|
+
this.updateBytesSinceCompaction.delete(flockDocId);
|
|
488
499
|
await this.storage?.deleteFlockDoc?.(flockDocId);
|
|
489
500
|
});
|
|
490
501
|
}
|
|
@@ -493,6 +504,7 @@ var FlockDocManager = class {
|
|
|
493
504
|
this.flushScheduled = false;
|
|
494
505
|
while (true) {
|
|
495
506
|
await this.enqueueOperation(() => this.drainDirtyFlockDocs());
|
|
507
|
+
await this.operationPromise;
|
|
496
508
|
if (this.dirtyFlockDocs.size === 0 && !this.flushScheduled) break;
|
|
497
509
|
this.flushScheduled = false;
|
|
498
510
|
}
|
|
@@ -513,6 +525,8 @@ var FlockDocManager = class {
|
|
|
513
525
|
this.dirtyFlockDocs.clear();
|
|
514
526
|
this.existenceMaterialized.clear();
|
|
515
527
|
this.docsWithVisibleContent.clear();
|
|
528
|
+
this.updateBytesSinceCompaction.clear();
|
|
529
|
+
this.queuedCompactions.clear();
|
|
516
530
|
}
|
|
517
531
|
}
|
|
518
532
|
async loadOrCreateFlockDoc(flockDocId) {
|
|
@@ -530,7 +544,7 @@ var FlockDocManager = class {
|
|
|
530
544
|
registerFlockDoc(flockDocId, flock) {
|
|
531
545
|
this.flocks.set(flockDocId, flock);
|
|
532
546
|
this.persistedVersions.set(flockDocId, flock.version());
|
|
533
|
-
if (hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
|
|
547
|
+
if (require_flock_snapshot.hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
|
|
534
548
|
this.ensureSubscription(flockDocId, flock);
|
|
535
549
|
}
|
|
536
550
|
ensureSubscription(flockDocId, flock) {
|
|
@@ -557,28 +571,27 @@ var FlockDocManager = class {
|
|
|
557
571
|
}
|
|
558
572
|
if (!previousVersion) {
|
|
559
573
|
await this.persistFlockDoc(flockDocId, flock);
|
|
560
|
-
this.persistedVersions.set(flockDocId, nextVersion);
|
|
561
|
-
this.clearDirtyIfStable(flockDocId, nextVersion, flock);
|
|
562
574
|
return;
|
|
563
575
|
}
|
|
564
576
|
if (versionEquals(previousVersion, nextVersion)) {
|
|
565
577
|
await this.materializeExistence(flockDocId, flock);
|
|
566
|
-
this.
|
|
578
|
+
this.clearDirtyIfStable(flockDocId, nextVersion, flock);
|
|
567
579
|
return;
|
|
568
580
|
}
|
|
569
581
|
const bundle = await exportJsonUpdate(flock, previousVersion);
|
|
570
582
|
if (!hasBundleEntries(bundle)) {
|
|
571
583
|
await this.materializeExistence(flockDocId, flock);
|
|
572
584
|
this.persistedVersions.set(flockDocId, nextVersion);
|
|
573
|
-
this.
|
|
585
|
+
this.clearDirtyIfStable(flockDocId, nextVersion, flock);
|
|
574
586
|
return;
|
|
575
587
|
}
|
|
588
|
+
const update = require_flock_snapshot.encodeMetaJsonUpdate(bundle);
|
|
576
589
|
this.persistedVersions.set(flockDocId, nextVersion);
|
|
577
590
|
try {
|
|
578
591
|
await this.storage.save({
|
|
579
592
|
type: "flock-doc-update",
|
|
580
593
|
flockDocId,
|
|
581
|
-
update
|
|
594
|
+
update
|
|
582
595
|
});
|
|
583
596
|
await this.materializeExistence(flockDocId, flock);
|
|
584
597
|
this.clearDirtyIfStable(flockDocId, nextVersion, flock);
|
|
@@ -586,11 +599,36 @@ var FlockDocManager = class {
|
|
|
586
599
|
this.persistedVersions.set(flockDocId, previousVersion);
|
|
587
600
|
throw error;
|
|
588
601
|
}
|
|
602
|
+
this.scheduleFlockDocCompaction(flockDocId, flock, update.byteLength);
|
|
603
|
+
}
|
|
604
|
+
scheduleFlockDocCompaction(flockDocId, flock, updateBytes) {
|
|
605
|
+
if (this.compactionByteThreshold === 0 || !this.storage?.compactFlockDoc) return;
|
|
606
|
+
const total = (this.updateBytesSinceCompaction.get(flockDocId) ?? 0) + updateBytes;
|
|
607
|
+
this.updateBytesSinceCompaction.set(flockDocId, total);
|
|
608
|
+
if (total < this.compactionByteThreshold) return;
|
|
609
|
+
this.queueFlockDocCompaction(flockDocId, flock);
|
|
610
|
+
}
|
|
611
|
+
queueFlockDocCompaction(flockDocId, flock) {
|
|
612
|
+
if (this.queuedCompactions.has(flockDocId)) return;
|
|
613
|
+
const storage = this.storage;
|
|
614
|
+
if (!storage?.compactFlockDoc) return;
|
|
615
|
+
this.queuedCompactions.add(flockDocId);
|
|
616
|
+
this.enqueueOperation(async () => {
|
|
617
|
+
try {
|
|
618
|
+
await yieldToMacroTask();
|
|
619
|
+
if (this.flocks.get(flockDocId) !== flock) return;
|
|
620
|
+
await storage.compactFlockDoc?.(flockDocId);
|
|
621
|
+
if (this.flocks.has(flockDocId)) this.updateBytesSinceCompaction.set(flockDocId, 0);
|
|
622
|
+
else this.updateBytesSinceCompaction.delete(flockDocId);
|
|
623
|
+
} finally {
|
|
624
|
+
this.queuedCompactions.delete(flockDocId);
|
|
625
|
+
}
|
|
626
|
+
}).catch(require_logging.logAsyncError(`Flock document "${flockDocId}" compaction`));
|
|
589
627
|
}
|
|
590
628
|
async materializeExistence(flockDocId, flock) {
|
|
591
629
|
if (this.existenceMaterialized.has(flockDocId)) return;
|
|
592
630
|
if (!this.docsWithVisibleContent.has(flockDocId)) {
|
|
593
|
-
if (!hasVisibleRows(flock)) return;
|
|
631
|
+
if (!require_flock_snapshot.hasVisibleRows(flock)) return;
|
|
594
632
|
this.docsWithVisibleContent.add(flockDocId);
|
|
595
633
|
}
|
|
596
634
|
await this.ensureFlockDocExists(flockDocId, this.computeExpiresAtMs(flockDocId));
|
|
@@ -607,7 +645,7 @@ var FlockDocManager = class {
|
|
|
607
645
|
onFlockDocEvent(flockDocId) {
|
|
608
646
|
const flock = this.flocks.get(flockDocId);
|
|
609
647
|
if (!flock) return;
|
|
610
|
-
if (!this.docsWithVisibleContent.has(flockDocId) && hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
|
|
648
|
+
if (!this.docsWithVisibleContent.has(flockDocId) && require_flock_snapshot.hasVisibleRows(flock)) this.docsWithVisibleContent.add(flockDocId);
|
|
611
649
|
this.dirtyFlockDocs.add(flockDocId);
|
|
612
650
|
this.scheduleDirtyFlush();
|
|
613
651
|
}
|
|
@@ -1507,7 +1545,7 @@ var SyncRunner = class {
|
|
|
1507
1545
|
} finally {
|
|
1508
1546
|
this.eventBus.popEventBy();
|
|
1509
1547
|
}
|
|
1510
|
-
await this.flockDocManager.
|
|
1548
|
+
await this.flockDocManager.persistFlockDocChanges(flockDocId, flock);
|
|
1511
1549
|
}
|
|
1512
1550
|
}
|
|
1513
1551
|
async buildSyncPlan(options, buildOptions = {}) {
|
|
@@ -1855,7 +1893,7 @@ var MetaPersister = class {
|
|
|
1855
1893
|
return;
|
|
1856
1894
|
}
|
|
1857
1895
|
if (!previousVersion || this.isEmptyVersion(previousVersion)) {
|
|
1858
|
-
const snapshot =
|
|
1896
|
+
const snapshot = await require_flock_snapshot.exportFlockFileSnapshot(flock);
|
|
1859
1897
|
await this.storage.save({
|
|
1860
1898
|
type: "meta-snapshot",
|
|
1861
1899
|
snapshot
|
|
@@ -1863,7 +1901,7 @@ var MetaPersister = class {
|
|
|
1863
1901
|
this.incrementalUpdatesSinceCompaction = 0;
|
|
1864
1902
|
this.incrementalUpdateBytesSinceCompaction = 0;
|
|
1865
1903
|
} else {
|
|
1866
|
-
const update =
|
|
1904
|
+
const update = require_flock_snapshot.encodeMetaJsonUpdate(flock.exportJson(previousVersion));
|
|
1867
1905
|
await this.storage.save({
|
|
1868
1906
|
type: "meta-update",
|
|
1869
1907
|
update
|
|
@@ -1987,7 +2025,8 @@ var LoroRepo = class LoroRepo {
|
|
|
1987
2025
|
storage: this.storage,
|
|
1988
2026
|
ensureFlockDocExists: (flockDocId, expiresAtMs) => this.flockDocRegistry.ensureExists(flockDocId, expiresAtMs),
|
|
1989
2027
|
retentionMs: getFlockDocRetentionMs,
|
|
1990
|
-
persistDebounceMs: options.docPersistDebounceMs
|
|
2028
|
+
persistDebounceMs: options.docPersistDebounceMs,
|
|
2029
|
+
compactionByteThreshold: options.flockDocCompactionByteThreshold
|
|
1991
2030
|
});
|
|
1992
2031
|
if (this.storage) this.metaPersister = new MetaPersister({
|
|
1993
2032
|
getMetaFlock,
|