loro-repo 0.20.3 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -23
- package/dist/flock-persistence-journal.cjs +33 -20
- package/dist/flock-persistence-journal.cjs.map +1 -1
- package/dist/flock-persistence-journal.js +33 -20
- package/dist/flock-persistence-journal.js.map +1 -1
- package/dist/flock-snapshot.cjs +19 -14
- package/dist/flock-snapshot.cjs.map +1 -1
- package/dist/flock-snapshot.js +19 -14
- package/dist/flock-snapshot.js.map +1 -1
- package/dist/index.cjs +390 -82
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +389 -81
- package/dist/index.js.map +1 -1
- package/dist/replica-checkpoints.cjs +105 -0
- package/dist/replica-checkpoints.cjs.map +1 -0
- package/dist/replica-checkpoints.js +81 -0
- package/dist/replica-checkpoints.js.map +1 -0
- package/dist/storage/indexeddb.cjs +213 -130
- package/dist/storage/indexeddb.cjs.map +1 -1
- package/dist/storage/indexeddb.d.cts +50 -8
- package/dist/storage/indexeddb.d.ts +50 -8
- package/dist/storage/indexeddb.js +213 -130
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/sqlite.cjs +224 -57
- package/dist/storage/sqlite.cjs.map +1 -1
- package/dist/storage/sqlite.d.cts +57 -0
- package/dist/storage/sqlite.d.ts +57 -0
- package/dist/storage/sqlite.js +224 -58
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/transport/streams.cjs +1 -1
- package/dist/transport/streams.cjs.map +1 -1
- package/dist/transport/streams.js +1 -1
- package/dist/transport/streams.js.map +1 -1
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +12 -0
- package/dist/types.d.ts +12 -0
- package/dist/types.js.map +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -322,9 +322,10 @@ await repo.addTransport(
|
|
|
322
322
|
```
|
|
323
323
|
|
|
324
324
|
The storage adapter restores each Meta/named Flock snapshot, update log and
|
|
325
|
-
Streams checkpoint from one capture transaction.
|
|
326
|
-
readonly transaction
|
|
327
|
-
|
|
325
|
+
Streams checkpoint from one capture transaction. Bound generations use a
|
|
326
|
+
readonly transaction. A missing generation, or one whose lineage marker is gone
|
|
327
|
+
(see below), is replaced while recapturing the resource in one readwrite
|
|
328
|
+
transaction. A live Flock only sees the cursor
|
|
328
329
|
captured with its own restored bytes, even when another tab writes a newer
|
|
329
330
|
cursor to the same database later. Strict Flock data commits precede the strict
|
|
330
331
|
checkpoint transaction, while LoroDoc body writes retain normal IndexedDB
|
|
@@ -333,6 +334,43 @@ deletion removes its checkpoint record with the data and invalidates outstanding
|
|
|
333
334
|
cursor stores. Version changes from another tab are handled by reopening the
|
|
334
335
|
database before subsequent checkpoint mutations.
|
|
335
336
|
|
|
337
|
+
**Older tabs and rollbacks.** loro-repo 0.18.0 through 0.20.2 can open the same
|
|
338
|
+
database without knowing that checkpoints exist. Their purge or TTL expiry of a
|
|
339
|
+
named Flock, and their destructive `meta-snapshot` save, remove data but leave
|
|
340
|
+
the checkpoint. Each checkpoint is therefore bound to a lineage marker stored
|
|
341
|
+
with the data: a `lineage` field on the named Flock's update-queue record, and a
|
|
342
|
+
`lineage:<metaKey>` row in `meta-updates`. Older releases ignore both when
|
|
343
|
+
reading. They drop or delete the marker whenever they purge a named Flock,
|
|
344
|
+
rewrite its update queue, or replace the metadata base. If a capture finds the
|
|
345
|
+
marker missing or foreign, it discards the cursors and rotates the generation.
|
|
346
|
+
That room bootstraps once and merges with local state. Stores still bound to
|
|
347
|
+
the old generation are rejected in every tab. Healthy captures stay read-only.
|
|
348
|
+
Upgrading from 0.20.3, whose checkpoints carry no marker, bootstraps each Meta
|
|
349
|
+
and named Flock room once. The marker prevents a skipped range; it does not
|
|
350
|
+
bring back metadata that an older tab's `meta-snapshot` removed before it was
|
|
351
|
+
sent. See `docs/flock-persistence.md` ("IndexedDB lineage markers") for the
|
|
352
|
+
format, audit and remaining risks.
|
|
353
|
+
|
|
354
|
+
**Batched strict commits.** A strict transaction waits for a disk sync, so
|
|
355
|
+
metadata and named Flock payloads share them. One `repo.flush()` writes every
|
|
356
|
+
dirty named Flock and metadata in a single strict transaction, together with
|
|
357
|
+
the automatic Flock persistence its own writes scheduled. Concurrent
|
|
358
|
+
`persistFlockDocNow`/`persistMetaNow` barriers (the Streams barriers) and
|
|
359
|
+
concurrent flushes share commits too: payloads staged while a commit is in
|
|
360
|
+
flight go into the next one. A barrier still resolves only after the commit
|
|
361
|
+
that contains every change applied before its call. If a shared commit fails,
|
|
362
|
+
each resource is retried in its own transaction, and failed payloads stay
|
|
363
|
+
queued for the next barrier. `StorageAdapter.saveMany` is the optional atomic
|
|
364
|
+
batch hook behind this; `IndexedDBStorageAdaptor` and `SqliteRepoStore`
|
|
365
|
+
implement it, and other adapters keep receiving one `save` per payload.
|
|
366
|
+
|
|
367
|
+
`new IndexedDBStorageAdaptor({ durability })` accepts `"strict"` (the default),
|
|
368
|
+
`"relaxed"` or `"default"` for metadata, named Flock and checkpoint writes.
|
|
369
|
+
Anything but `"strict"` lets a crash or power loss drop writes the repo has
|
|
370
|
+
already acknowledged, in any order, so a saved Streams cursor can outlive the
|
|
371
|
+
data it covers and the resumed stream skips that history. Relax it only when
|
|
372
|
+
no Streams cursor is persisted or such a loss is recovered by other means.
|
|
373
|
+
|
|
336
374
|
LoroDoc persistence keeps its existing storage path. Its cursor is memory-only
|
|
337
375
|
in the example above. Pass `{ documentRemoteCursorStore }` as the factory's
|
|
338
376
|
second argument when an application explicitly wants its existing LoroDoc
|
|
@@ -340,20 +378,20 @@ cursor behavior; this does not extend the Flock recovery guarantee to LoroDoc.
|
|
|
340
378
|
|
|
341
379
|
## SQLite storage (Node.js / Electron)
|
|
342
380
|
|
|
343
|
-
`SqliteRepoStore` packs repo data and cursor rows into a
|
|
344
|
-
file. Compared to `FileSystemStorageAdaptor`, it avoids
|
|
345
|
-
files for incremental updates, and it lets
|
|
346
|
-
single-row `UPDATE`s instead of rewriting a large JSON blob.
|
|
347
|
-
|
|
348
|
-
|
|
381
|
+
`SqliteRepoStore` packs repo data, Streams checkpoints and cursor rows into a
|
|
382
|
+
single SQLite database file. Compared to `FileSystemStorageAdaptor`, it avoids
|
|
383
|
+
creating many small files for incremental updates, and it lets cursor advances
|
|
384
|
+
become single-row `UPDATE`s instead of rewriting a large JSON blob. Like
|
|
385
|
+
`IndexedDBStorageAdaptor`, it implements replica-bound Streams checkpoints, so
|
|
386
|
+
it works with the no-argument `createRepoStreamsPersistence(repo)`.
|
|
349
387
|
|
|
350
388
|
A single `SqliteRepoStore` instance exposes two adapters that share one
|
|
351
389
|
database connection:
|
|
352
390
|
|
|
353
|
-
- `store.storage` — a `StorageAdapter` for doc snapshots, doc updates,
|
|
354
|
-
metadata (Flock) snapshots/updates
|
|
355
|
-
-
|
|
356
|
-
|
|
391
|
+
- `store.storage` — a `StorageAdapter` for doc snapshots, doc updates,
|
|
392
|
+
metadata (Flock) snapshots/updates, named Flock documents, and the
|
|
393
|
+
replica-bound Meta/named Flock checkpoints.
|
|
394
|
+
- `store.cursorStore` — a `RemoteCursorStore` for LoroDoc rooms.
|
|
357
395
|
|
|
358
396
|
```ts
|
|
359
397
|
import { LoroRepo } from "loro-repo";
|
|
@@ -374,9 +412,10 @@ await repo.addTransport(
|
|
|
374
412
|
new StreamsTransportAdapter({
|
|
375
413
|
bucketId: "your-bucket",
|
|
376
414
|
auth: yourAuthProvider,
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
415
|
+
persistence: createRepoStreamsPersistence(repo, {
|
|
416
|
+
// Optional: keep LoroDoc cursors durable in `remote_cursors`.
|
|
417
|
+
documentRemoteCursorStore: store.cursorStore,
|
|
418
|
+
}),
|
|
380
419
|
}),
|
|
381
420
|
);
|
|
382
421
|
|
|
@@ -385,13 +424,40 @@ await repo.destroy();
|
|
|
385
424
|
store.close();
|
|
386
425
|
```
|
|
387
426
|
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
427
|
+
Meta and each named Flock are restored from one SQLite transaction that reads
|
|
428
|
+
the base snapshot, update log and checkpoint row together. A resource that
|
|
429
|
+
already has a checkpoint generation is read in a deferred (read) transaction;
|
|
430
|
+
a legacy or empty resource is recaptured while creating a random generation in
|
|
431
|
+
one `IMMEDIATE` transaction. The returned Flock only resumes from the cursor
|
|
432
|
+
captured with its own bytes: several `SqliteRepoStore` instances or processes
|
|
433
|
+
may open the same file (each with its own in-memory replica), and a newer
|
|
434
|
+
cursor written by one of them is never adopted by a replica that loaded older
|
|
435
|
+
data. Cursor saves run in `IMMEDIATE` transactions that check the generation
|
|
436
|
+
after the transport's data barrier has committed; the bound replica updates its
|
|
437
|
+
in-memory cursor only after that commit. With `wal: true`
|
|
438
|
+
(`synchronous=NORMAL`) a power loss may drop the latest commits, but it cannot
|
|
439
|
+
keep a later cursor commit without the earlier data commit.
|
|
440
|
+
|
|
441
|
+
Deleting a named Flock (`purgeFlockDoc`, or `purgeExpiredFlockDocs` for
|
|
442
|
+
`flockDocRetentionMs`) removes its data and checkpoint in one transaction, so
|
|
443
|
+
outstanding cursor stores in every process are rejected. Delete triggers in the
|
|
444
|
+
schema drop the checkpoint as well whenever Flock data is removed by anything
|
|
445
|
+
other than compaction, including loro-repo 0.20.3 and older after a rollback.
|
|
446
|
+
Do not drop `replica_checkpoints` by hand without dropping those triggers.
|
|
447
|
+
|
|
448
|
+
**Upgrading.** The `replica_checkpoints` table and its triggers are created on
|
|
449
|
+
open. Existing Meta/named Flock rows in `remote_cursors` are deliberately not
|
|
450
|
+
copied: they were never bound to the stored data. Each Meta and named Flock
|
|
451
|
+
room therefore bootstraps once after the upgrade and merges the snapshot with
|
|
452
|
+
local state, including unsent local writes. Old rows are left in place, so a
|
|
453
|
+
rolled-back binary still resumes from them (it replays; it cannot skip). The
|
|
454
|
+
LoroDoc path is unchanged.
|
|
455
|
+
|
|
456
|
+
The deprecated two-argument form,
|
|
457
|
+
`createRepoStreamsPersistence(repo, store.cursorStore)`, still works and still
|
|
458
|
+
stores Meta/named Flock cursors in `remote_cursors` with data-before-cursor
|
|
459
|
+
ordering, but without the atomic recovery described above. Sessions that keep
|
|
460
|
+
nothing across restarts can use `persistence: { mode: "ephemeral" }`.
|
|
395
461
|
|
|
396
462
|
`SqliteRepoStoreOptions`:
|
|
397
463
|
|
|
@@ -412,8 +478,13 @@ flock_doc_updates (id, flock_doc_id, update_data, created_at)
|
|
|
412
478
|
meta_snapshot (id=1, snapshot, updated_at)
|
|
413
479
|
meta_updates (id, update_data, created_at)
|
|
414
480
|
remote_cursors (stream_url PK, next_offset, server_lower_bound_version, updated_at_ms)
|
|
481
|
+
replica_checkpoints (resource_key PK, generation, cursors, updated_at_ms)
|
|
415
482
|
```
|
|
416
483
|
|
|
484
|
+
`resource_key` is `meta` or `flock:<flockDocId>`; `cursors` is a JSON array of
|
|
485
|
+
Streams cursors keyed by stream URL. Checkpoints are never written into
|
|
486
|
+
synchronized Flock metadata.
|
|
487
|
+
|
|
417
488
|
Post-sync Flock document persistence is version-vector based: no-op syncs do
|
|
418
489
|
not write, and changed state is appended as a JSON delta rather than a full
|
|
419
490
|
Flock file. Local changes that arrive while sync persistence is in flight are
|
|
@@ -19,6 +19,14 @@ var FlockPersistenceJournal = class {
|
|
|
19
19
|
payloads = [];
|
|
20
20
|
unsubscribe;
|
|
21
21
|
remoteKeys;
|
|
22
|
+
/**
|
|
23
|
+
* An inclusive version vector observed with no Flock event since. Exporting
|
|
24
|
+
* a version vector from wasm costs time proportional to the number of
|
|
25
|
+
* peers, and Streams delivers many batches that change nothing (echoes of
|
|
26
|
+
* our own appends, replayed catch-ups). Reusing the vector read after the
|
|
27
|
+
* previous import keeps such a batch to a single read.
|
|
28
|
+
*/
|
|
29
|
+
knownVersion;
|
|
22
30
|
constructor(flock, onDirty) {
|
|
23
31
|
this.flock = flock;
|
|
24
32
|
this.onDirty = onDirty;
|
|
@@ -26,6 +34,7 @@ var FlockPersistenceJournal = class {
|
|
|
26
34
|
this.unsubscribe = flock.subscribe((batch) => {
|
|
27
35
|
const keys = batch.source !== "local" && this.remoteKeys ? this.remoteKeys : this.keys;
|
|
28
36
|
for (const event of batch.events) keys.add(JSON.stringify(event.key));
|
|
37
|
+
if (batch.events.length) this.knownVersion = void 0;
|
|
29
38
|
if (keys === this.keys && batch.events.length) this.onDirty();
|
|
30
39
|
});
|
|
31
40
|
}
|
|
@@ -37,7 +46,7 @@ var FlockPersistenceJournal = class {
|
|
|
37
46
|
let result;
|
|
38
47
|
const previousKeys = this.remoteKeys;
|
|
39
48
|
const remoteKeys = /* @__PURE__ */ new Set();
|
|
40
|
-
const previousVersion = this.flock.
|
|
49
|
+
const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();
|
|
41
50
|
this.remoteKeys = remoteKeys;
|
|
42
51
|
try {
|
|
43
52
|
result = apply();
|
|
@@ -54,7 +63,7 @@ var FlockPersistenceJournal = class {
|
|
|
54
63
|
this.retainChangedKeys(remoteKeys);
|
|
55
64
|
throw error;
|
|
56
65
|
}
|
|
57
|
-
if (remoteKeys.size > 0 || !
|
|
66
|
+
if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {
|
|
58
67
|
this.payloads.push(...copies.map((bytes) => ({
|
|
59
68
|
bytes,
|
|
60
69
|
failures: 0
|
|
@@ -63,32 +72,36 @@ var FlockPersistenceJournal = class {
|
|
|
63
72
|
}
|
|
64
73
|
return outcome;
|
|
65
74
|
}
|
|
75
|
+
versionUnchangedSince(previous) {
|
|
76
|
+
const current = this.flock.inclusiveVersion();
|
|
77
|
+
this.knownVersion = current;
|
|
78
|
+
return versionsEqual(previous, current);
|
|
79
|
+
}
|
|
66
80
|
retainChangedKeys(keys) {
|
|
67
81
|
for (const key of keys) this.keys.add(key);
|
|
68
82
|
if (keys.size) this.onDirty();
|
|
69
83
|
}
|
|
70
|
-
/**
|
|
71
|
-
|
|
84
|
+
/**
|
|
85
|
+
* Called within the owner's serialized persistence queue.
|
|
86
|
+
*
|
|
87
|
+
* Each round hands every pending payload to `write` at once, so a storage
|
|
88
|
+
* adapter can commit them in one transaction. Only the prefix `write`
|
|
89
|
+
* reports durable is acknowledged.
|
|
90
|
+
*/
|
|
91
|
+
async flush(write) {
|
|
72
92
|
this.flock.commit();
|
|
73
93
|
while (this.hasPending) {
|
|
74
94
|
if (this.keys.size) await this.stageChangedKeys();
|
|
95
|
+
const { durable, error: exactError } = await write(this.payloads.map((payload$1) => payload$1.bytes));
|
|
96
|
+
this.payloads.splice(0, durable);
|
|
97
|
+
if (exactError === void 0) continue;
|
|
75
98
|
const payload = this.payloads[0];
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
const snapshot = await require_flock_snapshot.exportFlockFileSnapshot(this.flock);
|
|
83
|
-
try {
|
|
84
|
-
await save(snapshot);
|
|
85
|
-
} catch (snapshotError) {
|
|
86
|
-
throw new AggregateError([exactError, snapshotError], "Failed to persist a Flock update and its snapshot fallback");
|
|
87
|
-
}
|
|
88
|
-
this.payloads.splice(0, coveredPayloads);
|
|
89
|
-
continue;
|
|
90
|
-
}
|
|
91
|
-
this.payloads.shift();
|
|
99
|
+
payload.failures += 1;
|
|
100
|
+
if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) throw exactError;
|
|
101
|
+
const coveredPayloads = this.payloads.length;
|
|
102
|
+
const fallback = await write([await require_flock_snapshot.exportFlockFileSnapshot(this.flock)]);
|
|
103
|
+
if (fallback.error !== void 0) throw new AggregateError([exactError, fallback.error], "Failed to persist a Flock update and its snapshot fallback");
|
|
104
|
+
this.payloads.splice(0, coveredPayloads);
|
|
92
105
|
}
|
|
93
106
|
}
|
|
94
107
|
async stageChangedKeys() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flock-persistence-journal.cjs","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","exportFlockFileSnapshot","bundle: ExportBundle","encodeMetaJsonUpdate"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n const previousVersion = this.flock.version();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes.\n if (\n remoteKeys.size > 0 ||\n !versionsEqual(previousVersion, this.flock.version())\n ) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /** Called within the owner's serialized persistence queue. */\n async flush(save: (update: Uint8Array) => Promise<void>): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const payload = this.payloads[0]!;\n try {\n await save(payload.bytes);\n } catch (exactError) {\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n try {\n await save(snapshot);\n } catch (snapshotError) {\n throw new AggregateError(\n [exactError, snapshotError],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n continue;\n }\n\n // New imports/local changes during the awaits remain behind this item.\n this.payloads.shift();\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAExC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EACpC,MAAM,kBAAkB,KAAK,MAAM,SAAS;AAC5C,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAOR,MACE,WAAW,OAAO,KAClB,CAAC,cAAc,iBAAiB,KAAK,MAAM,SAAS,CAAC,EACrD;AACA,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;CAI/B,MAAM,MAAM,MAA4D;AACtE,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,UAAU,KAAK,SAAS;AAC9B,OAAI;AACF,UAAM,KAAK,QAAQ,MAAM;YAClB,YAAY;AACnB,YAAQ,YAAY;AAIpB,QAAI,QAAQ,WAAW,gCACrB,OAAM;IAUR,MAAM,kBAAkB,KAAK,SAAS;IACtC,MAAM,WAAW,MAAMC,+CAAwB,KAAK,MAAM;AAC1D,QAAI;AACF,WAAM,KAAK,SAAS;aACb,eAAe;AACtB,WAAM,IAAI,eACR,CAAC,YAAY,cAAc,EAC3B,6DACD;;AAEH,SAAK,SAAS,OAAO,GAAG,gBAAgB;AACxC;;AAIF,QAAK,SAAS,OAAO;;;CAIzB,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAMD,+CAAwB,KAAK,MAAM,GACzCE,4CAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|
|
1
|
+
{"version":3,"file":"flock-persistence-journal.cjs","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","payload","exportFlockFileSnapshot","bundle: ExportBundle","encodeMetaJsonUpdate"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\n/**\n * Persists `updates` in order. Reports how many leading updates are durable\n * and, if not all of them are, the failure. It must not reject.\n */\nexport type JournalWrite = (\n updates: readonly Uint8Array[],\n) => Promise<{ readonly durable: number; readonly error?: unknown }>;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n /**\n * An inclusive version vector observed with no Flock event since. Exporting\n * a version vector from wasm costs time proportional to the number of\n * peers, and Streams delivers many batches that change nothing (echoes of\n * our own appends, replayed catch-ups). Reusing the vector read after the\n * previous import keeps such a batch to a single read.\n */\n private knownVersion?: VersionVector;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (batch.events.length) this.knownVersion = undefined;\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n // The inclusive vector only grows during the process lifetime, so a\n // cached vector that is stale because of an untracked import can only\n // cause a redundant save, never a skipped one.\n const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes. An import can also advance the version without\n // emitting events (an entry that loses last-writer-wins), so compare.\n if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private versionUnchangedSince(previous: VersionVector): boolean {\n const current = this.flock.inclusiveVersion();\n this.knownVersion = current;\n return versionsEqual(previous, current);\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /**\n * Called within the owner's serialized persistence queue.\n *\n * Each round hands every pending payload to `write` at once, so a storage\n * adapter can commit them in one transaction. Only the prefix `write`\n * reports durable is acknowledged.\n */\n async flush(write: JournalWrite): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const { durable, error: exactError } = await write(\n this.payloads.map((payload) => payload.bytes),\n );\n // Only this loop removes payloads, so the written ones are still the\n // queue's prefix. New imports/local changes during the write remain\n // behind them.\n this.payloads.splice(0, durable);\n if (exactError === undefined) continue;\n\n const payload = this.payloads[0]!;\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n const fallback = await write([snapshot]);\n if (fallback.error !== undefined) {\n throw new AggregateError(\n [exactError, fallback.error],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAUxC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;;;;;;;;CAQR,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,MAAM,OAAO,OAAQ,MAAK,eAAe;AAC7C,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EAIpC,MAAM,kBAAkB,KAAK,gBAAgB,KAAK,MAAM,kBAAkB;AAC1E,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAQR,MAAI,WAAW,OAAO,KAAK,CAAC,KAAK,sBAAsB,gBAAgB,EAAE;AACvE,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,sBAAsB,UAAkC;EAC9D,MAAM,UAAU,KAAK,MAAM,kBAAkB;AAC7C,OAAK,eAAe;AACpB,SAAO,cAAc,UAAU,QAAQ;;CAGzC,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;;;;;;;CAU/B,MAAM,MAAM,OAAoC;AAC9C,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,EAAE,SAAS,OAAO,eAAe,MAAM,MAC3C,KAAK,SAAS,KAAK,cAAYC,UAAQ,MAAM,CAC9C;AAID,QAAK,SAAS,OAAO,GAAG,QAAQ;AAChC,OAAI,eAAe,OAAW;GAE9B,MAAM,UAAU,KAAK,SAAS;AAC9B,WAAQ,YAAY;AAIpB,OAAI,QAAQ,WAAW,gCACrB,OAAM;GAUR,MAAM,kBAAkB,KAAK,SAAS;GAEtC,MAAM,WAAW,MAAM,MAAM,CADZ,MAAMC,+CAAwB,KAAK,MAAM,CACnB,CAAC;AACxC,OAAI,SAAS,UAAU,OACrB,OAAM,IAAI,eACR,CAAC,YAAY,SAAS,MAAM,EAC5B,6DACD;AAEH,QAAK,SAAS,OAAO,GAAG,gBAAgB;;;CAI5C,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAMD,+CAAwB,KAAK,MAAM,GACzCE,4CAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|
|
@@ -19,6 +19,14 @@ var FlockPersistenceJournal = class {
|
|
|
19
19
|
payloads = [];
|
|
20
20
|
unsubscribe;
|
|
21
21
|
remoteKeys;
|
|
22
|
+
/**
|
|
23
|
+
* An inclusive version vector observed with no Flock event since. Exporting
|
|
24
|
+
* a version vector from wasm costs time proportional to the number of
|
|
25
|
+
* peers, and Streams delivers many batches that change nothing (echoes of
|
|
26
|
+
* our own appends, replayed catch-ups). Reusing the vector read after the
|
|
27
|
+
* previous import keeps such a batch to a single read.
|
|
28
|
+
*/
|
|
29
|
+
knownVersion;
|
|
22
30
|
constructor(flock, onDirty) {
|
|
23
31
|
this.flock = flock;
|
|
24
32
|
this.onDirty = onDirty;
|
|
@@ -26,6 +34,7 @@ var FlockPersistenceJournal = class {
|
|
|
26
34
|
this.unsubscribe = flock.subscribe((batch) => {
|
|
27
35
|
const keys = batch.source !== "local" && this.remoteKeys ? this.remoteKeys : this.keys;
|
|
28
36
|
for (const event of batch.events) keys.add(JSON.stringify(event.key));
|
|
37
|
+
if (batch.events.length) this.knownVersion = void 0;
|
|
29
38
|
if (keys === this.keys && batch.events.length) this.onDirty();
|
|
30
39
|
});
|
|
31
40
|
}
|
|
@@ -37,7 +46,7 @@ var FlockPersistenceJournal = class {
|
|
|
37
46
|
let result;
|
|
38
47
|
const previousKeys = this.remoteKeys;
|
|
39
48
|
const remoteKeys = /* @__PURE__ */ new Set();
|
|
40
|
-
const previousVersion = this.flock.
|
|
49
|
+
const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();
|
|
41
50
|
this.remoteKeys = remoteKeys;
|
|
42
51
|
try {
|
|
43
52
|
result = apply();
|
|
@@ -54,7 +63,7 @@ var FlockPersistenceJournal = class {
|
|
|
54
63
|
this.retainChangedKeys(remoteKeys);
|
|
55
64
|
throw error;
|
|
56
65
|
}
|
|
57
|
-
if (remoteKeys.size > 0 || !
|
|
66
|
+
if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {
|
|
58
67
|
this.payloads.push(...copies.map((bytes) => ({
|
|
59
68
|
bytes,
|
|
60
69
|
failures: 0
|
|
@@ -63,32 +72,36 @@ var FlockPersistenceJournal = class {
|
|
|
63
72
|
}
|
|
64
73
|
return outcome;
|
|
65
74
|
}
|
|
75
|
+
versionUnchangedSince(previous) {
|
|
76
|
+
const current = this.flock.inclusiveVersion();
|
|
77
|
+
this.knownVersion = current;
|
|
78
|
+
return versionsEqual(previous, current);
|
|
79
|
+
}
|
|
66
80
|
retainChangedKeys(keys) {
|
|
67
81
|
for (const key of keys) this.keys.add(key);
|
|
68
82
|
if (keys.size) this.onDirty();
|
|
69
83
|
}
|
|
70
|
-
/**
|
|
71
|
-
|
|
84
|
+
/**
|
|
85
|
+
* Called within the owner's serialized persistence queue.
|
|
86
|
+
*
|
|
87
|
+
* Each round hands every pending payload to `write` at once, so a storage
|
|
88
|
+
* adapter can commit them in one transaction. Only the prefix `write`
|
|
89
|
+
* reports durable is acknowledged.
|
|
90
|
+
*/
|
|
91
|
+
async flush(write) {
|
|
72
92
|
this.flock.commit();
|
|
73
93
|
while (this.hasPending) {
|
|
74
94
|
if (this.keys.size) await this.stageChangedKeys();
|
|
95
|
+
const { durable, error: exactError } = await write(this.payloads.map((payload$1) => payload$1.bytes));
|
|
96
|
+
this.payloads.splice(0, durable);
|
|
97
|
+
if (exactError === void 0) continue;
|
|
75
98
|
const payload = this.payloads[0];
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
const snapshot = await exportFlockFileSnapshot(this.flock);
|
|
83
|
-
try {
|
|
84
|
-
await save(snapshot);
|
|
85
|
-
} catch (snapshotError) {
|
|
86
|
-
throw new AggregateError([exactError, snapshotError], "Failed to persist a Flock update and its snapshot fallback");
|
|
87
|
-
}
|
|
88
|
-
this.payloads.splice(0, coveredPayloads);
|
|
89
|
-
continue;
|
|
90
|
-
}
|
|
91
|
-
this.payloads.shift();
|
|
99
|
+
payload.failures += 1;
|
|
100
|
+
if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) throw exactError;
|
|
101
|
+
const coveredPayloads = this.payloads.length;
|
|
102
|
+
const fallback = await write([await exportFlockFileSnapshot(this.flock)]);
|
|
103
|
+
if (fallback.error !== void 0) throw new AggregateError([exactError, fallback.error], "Failed to persist a Flock update and its snapshot fallback");
|
|
104
|
+
this.payloads.splice(0, coveredPayloads);
|
|
92
105
|
}
|
|
93
106
|
}
|
|
94
107
|
async stageChangedKeys() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flock-persistence-journal.js","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","bundle: ExportBundle"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n const previousVersion = this.flock.version();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes.\n if (\n remoteKeys.size > 0 ||\n !versionsEqual(previousVersion, this.flock.version())\n ) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /** Called within the owner's serialized persistence queue. */\n async flush(save: (update: Uint8Array) => Promise<void>): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const payload = this.payloads[0]!;\n try {\n await save(payload.bytes);\n } catch (exactError) {\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n try {\n await save(snapshot);\n } catch (snapshotError) {\n throw new AggregateError(\n [exactError, snapshotError],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n continue;\n }\n\n // New imports/local changes during the awaits remain behind this item.\n this.payloads.shift();\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAExC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EACpC,MAAM,kBAAkB,KAAK,MAAM,SAAS;AAC5C,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAOR,MACE,WAAW,OAAO,KAClB,CAAC,cAAc,iBAAiB,KAAK,MAAM,SAAS,CAAC,EACrD;AACA,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;CAI/B,MAAM,MAAM,MAA4D;AACtE,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,UAAU,KAAK,SAAS;AAC9B,OAAI;AACF,UAAM,KAAK,QAAQ,MAAM;YAClB,YAAY;AACnB,YAAQ,YAAY;AAIpB,QAAI,QAAQ,WAAW,gCACrB,OAAM;IAUR,MAAM,kBAAkB,KAAK,SAAS;IACtC,MAAM,WAAW,MAAM,wBAAwB,KAAK,MAAM;AAC1D,QAAI;AACF,WAAM,KAAK,SAAS;aACb,eAAe;AACtB,WAAM,IAAI,eACR,CAAC,YAAY,cAAc,EAC3B,6DACD;;AAEH,SAAK,SAAS,OAAO,GAAG,gBAAgB;AACxC;;AAIF,QAAK,SAAS,OAAO;;;CAIzB,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAM,wBAAwB,KAAK,MAAM,GACzC,qBAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|
|
1
|
+
{"version":3,"file":"flock-persistence-journal.js","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","payload","bundle: ExportBundle"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\n/**\n * Persists `updates` in order. Reports how many leading updates are durable\n * and, if not all of them are, the failure. It must not reject.\n */\nexport type JournalWrite = (\n updates: readonly Uint8Array[],\n) => Promise<{ readonly durable: number; readonly error?: unknown }>;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n /**\n * An inclusive version vector observed with no Flock event since. Exporting\n * a version vector from wasm costs time proportional to the number of\n * peers, and Streams delivers many batches that change nothing (echoes of\n * our own appends, replayed catch-ups). Reusing the vector read after the\n * previous import keeps such a batch to a single read.\n */\n private knownVersion?: VersionVector;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (batch.events.length) this.knownVersion = undefined;\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n // The inclusive vector only grows during the process lifetime, so a\n // cached vector that is stale because of an untracked import can only\n // cause a redundant save, never a skipped one.\n const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes. An import can also advance the version without\n // emitting events (an entry that loses last-writer-wins), so compare.\n if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private versionUnchangedSince(previous: VersionVector): boolean {\n const current = this.flock.inclusiveVersion();\n this.knownVersion = current;\n return versionsEqual(previous, current);\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /**\n * Called within the owner's serialized persistence queue.\n *\n * Each round hands every pending payload to `write` at once, so a storage\n * adapter can commit them in one transaction. Only the prefix `write`\n * reports durable is acknowledged.\n */\n async flush(write: JournalWrite): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const { durable, error: exactError } = await write(\n this.payloads.map((payload) => payload.bytes),\n );\n // Only this loop removes payloads, so the written ones are still the\n // queue's prefix. New imports/local changes during the write remain\n // behind them.\n this.payloads.splice(0, durable);\n if (exactError === undefined) continue;\n\n const payload = this.payloads[0]!;\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n const fallback = await write([snapshot]);\n if (fallback.error !== undefined) {\n throw new AggregateError(\n [exactError, fallback.error],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAUxC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;;;;;;;;CAQR,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,MAAM,OAAO,OAAQ,MAAK,eAAe;AAC7C,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EAIpC,MAAM,kBAAkB,KAAK,gBAAgB,KAAK,MAAM,kBAAkB;AAC1E,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAQR,MAAI,WAAW,OAAO,KAAK,CAAC,KAAK,sBAAsB,gBAAgB,EAAE;AACvE,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,sBAAsB,UAAkC;EAC9D,MAAM,UAAU,KAAK,MAAM,kBAAkB;AAC7C,OAAK,eAAe;AACpB,SAAO,cAAc,UAAU,QAAQ;;CAGzC,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;;;;;;;CAU/B,MAAM,MAAM,OAAoC;AAC9C,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,EAAE,SAAS,OAAO,eAAe,MAAM,MAC3C,KAAK,SAAS,KAAK,cAAYC,UAAQ,MAAM,CAC9C;AAID,QAAK,SAAS,OAAO,GAAG,QAAQ;AAChC,OAAI,eAAe,OAAW;GAE9B,MAAM,UAAU,KAAK,SAAS;AAC9B,WAAQ,YAAY;AAIpB,OAAI,QAAQ,WAAW,gCACrB,OAAM;GAUR,MAAM,kBAAkB,KAAK,SAAS;GAEtC,MAAM,WAAW,MAAM,MAAM,CADZ,MAAM,wBAAwB,KAAK,MAAM,CACnB,CAAC;AACxC,OAAI,SAAS,UAAU,OACrB,OAAM,IAAI,eACR,CAAC,YAAY,SAAS,MAAM,EAC5B,6DACD;AAEH,QAAK,SAAS,OAAO,GAAG,gBAAgB;;;CAI5C,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAM,wBAAwB,KAAK,MAAM,GACzC,qBAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|
package/dist/flock-snapshot.cjs
CHANGED
|
@@ -116,7 +116,7 @@ function hydrateMetaSnapshots(legacy, updates) {
|
|
|
116
116
|
if (merged.format === "json") loadedJsonUpdate = true;
|
|
117
117
|
} else {
|
|
118
118
|
baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));
|
|
119
|
-
if (baseUpdateIndex >= 0) flock = __loro_dev_flock_wasm.Flock
|
|
119
|
+
if (baseUpdateIndex >= 0) flock = importFlockFile(new __loro_dev_flock_wasm.Flock(), updates[baseUpdateIndex]);
|
|
120
120
|
}
|
|
121
121
|
for (let index = 0; index < updates.length; index += 1) {
|
|
122
122
|
if (index === baseUpdateIndex) continue;
|
|
@@ -129,20 +129,25 @@ function hydrateMetaSnapshots(legacy, updates) {
|
|
|
129
129
|
loadedJsonUpdate
|
|
130
130
|
};
|
|
131
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* Imports a stored Flock file on Flock's fast path.
|
|
134
|
+
*
|
|
135
|
+
* Recovery decodes and rebuilds every entry, so using it for every stored file
|
|
136
|
+
* rebuilt the whole store on each load and periodic compaction. `importFile`
|
|
137
|
+
* checks v2 table CRCs first; into an empty Flock it then adopts the file
|
|
138
|
+
* lazily in O(1). If the file is rejected, flock-wasm retries through
|
|
139
|
+
* `recoverImportFile`, so detected corruption still drops only the unreadable
|
|
140
|
+
* entries. This matches how the live replica imported the same Streams bytes.
|
|
141
|
+
*/
|
|
142
|
+
function importFlockFile(flock, bytes) {
|
|
143
|
+
flock.importFile(bytes);
|
|
144
|
+
return flock;
|
|
145
|
+
}
|
|
132
146
|
function mergeMetaSnapshot(flock, bytes) {
|
|
133
|
-
if (isFlockFile(bytes)) {
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
flock,
|
|
138
|
-
format: "file"
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
return {
|
|
142
|
-
flock: __loro_dev_flock_wasm.Flock.recoverFromFile(bytes).flock,
|
|
143
|
-
format: "file"
|
|
144
|
-
};
|
|
145
|
-
}
|
|
147
|
+
if (isFlockFile(bytes)) return {
|
|
148
|
+
flock: importFlockFile(flock ?? new __loro_dev_flock_wasm.Flock(), bytes),
|
|
149
|
+
format: "file"
|
|
150
|
+
};
|
|
146
151
|
const bundle = JSON.parse(textDecoder.decode(bytes));
|
|
147
152
|
if (flock) {
|
|
148
153
|
flock.importJson(bundle);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"flock-snapshot.cjs","names":["CURRENT_SNAPSHOT_FORMAT: PersistedFormatId","flock: Flock | undefined","Flock"],"sources":["../src/storage/format.ts","../src/storage/meta-snapshot.ts","../src/internal/flock-snapshot.ts"],"sourcesContent":["/**\n * Persisted-format identification, in one place.\n *\n * ## Why this module exists\n *\n * Every persisted format this package reads is identified by sniffing:\n * magic bytes for Flock files, \"does it parse as JSON\" for update bundles,\n * a literal file name for the Filesystem adapter's full snapshot, a key\n * prefix for IndexedDB's queue layout. There has never been an explicit\n * version stamp, and each storage adapter grew its own copy of the\n * \"is this layout current?\" decision. Those copies are not equivalent —\n * one checks a legacy queue key, another compares a file name, a third\n * skips Flock-doc JSON entirely — which is exactly where a future format\n * change would silently diverge.\n *\n * Detection stays sniff-based, because that is what already-written data\n * supports: bytes on disk today carry no version field, and inventing one\n * cannot retroactively apply to them. What this module adds is a single\n * definition of each format, one shared \"needs rewriting\" predicate, and an\n * explicit version constant that new writers can stamp going forward.\n *\n * ## Format registry\n *\n * | id | encoding | written today |\n * |---------------|---------------------------------|---------------|\n * | `flock-v1` | Flock file, `FLOK` magic | no (read-only)|\n * | `flock-v2` | Flock file, `FLK2` magic | yes |\n * | `json-bundle` | UTF-8 JSON `ExportBundle` | yes (updates) |\n *\n * `flock-v1` is retained deliberately: it is the on-disk shape of data\n * written by older versions, and dropping the reader would make that data\n * unreadable rather than merely stale.\n */\n\nconst FLOCK_FILE_MAGIC_LENGTH = 4;\nconst FLOCK_FILE_MAGIC_V1 = [0x46, 0x4c, 0x4f, 0x4b]; // \"FLOK\"\nconst FLOCK_FILE_MAGIC_V2 = [0x46, 0x4c, 0x4b, 0x32]; // \"FLK2\"\n\n/**\n * Formats this package can read.\n *\n * `\"unknown\"` is reported rather than guessed: a caller that cannot\n * identify bytes should surface that, not assume the newest shape.\n */\nexport type PersistedFormatId =\n | \"flock-v1\"\n | \"flock-v2\"\n | \"json-bundle\"\n | \"unknown\";\n\n/**\n * Format new full snapshots are written in.\n *\n * Bump this together with a reader for the previous value, never instead of\n * one — the compatibility window is the whole point.\n */\nexport const CURRENT_SNAPSHOT_FORMAT: PersistedFormatId = \"flock-v2\";\n\n/**\n * Formats whose readers must keep working for already-persisted data.\n *\n * Listed explicitly so removing one is a visible decision rather than a\n * side effect of tidying a branch that \"looks dead\".\n */\nexport const SUPPORTED_READ_FORMATS: ReadonlyArray<PersistedFormatId> = [\n \"flock-v1\",\n \"flock-v2\",\n \"json-bundle\",\n];\n\nfunction matchesMagic(bytes: Uint8Array, magic: readonly number[]): boolean {\n for (let i = 0; i < magic.length; i += 1) {\n if (bytes[i] !== magic[i]) return false;\n }\n return true;\n}\n\n/**\n * Identifies persisted bytes.\n *\n * Anything without a recognized Flock magic is treated as a JSON bundle,\n * matching how the hydration path has always behaved; `\"unknown\"` is\n * reserved for input too short to classify at all.\n */\nexport function detectPersistedFormat(bytes: Uint8Array): PersistedFormatId {\n // Too short to carry a magic header, and too short to be any bundle this\n // package writes — say so rather than defaulting it into a format whose\n // reader would then fail on it.\n if (bytes.length < FLOCK_FILE_MAGIC_LENGTH) return \"unknown\";\n if (matchesMagic(bytes, FLOCK_FILE_MAGIC_V2)) return \"flock-v2\";\n if (matchesMagic(bytes, FLOCK_FILE_MAGIC_V1)) return \"flock-v1\";\n return \"json-bundle\";\n}\n\n/** Whether `bytes` is a Flock file snapshot in any supported version. */\nexport function isFlockFile(bytes: Uint8Array): boolean {\n const format = detectPersistedFormat(bytes);\n return format === \"flock-v1\" || format === \"flock-v2\";\n}\n\n/** Whether `bytes` is written in the format new snapshots use. */\nexport function isCurrentSnapshotFormat(bytes: Uint8Array): boolean {\n return detectPersistedFormat(bytes) === CURRENT_SNAPSHOT_FORMAT;\n}\n\n/**\n * Inputs to the shared \"should this metadata layout be rewritten?\" decision.\n *\n * Each storage adapter supplies what it can observe; the policy itself lives\n * here so the three adapters cannot drift apart on it.\n */\nexport interface MetaLayoutState {\n /** The base snapshot as loaded, if any. */\n readonly snapshot?: Uint8Array;\n /** Number of queued update records sitting alongside the snapshot. */\n readonly queuedUpdates: number;\n /** True when hydration had to parse a JSON bundle as a full snapshot. */\n readonly loadedJsonUpdate: boolean;\n /**\n * True when the adapter found data in a layout it no longer writes —\n * IndexedDB's single `\"queue\"` key, the Filesystem adapter's `meta.json`.\n */\n readonly legacyLayout?: boolean;\n}\n\n/**\n * Whether the persisted metadata layout should be rewritten into the current\n * one on load.\n *\n * Rewriting is what stops old formats from lingering and being re-parsed on\n * every start. Missing a case does not corrupt anything, but it does mean\n * paying the legacy parse cost forever — which is why the decision is one\n * shared function rather than three lookalikes.\n */\nexport function metaLayoutNeedsRewrite(state: MetaLayoutState): boolean {\n if (state.legacyLayout) return true;\n if (state.loadedJsonUpdate) return true;\n if (state.queuedUpdates > 0) return true;\n if (state.snapshot !== undefined && !isCurrentSnapshotFormat(state.snapshot)) {\n return true;\n }\n return false;\n}\n","import { Flock, type ExportBundle } from \"@loro-dev/flock-wasm\";\n\nimport { isFlockFile } from \"./format\";\n\nexport { isFlockFile };\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\n\ntype SnapshotFormat = \"file\" | \"json\";\n\ninterface MergeResult {\n flock: Flock;\n format: SnapshotFormat;\n}\n\nexport interface HydrateMetaSnapshotResult {\n flock: Flock | undefined;\n loadedJsonUpdate: boolean;\n}\n\nexport function encodeMetaJsonUpdate(bundle: ExportBundle): Uint8Array {\n return textEncoder.encode(JSON.stringify(bundle));\n}\n\nexport function hydrateMetaSnapshots(\n legacy: Uint8Array | undefined,\n updates: readonly Uint8Array[],\n): HydrateMetaSnapshotResult {\n let loadedJsonUpdate = false;\n let baseUpdateIndex = -1;\n let flock: Flock | undefined;\n\n if (legacy) {\n const merged = mergeMetaSnapshot(flock, legacy);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n } else {\n baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));\n if (baseUpdateIndex >= 0) {\n // A remote bootstrap may contain recoverable corrupt entries. Build a\n // canonical, hydratable base from the accepted subset instead of\n // retaining an object produced by the compatibility import path.\n flock = Flock.recoverFromFile(\n updates[baseUpdateIndex] as Uint8Array,\n ).flock;\n }\n }\n\n for (let index = 0; index < updates.length; index += 1) {\n if (index === baseUpdateIndex) continue;\n const merged = mergeMetaSnapshot(flock, updates[index] as Uint8Array);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n }\n\n return { flock, loadedJsonUpdate };\n}\n\nfunction mergeMetaSnapshot(\n flock: Flock | undefined,\n bytes: Uint8Array,\n): MergeResult {\n if (isFlockFile(bytes)) {\n if (flock) {\n flock.recoverImportFile(bytes);\n return { flock, format: \"file\" };\n }\n return { flock: Flock.recoverFromFile(bytes).flock, format: \"file\" };\n }\n\n const bundle = JSON.parse(textDecoder.decode(bytes)) as ExportBundle;\n if (flock) {\n flock.importJson(bundle);\n return { flock, format: \"json\" };\n }\n const hydrated = new Flock();\n hydrated.importJson(bundle);\n return { flock: hydrated, format: \"json\" };\n}\n","import type { Flock } from \"@loro-dev/flock-wasm\";\n\nexport const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;\n\nexport interface ExportFlockFileSnapshotOptions {\n readonly budgetMs?: number;\n readonly signal?: AbortSignal;\n}\n\ntype FlockWithIncrementalFileExport = Flock & {\n exportFileV2Incremental?: (\n options?: ExportFlockFileSnapshotOptions,\n ) => Promise<Uint8Array>;\n};\n\ntype FlockWithIsEmpty = Flock & {\n isEmpty?: () => boolean;\n};\n\nexport async function exportFlockFileSnapshot(\n flock: Flock,\n options: ExportFlockFileSnapshotOptions = {},\n): Promise<Uint8Array> {\n const exportIncremental = (flock as FlockWithIncrementalFileExport)\n .exportFileV2Incremental;\n if (typeof exportIncremental === \"function\") {\n return exportIncremental.call(flock, {\n budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,\n signal: options.signal,\n });\n }\n return flock.exportFile();\n}\n\nexport function hasVisibleRows(flock: Flock): boolean {\n const isEmpty = (flock as FlockWithIsEmpty).isEmpty;\n if (typeof isEmpty === \"function\") {\n return !isEmpty.call(flock);\n }\n\n const rows = flock.scan({});\n for (const row of rows) {\n if (row.value !== undefined) return true;\n }\n return false;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,0BAA0B;AAChC,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AACpD,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;;;;;;;AAoBpD,MAAaA,0BAA6C;AAc1D,SAAS,aAAa,OAAmB,OAAmC;AAC1E,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,EACrC,KAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAEpC,QAAO;;;;;;;;;AAUT,SAAgB,sBAAsB,OAAsC;AAI1E,KAAI,MAAM,SAAS,wBAAyB,QAAO;AACnD,KAAI,aAAa,OAAO,oBAAoB,CAAE,QAAO;AACrD,KAAI,aAAa,OAAO,oBAAoB,CAAE,QAAO;AACrD,QAAO;;;AAIT,SAAgB,YAAY,OAA4B;CACtD,MAAM,SAAS,sBAAsB,MAAM;AAC3C,QAAO,WAAW,cAAc,WAAW;;;AAI7C,SAAgB,wBAAwB,OAA4B;AAClE,QAAO,sBAAsB,MAAM,KAAK;;;;;;;;;;;AAgC1C,SAAgB,uBAAuB,OAAiC;AACtE,KAAI,MAAM,aAAc,QAAO;AAC/B,KAAI,MAAM,iBAAkB,QAAO;AACnC,KAAI,MAAM,gBAAgB,EAAG,QAAO;AACpC,KAAI,MAAM,aAAa,UAAa,CAAC,wBAAwB,MAAM,SAAS,CAC1E,QAAO;AAET,QAAO;;;;;ACvIT,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,cAAc,IAAI,aAAa;AAcrC,SAAgB,qBAAqB,QAAkC;AACrE,QAAO,YAAY,OAAO,KAAK,UAAU,OAAO,CAAC;;AAGnD,SAAgB,qBACd,QACA,SAC2B;CAC3B,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,IAAIC;AAEJ,KAAI,QAAQ;EACV,MAAM,SAAS,kBAAkB,OAAO,OAAO;AAC/C,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;QAEhB;AACL,oBAAkB,QAAQ,WAAW,UAAU,YAAY,MAAM,CAAC;AAClE,MAAI,mBAAmB,EAIrB,SAAQC,4BAAM,gBACZ,QAAQ,iBACT,CAAC;;AAIN,MAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;AACtD,MAAI,UAAU,gBAAiB;EAC/B,MAAM,SAAS,kBAAkB,OAAO,QAAQ,OAAqB;AACrE,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;;AAIvB,QAAO;EAAE;EAAO;EAAkB;;AAGpC,SAAS,kBACP,OACA,OACa;AACb,KAAI,YAAY,MAAM,EAAE;AACtB,MAAI,OAAO;AACT,SAAM,kBAAkB,MAAM;AAC9B,UAAO;IAAE;IAAO,QAAQ;IAAQ;;AAElC,SAAO;GAAE,OAAOA,4BAAM,gBAAgB,MAAM,CAAC;GAAO,QAAQ;GAAQ;;CAGtE,MAAM,SAAS,KAAK,MAAM,YAAY,OAAO,MAAM,CAAC;AACpD,KAAI,OAAO;AACT,QAAM,WAAW,OAAO;AACxB,SAAO;GAAE;GAAO,QAAQ;GAAQ;;CAElC,MAAM,WAAW,IAAIA,6BAAO;AAC5B,UAAS,WAAW,OAAO;AAC3B,QAAO;EAAE,OAAO;EAAU,QAAQ;EAAQ;;;;;AChF5C,MAAa,sCAAsC;AAiBnD,eAAsB,wBACpB,OACA,UAA0C,EAAE,EACvB;CACrB,MAAM,oBAAqB,MACxB;AACH,KAAI,OAAO,sBAAsB,WAC/B,QAAO,kBAAkB,KAAK,OAAO;EACnC,UAAU,QAAQ,YAAY;EAC9B,QAAQ,QAAQ;EACjB,CAAC;AAEJ,QAAO,MAAM,YAAY;;AAG3B,SAAgB,eAAe,OAAuB;CACpD,MAAM,UAAW,MAA2B;AAC5C,KAAI,OAAO,YAAY,WACrB,QAAO,CAAC,QAAQ,KAAK,MAAM;CAG7B,MAAM,OAAO,MAAM,KAAK,EAAE,CAAC;AAC3B,MAAK,MAAM,OAAO,KAChB,KAAI,IAAI,UAAU,OAAW,QAAO;AAEtC,QAAO"}
|
|
1
|
+
{"version":3,"file":"flock-snapshot.cjs","names":["CURRENT_SNAPSHOT_FORMAT: PersistedFormatId","flock: Flock | undefined","Flock"],"sources":["../src/storage/format.ts","../src/storage/meta-snapshot.ts","../src/internal/flock-snapshot.ts"],"sourcesContent":["/**\n * Persisted-format identification, in one place.\n *\n * ## Why this module exists\n *\n * Every persisted format this package reads is identified by sniffing:\n * magic bytes for Flock files, \"does it parse as JSON\" for update bundles,\n * a literal file name for the Filesystem adapter's full snapshot, a key\n * prefix for IndexedDB's queue layout. There has never been an explicit\n * version stamp, and each storage adapter grew its own copy of the\n * \"is this layout current?\" decision. Those copies are not equivalent —\n * one checks a legacy queue key, another compares a file name, a third\n * skips Flock-doc JSON entirely — which is exactly where a future format\n * change would silently diverge.\n *\n * Detection stays sniff-based, because that is what already-written data\n * supports: bytes on disk today carry no version field, and inventing one\n * cannot retroactively apply to them. What this module adds is a single\n * definition of each format, one shared \"needs rewriting\" predicate, and an\n * explicit version constant that new writers can stamp going forward.\n *\n * ## Format registry\n *\n * | id | encoding | written today |\n * |---------------|---------------------------------|---------------|\n * | `flock-v1` | Flock file, `FLOK` magic | no (read-only)|\n * | `flock-v2` | Flock file, `FLK2` magic | yes |\n * | `json-bundle` | UTF-8 JSON `ExportBundle` | yes (updates) |\n *\n * `flock-v1` is retained deliberately: it is the on-disk shape of data\n * written by older versions, and dropping the reader would make that data\n * unreadable rather than merely stale.\n */\n\nconst FLOCK_FILE_MAGIC_LENGTH = 4;\nconst FLOCK_FILE_MAGIC_V1 = [0x46, 0x4c, 0x4f, 0x4b]; // \"FLOK\"\nconst FLOCK_FILE_MAGIC_V2 = [0x46, 0x4c, 0x4b, 0x32]; // \"FLK2\"\n\n/**\n * Formats this package can read.\n *\n * `\"unknown\"` is reported rather than guessed: a caller that cannot\n * identify bytes should surface that, not assume the newest shape.\n */\nexport type PersistedFormatId =\n | \"flock-v1\"\n | \"flock-v2\"\n | \"json-bundle\"\n | \"unknown\";\n\n/**\n * Format new full snapshots are written in.\n *\n * Bump this together with a reader for the previous value, never instead of\n * one — the compatibility window is the whole point.\n */\nexport const CURRENT_SNAPSHOT_FORMAT: PersistedFormatId = \"flock-v2\";\n\n/**\n * Formats whose readers must keep working for already-persisted data.\n *\n * Listed explicitly so removing one is a visible decision rather than a\n * side effect of tidying a branch that \"looks dead\".\n */\nexport const SUPPORTED_READ_FORMATS: ReadonlyArray<PersistedFormatId> = [\n \"flock-v1\",\n \"flock-v2\",\n \"json-bundle\",\n];\n\nfunction matchesMagic(bytes: Uint8Array, magic: readonly number[]): boolean {\n for (let i = 0; i < magic.length; i += 1) {\n if (bytes[i] !== magic[i]) return false;\n }\n return true;\n}\n\n/**\n * Identifies persisted bytes.\n *\n * Anything without a recognized Flock magic is treated as a JSON bundle,\n * matching how the hydration path has always behaved; `\"unknown\"` is\n * reserved for input too short to classify at all.\n */\nexport function detectPersistedFormat(bytes: Uint8Array): PersistedFormatId {\n // Too short to carry a magic header, and too short to be any bundle this\n // package writes — say so rather than defaulting it into a format whose\n // reader would then fail on it.\n if (bytes.length < FLOCK_FILE_MAGIC_LENGTH) return \"unknown\";\n if (matchesMagic(bytes, FLOCK_FILE_MAGIC_V2)) return \"flock-v2\";\n if (matchesMagic(bytes, FLOCK_FILE_MAGIC_V1)) return \"flock-v1\";\n return \"json-bundle\";\n}\n\n/** Whether `bytes` is a Flock file snapshot in any supported version. */\nexport function isFlockFile(bytes: Uint8Array): boolean {\n const format = detectPersistedFormat(bytes);\n return format === \"flock-v1\" || format === \"flock-v2\";\n}\n\n/** Whether `bytes` is written in the format new snapshots use. */\nexport function isCurrentSnapshotFormat(bytes: Uint8Array): boolean {\n return detectPersistedFormat(bytes) === CURRENT_SNAPSHOT_FORMAT;\n}\n\n/**\n * Inputs to the shared \"should this metadata layout be rewritten?\" decision.\n *\n * Each storage adapter supplies what it can observe; the policy itself lives\n * here so the three adapters cannot drift apart on it.\n */\nexport interface MetaLayoutState {\n /** The base snapshot as loaded, if any. */\n readonly snapshot?: Uint8Array;\n /** Number of queued update records sitting alongside the snapshot. */\n readonly queuedUpdates: number;\n /** True when hydration had to parse a JSON bundle as a full snapshot. */\n readonly loadedJsonUpdate: boolean;\n /**\n * True when the adapter found data in a layout it no longer writes —\n * IndexedDB's single `\"queue\"` key, the Filesystem adapter's `meta.json`.\n */\n readonly legacyLayout?: boolean;\n}\n\n/**\n * Whether the persisted metadata layout should be rewritten into the current\n * one on load.\n *\n * Rewriting is what stops old formats from lingering and being re-parsed on\n * every start. Missing a case does not corrupt anything, but it does mean\n * paying the legacy parse cost forever — which is why the decision is one\n * shared function rather than three lookalikes.\n */\nexport function metaLayoutNeedsRewrite(state: MetaLayoutState): boolean {\n if (state.legacyLayout) return true;\n if (state.loadedJsonUpdate) return true;\n if (state.queuedUpdates > 0) return true;\n if (state.snapshot !== undefined && !isCurrentSnapshotFormat(state.snapshot)) {\n return true;\n }\n return false;\n}\n","import { Flock, type ExportBundle } from \"@loro-dev/flock-wasm\";\n\nimport { isFlockFile } from \"./format\";\n\nexport { isFlockFile };\n\nconst textEncoder = new TextEncoder();\nconst textDecoder = new TextDecoder();\n\ntype SnapshotFormat = \"file\" | \"json\";\n\ninterface MergeResult {\n flock: Flock;\n format: SnapshotFormat;\n}\n\nexport interface HydrateMetaSnapshotResult {\n flock: Flock | undefined;\n loadedJsonUpdate: boolean;\n}\n\nexport function encodeMetaJsonUpdate(bundle: ExportBundle): Uint8Array {\n return textEncoder.encode(JSON.stringify(bundle));\n}\n\nexport function hydrateMetaSnapshots(\n legacy: Uint8Array | undefined,\n updates: readonly Uint8Array[],\n): HydrateMetaSnapshotResult {\n let loadedJsonUpdate = false;\n let baseUpdateIndex = -1;\n let flock: Flock | undefined;\n\n if (legacy) {\n const merged = mergeMetaSnapshot(flock, legacy);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n } else {\n baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));\n if (baseUpdateIndex >= 0) {\n flock = importFlockFile(\n new Flock(),\n updates[baseUpdateIndex] as Uint8Array,\n );\n }\n }\n\n for (let index = 0; index < updates.length; index += 1) {\n if (index === baseUpdateIndex) continue;\n const merged = mergeMetaSnapshot(flock, updates[index] as Uint8Array);\n flock = merged.flock;\n if (merged.format === \"json\") {\n loadedJsonUpdate = true;\n }\n }\n\n return { flock, loadedJsonUpdate };\n}\n\n/**\n * Imports a stored Flock file on Flock's fast path.\n *\n * Recovery decodes and rebuilds every entry, so using it for every stored file\n * rebuilt the whole store on each load and periodic compaction. `importFile`\n * checks v2 table CRCs first; into an empty Flock it then adopts the file\n * lazily in O(1). If the file is rejected, flock-wasm retries through\n * `recoverImportFile`, so detected corruption still drops only the unreadable\n * entries. This matches how the live replica imported the same Streams bytes.\n */\nfunction importFlockFile(flock: Flock, bytes: Uint8Array): Flock {\n flock.importFile(bytes);\n return flock;\n}\n\nfunction mergeMetaSnapshot(\n flock: Flock | undefined,\n bytes: Uint8Array,\n): MergeResult {\n if (isFlockFile(bytes)) {\n return { flock: importFlockFile(flock ?? new Flock(), bytes), format: \"file\" };\n }\n\n const bundle = JSON.parse(textDecoder.decode(bytes)) as ExportBundle;\n if (flock) {\n flock.importJson(bundle);\n return { flock, format: \"json\" };\n }\n const hydrated = new Flock();\n hydrated.importJson(bundle);\n return { flock: hydrated, format: \"json\" };\n}\n","import type { Flock } from \"@loro-dev/flock-wasm\";\n\nexport const DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS = 8;\n\nexport interface ExportFlockFileSnapshotOptions {\n readonly budgetMs?: number;\n readonly signal?: AbortSignal;\n}\n\ntype FlockWithIncrementalFileExport = Flock & {\n exportFileV2Incremental?: (\n options?: ExportFlockFileSnapshotOptions,\n ) => Promise<Uint8Array>;\n};\n\ntype FlockWithIsEmpty = Flock & {\n isEmpty?: () => boolean;\n};\n\nexport async function exportFlockFileSnapshot(\n flock: Flock,\n options: ExportFlockFileSnapshotOptions = {},\n): Promise<Uint8Array> {\n const exportIncremental = (flock as FlockWithIncrementalFileExport)\n .exportFileV2Incremental;\n if (typeof exportIncremental === \"function\") {\n return exportIncremental.call(flock, {\n budgetMs: options.budgetMs ?? DEFAULT_FLOCK_FILE_EXPORT_BUDGET_MS,\n signal: options.signal,\n });\n }\n return flock.exportFile();\n}\n\nexport function hasVisibleRows(flock: Flock): boolean {\n const isEmpty = (flock as FlockWithIsEmpty).isEmpty;\n if (typeof isEmpty === \"function\") {\n return !isEmpty.call(flock);\n }\n\n const rows = flock.scan({});\n for (const row of rows) {\n if (row.value !== undefined) return true;\n }\n return false;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,MAAM,0BAA0B;AAChC,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;AACpD,MAAM,sBAAsB;CAAC;CAAM;CAAM;CAAM;CAAK;;;;;;;AAoBpD,MAAaA,0BAA6C;AAc1D,SAAS,aAAa,OAAmB,OAAmC;AAC1E,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,EACrC,KAAI,MAAM,OAAO,MAAM,GAAI,QAAO;AAEpC,QAAO;;;;;;;;;AAUT,SAAgB,sBAAsB,OAAsC;AAI1E,KAAI,MAAM,SAAS,wBAAyB,QAAO;AACnD,KAAI,aAAa,OAAO,oBAAoB,CAAE,QAAO;AACrD,KAAI,aAAa,OAAO,oBAAoB,CAAE,QAAO;AACrD,QAAO;;;AAIT,SAAgB,YAAY,OAA4B;CACtD,MAAM,SAAS,sBAAsB,MAAM;AAC3C,QAAO,WAAW,cAAc,WAAW;;;AAI7C,SAAgB,wBAAwB,OAA4B;AAClE,QAAO,sBAAsB,MAAM,KAAK;;;;;;;;;;;AAgC1C,SAAgB,uBAAuB,OAAiC;AACtE,KAAI,MAAM,aAAc,QAAO;AAC/B,KAAI,MAAM,iBAAkB,QAAO;AACnC,KAAI,MAAM,gBAAgB,EAAG,QAAO;AACpC,KAAI,MAAM,aAAa,UAAa,CAAC,wBAAwB,MAAM,SAAS,CAC1E,QAAO;AAET,QAAO;;;;;ACvIT,MAAM,cAAc,IAAI,aAAa;AACrC,MAAM,cAAc,IAAI,aAAa;AAcrC,SAAgB,qBAAqB,QAAkC;AACrE,QAAO,YAAY,OAAO,KAAK,UAAU,OAAO,CAAC;;AAGnD,SAAgB,qBACd,QACA,SAC2B;CAC3B,IAAI,mBAAmB;CACvB,IAAI,kBAAkB;CACtB,IAAIC;AAEJ,KAAI,QAAQ;EACV,MAAM,SAAS,kBAAkB,OAAO,OAAO;AAC/C,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;QAEhB;AACL,oBAAkB,QAAQ,WAAW,UAAU,YAAY,MAAM,CAAC;AAClE,MAAI,mBAAmB,EACrB,SAAQ,gBACN,IAAIC,6BAAO,EACX,QAAQ,iBACT;;AAIL,MAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,GAAG;AACtD,MAAI,UAAU,gBAAiB;EAC/B,MAAM,SAAS,kBAAkB,OAAO,QAAQ,OAAqB;AACrE,UAAQ,OAAO;AACf,MAAI,OAAO,WAAW,OACpB,oBAAmB;;AAIvB,QAAO;EAAE;EAAO;EAAkB;;;;;;;;;;;;AAapC,SAAS,gBAAgB,OAAc,OAA0B;AAC/D,OAAM,WAAW,MAAM;AACvB,QAAO;;AAGT,SAAS,kBACP,OACA,OACa;AACb,KAAI,YAAY,MAAM,CACpB,QAAO;EAAE,OAAO,gBAAgB,SAAS,IAAIA,6BAAO,EAAE,MAAM;EAAE,QAAQ;EAAQ;CAGhF,MAAM,SAAS,KAAK,MAAM,YAAY,OAAO,MAAM,CAAC;AACpD,KAAI,OAAO;AACT,QAAM,WAAW,OAAO;AACxB,SAAO;GAAE;GAAO,QAAQ;GAAQ;;CAElC,MAAM,WAAW,IAAIA,6BAAO;AAC5B,UAAS,WAAW,OAAO;AAC3B,QAAO;EAAE,OAAO;EAAU,QAAQ;EAAQ;;;;;ACzF5C,MAAa,sCAAsC;AAiBnD,eAAsB,wBACpB,OACA,UAA0C,EAAE,EACvB;CACrB,MAAM,oBAAqB,MACxB;AACH,KAAI,OAAO,sBAAsB,WAC/B,QAAO,kBAAkB,KAAK,OAAO;EACnC,UAAU,QAAQ,YAAY;EAC9B,QAAQ,QAAQ;EACjB,CAAC;AAEJ,QAAO,MAAM,YAAY;;AAG3B,SAAgB,eAAe,OAAuB;CACpD,MAAM,UAAW,MAA2B;AAC5C,KAAI,OAAO,YAAY,WACrB,QAAO,CAAC,QAAQ,KAAK,MAAM;CAG7B,MAAM,OAAO,MAAM,KAAK,EAAE,CAAC;AAC3B,MAAK,MAAM,OAAO,KAChB,KAAI,IAAI,UAAU,OAAW,QAAO;AAEtC,QAAO"}
|
package/dist/flock-snapshot.js
CHANGED
|
@@ -114,7 +114,7 @@ function hydrateMetaSnapshots(legacy, updates) {
|
|
|
114
114
|
if (merged.format === "json") loadedJsonUpdate = true;
|
|
115
115
|
} else {
|
|
116
116
|
baseUpdateIndex = updates.findIndex((bytes) => isFlockFile(bytes));
|
|
117
|
-
if (baseUpdateIndex >= 0) flock = Flock
|
|
117
|
+
if (baseUpdateIndex >= 0) flock = importFlockFile(new Flock(), updates[baseUpdateIndex]);
|
|
118
118
|
}
|
|
119
119
|
for (let index = 0; index < updates.length; index += 1) {
|
|
120
120
|
if (index === baseUpdateIndex) continue;
|
|
@@ -127,20 +127,25 @@ function hydrateMetaSnapshots(legacy, updates) {
|
|
|
127
127
|
loadedJsonUpdate
|
|
128
128
|
};
|
|
129
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* Imports a stored Flock file on Flock's fast path.
|
|
132
|
+
*
|
|
133
|
+
* Recovery decodes and rebuilds every entry, so using it for every stored file
|
|
134
|
+
* rebuilt the whole store on each load and periodic compaction. `importFile`
|
|
135
|
+
* checks v2 table CRCs first; into an empty Flock it then adopts the file
|
|
136
|
+
* lazily in O(1). If the file is rejected, flock-wasm retries through
|
|
137
|
+
* `recoverImportFile`, so detected corruption still drops only the unreadable
|
|
138
|
+
* entries. This matches how the live replica imported the same Streams bytes.
|
|
139
|
+
*/
|
|
140
|
+
function importFlockFile(flock, bytes) {
|
|
141
|
+
flock.importFile(bytes);
|
|
142
|
+
return flock;
|
|
143
|
+
}
|
|
130
144
|
function mergeMetaSnapshot(flock, bytes) {
|
|
131
|
-
if (isFlockFile(bytes)) {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
flock,
|
|
136
|
-
format: "file"
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
return {
|
|
140
|
-
flock: Flock.recoverFromFile(bytes).flock,
|
|
141
|
-
format: "file"
|
|
142
|
-
};
|
|
143
|
-
}
|
|
145
|
+
if (isFlockFile(bytes)) return {
|
|
146
|
+
flock: importFlockFile(flock ?? new Flock(), bytes),
|
|
147
|
+
format: "file"
|
|
148
|
+
};
|
|
144
149
|
const bundle = JSON.parse(textDecoder.decode(bytes));
|
|
145
150
|
if (flock) {
|
|
146
151
|
flock.importJson(bundle);
|