loro-repo 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +195 -22
- package/dist/flock-persistence-journal.cjs +169 -0
- package/dist/flock-persistence-journal.cjs.map +1 -0
- package/dist/flock-persistence-journal.js +158 -0
- package/dist/flock-persistence-journal.js.map +1 -0
- package/dist/flock-snapshot.cjs +3 -3
- package/dist/flock-snapshot.cjs.map +1 -1
- package/dist/flock-snapshot.js +3 -3
- package/dist/flock-snapshot.js.map +1 -1
- package/dist/index.cjs +452 -206
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +11 -2
- package/dist/index.d.ts +11 -2
- package/dist/index.js +454 -209
- package/dist/index.js.map +1 -1
- package/dist/replica-checkpoints.cjs +105 -0
- package/dist/replica-checkpoints.cjs.map +1 -0
- package/dist/replica-checkpoints.js +81 -0
- package/dist/replica-checkpoints.js.map +1 -0
- package/dist/storage/filesystem.cjs +2 -2
- package/dist/storage/filesystem.cjs.map +1 -1
- package/dist/storage/filesystem.d.cts +1 -1
- package/dist/storage/filesystem.d.ts +1 -1
- package/dist/storage/filesystem.js +2 -2
- package/dist/storage/filesystem.js.map +1 -1
- package/dist/storage/indexeddb.cjs +544 -148
- package/dist/storage/indexeddb.cjs.map +1 -1
- package/dist/storage/indexeddb.d.cts +99 -12
- package/dist/storage/indexeddb.d.ts +99 -12
- package/dist/storage/indexeddb.js +543 -149
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/sqlite.cjs +242 -58
- package/dist/storage/sqlite.cjs.map +1 -1
- package/dist/storage/sqlite.d.cts +60 -2
- package/dist/storage/sqlite.d.ts +59 -1
- package/dist/storage/sqlite.js +242 -59
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/transport/broadcast-channel.d.cts +1 -1
- package/dist/transport/broadcast-channel.d.ts +1 -1
- package/dist/transport/streams.cjs +46 -26
- package/dist/transport/streams.cjs.map +1 -1
- package/dist/transport/streams.d.cts +33 -5
- package/dist/transport/streams.d.ts +32 -4
- package/dist/transport/streams.js +47 -27
- package/dist/transport/streams.js.map +1 -1
- package/dist/transport/websocket.d.cts +1 -1
- package/dist/transport/websocket.d.ts +1 -1
- package/dist/types.cjs +37 -0
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +107 -12
- package/dist/types.d.ts +107 -12
- package/dist/types.js +32 -1
- package/dist/types.js.map +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -148,6 +148,67 @@ Key points:
|
|
|
148
148
|
- `FileSystemStorageAdaptor` (`src/storage/filesystem.ts`)
|
|
149
149
|
- `SqliteRepoStore` (`src/storage/sqlite.ts`, Node.js only)
|
|
150
150
|
|
|
151
|
+
## Classifying storage failures
|
|
152
|
+
|
|
153
|
+
`IndexedDBStorageAdaptor` wraps backend failures in a `RepoStorageError`
|
|
154
|
+
carrying a stable `code`, so callers do not have to pattern-match browser error
|
|
155
|
+
text (which differs across engines, versions, and UI locales). Misusing the
|
|
156
|
+
adaptor — an unsupported payload type, a call after `close()` — still throws a
|
|
157
|
+
plain `Error`: those are bugs in the caller, not storage failures.
|
|
158
|
+
|
|
159
|
+
```ts
|
|
160
|
+
import { RepoStorageError } from "loro-repo";
|
|
161
|
+
|
|
162
|
+
try {
|
|
163
|
+
await repo.flush();
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if (error instanceof RepoStorageError) {
|
|
166
|
+
switch (error.code) {
|
|
167
|
+
case "quota":
|
|
168
|
+
// Out of disk / origin quota. Retrying will keep failing until space
|
|
169
|
+
// is freed — tell the user, and stop accepting writes.
|
|
170
|
+
break;
|
|
171
|
+
case "unavailable":
|
|
172
|
+
// The store is not usable right now: connection closing, backing
|
|
173
|
+
// store unreachable, open blocked by another connection, or the
|
|
174
|
+
// context is not allowed to use IndexedDB (sandboxed iframe,
|
|
175
|
+
// opaque origin).
|
|
176
|
+
break;
|
|
177
|
+
case "unknown":
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
`error.cause` is the original `DOMException` and `error.causeName` is its
|
|
185
|
+
`name` (`"QuotaExceededError"`, `"InvalidStateError"`, …), which the wrapped
|
|
186
|
+
`message` does not include.
|
|
187
|
+
|
|
188
|
+
Three caveats worth knowing:
|
|
189
|
+
|
|
190
|
+
- **`code` is best-effort for two error names.** `QuotaExceededError` and
|
|
191
|
+
`SecurityError` mean one thing each, so the name alone classifies them.
|
|
192
|
+
`InvalidStateError` and `UnknownError` are overloaded, so the adaptor also
|
|
193
|
+
probes the message — and that text is Chromium's. On other engines those two
|
|
194
|
+
fall back to `"unknown"` rather than being guessed into an actionable bucket.
|
|
195
|
+
- **Background persistence logs, it does not throw.** Debounced auto-saves
|
|
196
|
+
report failures through `console.error` and stay dirty for the next attempt.
|
|
197
|
+
To act on a `code`, `await repo.flush()` (or a load) where the error reaches
|
|
198
|
+
you.
|
|
199
|
+
- **Loading never fails for a write-related reason.** Folding queued updates
|
|
200
|
+
back into a snapshot is an optimization; if that write fails, the document is
|
|
201
|
+
still returned and the updates stay queued for the next load. The compaction
|
|
202
|
+
entry points (`compactMeta`, `compactFlockDoc`) are the opposite: the write
|
|
203
|
+
*is* the request, so they reject.
|
|
204
|
+
|
|
205
|
+
A failed *open* is retried on the next call, so a transient `blocked` (another
|
|
206
|
+
tab still holding the previous version) does not permanently disable the
|
|
207
|
+
adaptor. A *live* connection that has started failing is a different matter: the
|
|
208
|
+
adaptor deliberately does **not** reopen it on its own, because a reopen usually
|
|
209
|
+
fails for the same reason and silent self-healing makes it harder for an
|
|
210
|
+
application to fail closed. That recovery is the caller's decision.
|
|
211
|
+
|
|
151
212
|
## Per-room Streams payload protection
|
|
152
213
|
|
|
153
214
|
`StreamsTransportAdapter` can select one streams-crdt provider config for each
|
|
@@ -234,21 +295,103 @@ protected-payload boundary and `payloadProtectionRequired` guard. No 0.12.x
|
|
|
234
295
|
encryption compatibility shim is retained; the adapter passes the published
|
|
235
296
|
0.15 opaque `streamUrl` transport target directly.
|
|
236
297
|
|
|
298
|
+
## Replica-bound Streams checkpoints (IndexedDB)
|
|
299
|
+
|
|
300
|
+
For browser Flock rooms, let the repo's IndexedDB storage own checkpoints:
|
|
301
|
+
|
|
302
|
+
```ts
|
|
303
|
+
import { LoroRepo } from "loro-repo";
|
|
304
|
+
import { IndexedDBStorageAdaptor } from "loro-repo/storage/indexeddb";
|
|
305
|
+
import {
|
|
306
|
+
StreamsTransportAdapter,
|
|
307
|
+
createRepoStreamsPersistence,
|
|
308
|
+
} from "loro-repo/transport/streams";
|
|
309
|
+
|
|
310
|
+
const repo = await LoroRepo.create({
|
|
311
|
+
storageAdapter: new IndexedDBStorageAdaptor({ dbName: "workspace-42" }),
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
await repo.addTransport(
|
|
315
|
+
"streams",
|
|
316
|
+
new StreamsTransportAdapter({
|
|
317
|
+
bucketId: "your-bucket",
|
|
318
|
+
auth: yourAuthProvider,
|
|
319
|
+
persistence: createRepoStreamsPersistence(repo),
|
|
320
|
+
}),
|
|
321
|
+
);
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
The storage adapter restores each Meta/named Flock snapshot, update log and
|
|
325
|
+
Streams checkpoint from one capture transaction. Bound generations use a
|
|
326
|
+
readonly transaction. A missing generation, or one whose lineage marker is gone
|
|
327
|
+
(see below), is replaced while recapturing the resource in one readwrite
|
|
328
|
+
transaction. A live Flock only sees the cursor
|
|
329
|
+
captured with its own restored bytes, even when another tab writes a newer
|
|
330
|
+
cursor to the same database later. Strict Flock data commits precede the strict
|
|
331
|
+
checkpoint transaction, while LoroDoc body writes retain normal IndexedDB
|
|
332
|
+
durability. Named Flock
|
|
333
|
+
deletion removes its checkpoint record with the data and invalidates outstanding
|
|
334
|
+
cursor stores. Version changes from another tab are handled by reopening the
|
|
335
|
+
database before subsequent checkpoint mutations.
|
|
336
|
+
|
|
337
|
+
**Older tabs and rollbacks.** loro-repo 0.18.0 through 0.20.2 can open the same
|
|
338
|
+
database without knowing that checkpoints exist. Their purge or TTL expiry of a
|
|
339
|
+
named Flock, and their destructive `meta-snapshot` save, remove data but leave
|
|
340
|
+
the checkpoint. Each checkpoint is therefore bound to a lineage marker stored
|
|
341
|
+
with the data: a `lineage` field on the named Flock's update-queue record, and a
|
|
342
|
+
`lineage:<metaKey>` row in `meta-updates`. Older releases ignore both when
|
|
343
|
+
reading. They drop or delete the marker whenever they purge a named Flock,
|
|
344
|
+
rewrite its update queue, or replace the metadata base. If a capture finds the
|
|
345
|
+
marker missing or foreign, it discards the cursors and rotates the generation.
|
|
346
|
+
That room bootstraps once and merges with local state. Stores still bound to
|
|
347
|
+
the old generation are rejected in every tab. Healthy captures stay read-only.
|
|
348
|
+
Upgrading from 0.20.3, whose checkpoints carry no marker, bootstraps each Meta
|
|
349
|
+
and named Flock room once. The marker prevents a skipped range; it does not
|
|
350
|
+
bring back metadata that an older tab's `meta-snapshot` removed before it was
|
|
351
|
+
sent. See `docs/flock-persistence.md` ("IndexedDB lineage markers") for the
|
|
352
|
+
format, audit and remaining risks.
|
|
353
|
+
|
|
354
|
+
**Batched strict commits.** A strict transaction waits for a disk sync, so
|
|
355
|
+
metadata and named Flock payloads share them. One `repo.flush()` writes every
|
|
356
|
+
dirty named Flock and metadata in a single strict transaction, together with
|
|
357
|
+
the automatic Flock persistence its own writes scheduled. Concurrent
|
|
358
|
+
`persistFlockDocNow`/`persistMetaNow` barriers (the Streams barriers) and
|
|
359
|
+
concurrent flushes share commits too: payloads staged while a commit is in
|
|
360
|
+
flight go into the next one. A barrier still resolves only after the commit
|
|
361
|
+
that contains every change applied before its call. If a shared commit fails,
|
|
362
|
+
each resource is retried in its own transaction, and failed payloads stay
|
|
363
|
+
queued for the next barrier. `StorageAdapter.saveMany` is the optional atomic
|
|
364
|
+
batch hook behind this; `IndexedDBStorageAdaptor` and `SqliteRepoStore`
|
|
365
|
+
implement it, and other adapters keep receiving one `save` per payload.
|
|
366
|
+
|
|
367
|
+
`new IndexedDBStorageAdaptor({ durability })` accepts `"strict"` (the default),
|
|
368
|
+
`"relaxed"` or `"default"` for metadata, named Flock and checkpoint writes.
|
|
369
|
+
Anything but `"strict"` lets a crash or power loss drop writes the repo has
|
|
370
|
+
already acknowledged, in any order, so a saved Streams cursor can outlive the
|
|
371
|
+
data it covers and the resumed stream skips that history. Relax it only when
|
|
372
|
+
no Streams cursor is persisted or such a loss is recovered by other means.
|
|
373
|
+
|
|
374
|
+
LoroDoc persistence keeps its existing storage path. Its cursor is memory-only
|
|
375
|
+
in the example above. Pass `{ documentRemoteCursorStore }` as the factory's
|
|
376
|
+
second argument when an application explicitly wants its existing LoroDoc
|
|
377
|
+
cursor behavior; this does not extend the Flock recovery guarantee to LoroDoc.
|
|
378
|
+
|
|
237
379
|
## SQLite storage (Node.js / Electron)
|
|
238
380
|
|
|
239
|
-
`SqliteRepoStore` packs
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
381
|
+
`SqliteRepoStore` packs repo data, Streams checkpoints and cursor rows into a
|
|
382
|
+
single SQLite database file. Compared to `FileSystemStorageAdaptor`, it avoids
|
|
383
|
+
creating many small files for incremental updates, and it lets cursor advances
|
|
384
|
+
become single-row `UPDATE`s instead of rewriting a large JSON blob. Like
|
|
385
|
+
`IndexedDBStorageAdaptor`, it implements replica-bound Streams checkpoints, so
|
|
386
|
+
it works with the no-argument `createRepoStreamsPersistence(repo)`.
|
|
244
387
|
|
|
245
388
|
A single `SqliteRepoStore` instance exposes two adapters that share one
|
|
246
389
|
database connection:
|
|
247
390
|
|
|
248
|
-
- `store.storage` — a `StorageAdapter` for doc snapshots, doc updates,
|
|
249
|
-
metadata (Flock) snapshots/updates
|
|
250
|
-
-
|
|
251
|
-
|
|
391
|
+
- `store.storage` — a `StorageAdapter` for doc snapshots, doc updates,
|
|
392
|
+
metadata (Flock) snapshots/updates, named Flock documents, and the
|
|
393
|
+
replica-bound Meta/named Flock checkpoints.
|
|
394
|
+
- `store.cursorStore` — a `RemoteCursorStore` for LoroDoc rooms.
|
|
252
395
|
|
|
253
396
|
```ts
|
|
254
397
|
import { LoroRepo } from "loro-repo";
|
|
@@ -269,10 +412,10 @@ await repo.addTransport(
|
|
|
269
412
|
new StreamsTransportAdapter({
|
|
270
413
|
bucketId: "your-bucket",
|
|
271
414
|
auth: yourAuthProvider,
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
415
|
+
persistence: createRepoStreamsPersistence(repo, {
|
|
416
|
+
// Optional: keep LoroDoc cursors durable in `remote_cursors`.
|
|
417
|
+
documentRemoteCursorStore: store.cursorStore,
|
|
418
|
+
}),
|
|
276
419
|
}),
|
|
277
420
|
);
|
|
278
421
|
|
|
@@ -281,15 +424,40 @@ await repo.destroy();
|
|
|
281
424
|
store.close();
|
|
282
425
|
```
|
|
283
426
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
427
|
+
Meta and each named Flock are restored from one SQLite transaction that reads
|
|
428
|
+
the base snapshot, update log and checkpoint row together. A resource that
|
|
429
|
+
already has a checkpoint generation is read in a deferred (read) transaction;
|
|
430
|
+
a legacy or empty resource is recaptured while creating a random generation in
|
|
431
|
+
one `IMMEDIATE` transaction. The returned Flock only resumes from the cursor
|
|
432
|
+
captured with its own bytes: several `SqliteRepoStore` instances or processes
|
|
433
|
+
may open the same file (each with its own in-memory replica), and a newer
|
|
434
|
+
cursor written by one of them is never adopted by a replica that loaded older
|
|
435
|
+
data. Cursor saves run in `IMMEDIATE` transactions that check the generation
|
|
436
|
+
after the transport's data barrier has committed; the bound replica updates its
|
|
437
|
+
in-memory cursor only after that commit. With `wal: true`
|
|
438
|
+
(`synchronous=NORMAL`) a power loss may drop the latest commits, but it cannot
|
|
439
|
+
keep a later cursor commit without the earlier data commit.
|
|
440
|
+
|
|
441
|
+
Deleting a named Flock (`purgeFlockDoc`, or `purgeExpiredFlockDocs` for
|
|
442
|
+
`flockDocRetentionMs`) removes its data and checkpoint in one transaction, so
|
|
443
|
+
outstanding cursor stores in every process are rejected. Delete triggers in the
|
|
444
|
+
schema drop the checkpoint as well whenever Flock data is removed by anything
|
|
445
|
+
other than compaction, including loro-repo 0.20.3 and older after a rollback.
|
|
446
|
+
Do not drop `replica_checkpoints` by hand without dropping those triggers.
|
|
447
|
+
|
|
448
|
+
**Upgrading.** The `replica_checkpoints` table and its triggers are created on
|
|
449
|
+
open. Existing Meta/named Flock rows in `remote_cursors` are deliberately not
|
|
450
|
+
copied: they were never bound to the stored data. Each Meta and named Flock
|
|
451
|
+
room therefore bootstraps once after the upgrade and merges the snapshot with
|
|
452
|
+
local state, including unsent local writes. Old rows are left in place, so a
|
|
453
|
+
rolled-back binary still resumes from them (it replays; it cannot skip). The
|
|
454
|
+
LoroDoc path is unchanged.
|
|
455
|
+
|
|
456
|
+
The deprecated two-argument form,
|
|
457
|
+
`createRepoStreamsPersistence(repo, store.cursorStore)`, still works and still
|
|
458
|
+
stores Meta/named Flock cursors in `remote_cursors` with data-before-cursor
|
|
459
|
+
ordering, but without the atomic recovery described above. Sessions that keep
|
|
460
|
+
nothing across restarts can use `persistence: { mode: "ephemeral" }`.
|
|
293
461
|
|
|
294
462
|
`SqliteRepoStoreOptions`:
|
|
295
463
|
|
|
@@ -310,8 +478,13 @@ flock_doc_updates (id, flock_doc_id, update_data, created_at)
|
|
|
310
478
|
meta_snapshot (id=1, snapshot, updated_at)
|
|
311
479
|
meta_updates (id, update_data, created_at)
|
|
312
480
|
remote_cursors (stream_url PK, next_offset, server_lower_bound_version, updated_at_ms)
|
|
481
|
+
replica_checkpoints (resource_key PK, generation, cursors, updated_at_ms)
|
|
313
482
|
```
|
|
314
483
|
|
|
484
|
+
`resource_key` is `meta` or `flock:<flockDocId>`; `cursors` is a JSON array of
|
|
485
|
+
Streams cursors keyed by stream URL. Checkpoints are never written into
|
|
486
|
+
synchronized Flock metadata.
|
|
487
|
+
|
|
315
488
|
Post-sync Flock document persistence is version-vector based: no-op syncs do
|
|
316
489
|
not write, and changed state is appended as a JSON delta rather than a full
|
|
317
490
|
Flock file. Local changes that arrive while sync persistence is in flight are
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
const require_flock_snapshot = require('./flock-snapshot.cjs');
|
|
2
|
+
|
|
3
|
+
//#region src/internal/flock-persistence-journal.ts
|
|
4
|
+
const journalKey = Symbol.for("loro-repo.flock-persistence-journal.v1");
|
|
5
|
+
const MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;
|
|
6
|
+
function versionsEqual(left, right) {
|
|
7
|
+
const leftPeers = Object.keys(left);
|
|
8
|
+
const rightPeers = Object.keys(right);
|
|
9
|
+
if (leftPeers.length !== rightPeers.length) return false;
|
|
10
|
+
return leftPeers.every((peer) => {
|
|
11
|
+
const a = left[peer];
|
|
12
|
+
const b = right[peer];
|
|
13
|
+
return a?.physicalTime === b?.physicalTime && a?.logicalCounter === b?.logicalCounter;
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
/** Pending exact records and received payloads, acknowledged only after save. */
|
|
17
|
+
var FlockPersistenceJournal = class {
|
|
18
|
+
keys = /* @__PURE__ */ new Set();
|
|
19
|
+
payloads = [];
|
|
20
|
+
unsubscribe;
|
|
21
|
+
remoteKeys;
|
|
22
|
+
/**
|
|
23
|
+
* An inclusive version vector observed with no Flock event since. Exporting
|
|
24
|
+
* a version vector from wasm costs time proportional to the number of
|
|
25
|
+
* peers, and Streams delivers many batches that change nothing (echoes of
|
|
26
|
+
* our own appends, replayed catch-ups). Reusing the vector read after the
|
|
27
|
+
* previous import keeps such a batch to a single read.
|
|
28
|
+
*/
|
|
29
|
+
knownVersion;
|
|
30
|
+
constructor(flock, onDirty) {
|
|
31
|
+
this.flock = flock;
|
|
32
|
+
this.onDirty = onDirty;
|
|
33
|
+
flock[journalKey] = this;
|
|
34
|
+
this.unsubscribe = flock.subscribe((batch) => {
|
|
35
|
+
const keys = batch.source !== "local" && this.remoteKeys ? this.remoteKeys : this.keys;
|
|
36
|
+
for (const event of batch.events) keys.add(JSON.stringify(event.key));
|
|
37
|
+
if (batch.events.length) this.knownVersion = void 0;
|
|
38
|
+
if (keys === this.keys && batch.events.length) this.onDirty();
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
get hasPending() {
|
|
42
|
+
return this.keys.size > 0 || this.payloads.length > 0;
|
|
43
|
+
}
|
|
44
|
+
async receive(payloads, apply) {
|
|
45
|
+
const copies = payloads.filter((bytes) => bytes.length > 0).map((bytes) => bytes.slice());
|
|
46
|
+
let result;
|
|
47
|
+
const previousKeys = this.remoteKeys;
|
|
48
|
+
const remoteKeys = /* @__PURE__ */ new Set();
|
|
49
|
+
const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();
|
|
50
|
+
this.remoteKeys = remoteKeys;
|
|
51
|
+
try {
|
|
52
|
+
result = apply();
|
|
53
|
+
} catch (error) {
|
|
54
|
+
this.retainChangedKeys(remoteKeys);
|
|
55
|
+
throw error;
|
|
56
|
+
} finally {
|
|
57
|
+
this.remoteKeys = previousKeys;
|
|
58
|
+
}
|
|
59
|
+
let outcome;
|
|
60
|
+
try {
|
|
61
|
+
outcome = await result;
|
|
62
|
+
} catch (error) {
|
|
63
|
+
this.retainChangedKeys(remoteKeys);
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {
|
|
67
|
+
this.payloads.push(...copies.map((bytes) => ({
|
|
68
|
+
bytes,
|
|
69
|
+
failures: 0
|
|
70
|
+
})));
|
|
71
|
+
if (copies.length) this.onDirty();
|
|
72
|
+
}
|
|
73
|
+
return outcome;
|
|
74
|
+
}
|
|
75
|
+
versionUnchangedSince(previous) {
|
|
76
|
+
const current = this.flock.inclusiveVersion();
|
|
77
|
+
this.knownVersion = current;
|
|
78
|
+
return versionsEqual(previous, current);
|
|
79
|
+
}
|
|
80
|
+
retainChangedKeys(keys) {
|
|
81
|
+
for (const key of keys) this.keys.add(key);
|
|
82
|
+
if (keys.size) this.onDirty();
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Called within the owner's serialized persistence queue.
|
|
86
|
+
*
|
|
87
|
+
* Each round hands every pending payload to `write` at once, so a storage
|
|
88
|
+
* adapter can commit them in one transaction. Only the prefix `write`
|
|
89
|
+
* reports durable is acknowledged.
|
|
90
|
+
*/
|
|
91
|
+
async flush(write) {
|
|
92
|
+
this.flock.commit();
|
|
93
|
+
while (this.hasPending) {
|
|
94
|
+
if (this.keys.size) await this.stageChangedKeys();
|
|
95
|
+
const { durable, error: exactError } = await write(this.payloads.map((payload$1) => payload$1.bytes));
|
|
96
|
+
this.payloads.splice(0, durable);
|
|
97
|
+
if (exactError === void 0) continue;
|
|
98
|
+
const payload = this.payloads[0];
|
|
99
|
+
payload.failures += 1;
|
|
100
|
+
if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) throw exactError;
|
|
101
|
+
const coveredPayloads = this.payloads.length;
|
|
102
|
+
const fallback = await write([await require_flock_snapshot.exportFlockFileSnapshot(this.flock)]);
|
|
103
|
+
if (fallback.error !== void 0) throw new AggregateError([exactError, fallback.error], "Failed to persist a Flock update and its snapshot fallback");
|
|
104
|
+
this.payloads.splice(0, coveredPayloads);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
async stageChangedKeys() {
|
|
108
|
+
const capturedKeys = [...this.keys];
|
|
109
|
+
for (const key of capturedKeys) this.keys.delete(key);
|
|
110
|
+
try {
|
|
111
|
+
const bundle = {
|
|
112
|
+
version: 0,
|
|
113
|
+
entries: {}
|
|
114
|
+
};
|
|
115
|
+
let useSnapshot = false;
|
|
116
|
+
for (const key of capturedKeys) {
|
|
117
|
+
let entry;
|
|
118
|
+
try {
|
|
119
|
+
entry = this.flock.getEntry(JSON.parse(key));
|
|
120
|
+
} catch {
|
|
121
|
+
useSnapshot = true;
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
if (!entry) {
|
|
125
|
+
useSnapshot = true;
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
const { physicalTime, logicalCounter, peerId } = entry.clock;
|
|
129
|
+
bundle.entries[key] = {
|
|
130
|
+
c: `${physicalTime},${logicalCounter},${peerId}`,
|
|
131
|
+
...entry.data === void 0 ? {} : { d: entry.data },
|
|
132
|
+
...Object.keys(entry.metadata).length ? { m: entry.metadata } : {}
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
const bytes = useSnapshot ? await require_flock_snapshot.exportFlockFileSnapshot(this.flock) : require_flock_snapshot.encodeMetaJsonUpdate(bundle);
|
|
136
|
+
this.payloads.push({
|
|
137
|
+
bytes,
|
|
138
|
+
failures: 0
|
|
139
|
+
});
|
|
140
|
+
} catch (error) {
|
|
141
|
+
for (const key of capturedKeys) this.keys.add(key);
|
|
142
|
+
throw error;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
destroy() {
|
|
146
|
+
this.unsubscribe();
|
|
147
|
+
if (this.flock[journalKey] === this) delete this.flock[journalKey];
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */
|
|
151
|
+
async function receiveFlockPayloads(flock, payloads, apply) {
|
|
152
|
+
const journal = flock[journalKey];
|
|
153
|
+
return journal ? journal.receive(payloads, apply) : await apply();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
//#endregion
|
|
157
|
+
Object.defineProperty(exports, 'FlockPersistenceJournal', {
|
|
158
|
+
enumerable: true,
|
|
159
|
+
get: function () {
|
|
160
|
+
return FlockPersistenceJournal;
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
Object.defineProperty(exports, 'receiveFlockPayloads', {
|
|
164
|
+
enumerable: true,
|
|
165
|
+
get: function () {
|
|
166
|
+
return receiveFlockPayloads;
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
//# sourceMappingURL=flock-persistence-journal.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flock-persistence-journal.cjs","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","payload","exportFlockFileSnapshot","bundle: ExportBundle","encodeMetaJsonUpdate"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\n/**\n * Persists `updates` in order. Reports how many leading updates are durable\n * and, if not all of them are, the failure. It must not reject.\n */\nexport type JournalWrite = (\n updates: readonly Uint8Array[],\n) => Promise<{ readonly durable: number; readonly error?: unknown }>;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n /**\n * An inclusive version vector observed with no Flock event since. Exporting\n * a version vector from wasm costs time proportional to the number of\n * peers, and Streams delivers many batches that change nothing (echoes of\n * our own appends, replayed catch-ups). Reusing the vector read after the\n * previous import keeps such a batch to a single read.\n */\n private knownVersion?: VersionVector;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (batch.events.length) this.knownVersion = undefined;\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n // The inclusive vector only grows during the process lifetime, so a\n // cached vector that is stale because of an untracked import can only\n // cause a redundant save, never a skipped one.\n const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes. An import can also advance the version without\n // emitting events (an entry that loses last-writer-wins), so compare.\n if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private versionUnchangedSince(previous: VersionVector): boolean {\n const current = this.flock.inclusiveVersion();\n this.knownVersion = current;\n return versionsEqual(previous, current);\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /**\n * Called within the owner's serialized persistence queue.\n *\n * Each round hands every pending payload to `write` at once, so a storage\n * adapter can commit them in one transaction. Only the prefix `write`\n * reports durable is acknowledged.\n */\n async flush(write: JournalWrite): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const { durable, error: exactError } = await write(\n this.payloads.map((payload) => payload.bytes),\n );\n // Only this loop removes payloads, so the written ones are still the\n // queue's prefix. New imports/local changes during the write remain\n // behind them.\n this.payloads.splice(0, durable);\n if (exactError === undefined) continue;\n\n const payload = this.payloads[0]!;\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n const fallback = await write([snapshot]);\n if (fallback.error !== undefined) {\n throw new AggregateError(\n [exactError, fallback.error],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAUxC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;;;;;;;;CAQR,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,MAAM,OAAO,OAAQ,MAAK,eAAe;AAC7C,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EAIpC,MAAM,kBAAkB,KAAK,gBAAgB,KAAK,MAAM,kBAAkB;AAC1E,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAQR,MAAI,WAAW,OAAO,KAAK,CAAC,KAAK,sBAAsB,gBAAgB,EAAE;AACvE,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,sBAAsB,UAAkC;EAC9D,MAAM,UAAU,KAAK,MAAM,kBAAkB;AAC7C,OAAK,eAAe;AACpB,SAAO,cAAc,UAAU,QAAQ;;CAGzC,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;;;;;;;CAU/B,MAAM,MAAM,OAAoC;AAC9C,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,EAAE,SAAS,OAAO,eAAe,MAAM,MAC3C,KAAK,SAAS,KAAK,cAAYC,UAAQ,MAAM,CAC9C;AAID,QAAK,SAAS,OAAO,GAAG,QAAQ;AAChC,OAAI,eAAe,OAAW;GAE9B,MAAM,UAAU,KAAK,SAAS;AAC9B,WAAQ,YAAY;AAIpB,OAAI,QAAQ,WAAW,gCACrB,OAAM;GAUR,MAAM,kBAAkB,KAAK,SAAS;GAEtC,MAAM,WAAW,MAAM,MAAM,CADZ,MAAMC,+CAAwB,KAAK,MAAM,CACnB,CAAC;AACxC,OAAI,SAAS,UAAU,OACrB,OAAM,IAAI,eACR,CAAC,YAAY,SAAS,MAAM,EAC5B,6DACD;AAEH,QAAK,SAAS,OAAO,GAAG,gBAAgB;;;CAI5C,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAMD,+CAAwB,KAAK,MAAM,GACzCE,4CAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { r as encodeMetaJsonUpdate, t as exportFlockFileSnapshot } from "./flock-snapshot.js";
|
|
2
|
+
|
|
3
|
+
//#region src/internal/flock-persistence-journal.ts
|
|
4
|
+
const journalKey = Symbol.for("loro-repo.flock-persistence-journal.v1");
|
|
5
|
+
const MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;
|
|
6
|
+
function versionsEqual(left, right) {
|
|
7
|
+
const leftPeers = Object.keys(left);
|
|
8
|
+
const rightPeers = Object.keys(right);
|
|
9
|
+
if (leftPeers.length !== rightPeers.length) return false;
|
|
10
|
+
return leftPeers.every((peer) => {
|
|
11
|
+
const a = left[peer];
|
|
12
|
+
const b = right[peer];
|
|
13
|
+
return a?.physicalTime === b?.physicalTime && a?.logicalCounter === b?.logicalCounter;
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
/** Pending exact records and received payloads, acknowledged only after save. */
|
|
17
|
+
var FlockPersistenceJournal = class {
|
|
18
|
+
keys = /* @__PURE__ */ new Set();
|
|
19
|
+
payloads = [];
|
|
20
|
+
unsubscribe;
|
|
21
|
+
remoteKeys;
|
|
22
|
+
/**
|
|
23
|
+
* An inclusive version vector observed with no Flock event since. Exporting
|
|
24
|
+
* a version vector from wasm costs time proportional to the number of
|
|
25
|
+
* peers, and Streams delivers many batches that change nothing (echoes of
|
|
26
|
+
* our own appends, replayed catch-ups). Reusing the vector read after the
|
|
27
|
+
* previous import keeps such a batch to a single read.
|
|
28
|
+
*/
|
|
29
|
+
knownVersion;
|
|
30
|
+
constructor(flock, onDirty) {
|
|
31
|
+
this.flock = flock;
|
|
32
|
+
this.onDirty = onDirty;
|
|
33
|
+
flock[journalKey] = this;
|
|
34
|
+
this.unsubscribe = flock.subscribe((batch) => {
|
|
35
|
+
const keys = batch.source !== "local" && this.remoteKeys ? this.remoteKeys : this.keys;
|
|
36
|
+
for (const event of batch.events) keys.add(JSON.stringify(event.key));
|
|
37
|
+
if (batch.events.length) this.knownVersion = void 0;
|
|
38
|
+
if (keys === this.keys && batch.events.length) this.onDirty();
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
get hasPending() {
|
|
42
|
+
return this.keys.size > 0 || this.payloads.length > 0;
|
|
43
|
+
}
|
|
44
|
+
async receive(payloads, apply) {
|
|
45
|
+
const copies = payloads.filter((bytes) => bytes.length > 0).map((bytes) => bytes.slice());
|
|
46
|
+
let result;
|
|
47
|
+
const previousKeys = this.remoteKeys;
|
|
48
|
+
const remoteKeys = /* @__PURE__ */ new Set();
|
|
49
|
+
const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();
|
|
50
|
+
this.remoteKeys = remoteKeys;
|
|
51
|
+
try {
|
|
52
|
+
result = apply();
|
|
53
|
+
} catch (error) {
|
|
54
|
+
this.retainChangedKeys(remoteKeys);
|
|
55
|
+
throw error;
|
|
56
|
+
} finally {
|
|
57
|
+
this.remoteKeys = previousKeys;
|
|
58
|
+
}
|
|
59
|
+
let outcome;
|
|
60
|
+
try {
|
|
61
|
+
outcome = await result;
|
|
62
|
+
} catch (error) {
|
|
63
|
+
this.retainChangedKeys(remoteKeys);
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {
|
|
67
|
+
this.payloads.push(...copies.map((bytes) => ({
|
|
68
|
+
bytes,
|
|
69
|
+
failures: 0
|
|
70
|
+
})));
|
|
71
|
+
if (copies.length) this.onDirty();
|
|
72
|
+
}
|
|
73
|
+
return outcome;
|
|
74
|
+
}
|
|
75
|
+
versionUnchangedSince(previous) {
|
|
76
|
+
const current = this.flock.inclusiveVersion();
|
|
77
|
+
this.knownVersion = current;
|
|
78
|
+
return versionsEqual(previous, current);
|
|
79
|
+
}
|
|
80
|
+
retainChangedKeys(keys) {
|
|
81
|
+
for (const key of keys) this.keys.add(key);
|
|
82
|
+
if (keys.size) this.onDirty();
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Called within the owner's serialized persistence queue.
|
|
86
|
+
*
|
|
87
|
+
* Each round hands every pending payload to `write` at once, so a storage
|
|
88
|
+
* adapter can commit them in one transaction. Only the prefix `write`
|
|
89
|
+
* reports durable is acknowledged.
|
|
90
|
+
*/
|
|
91
|
+
async flush(write) {
|
|
92
|
+
this.flock.commit();
|
|
93
|
+
while (this.hasPending) {
|
|
94
|
+
if (this.keys.size) await this.stageChangedKeys();
|
|
95
|
+
const { durable, error: exactError } = await write(this.payloads.map((payload$1) => payload$1.bytes));
|
|
96
|
+
this.payloads.splice(0, durable);
|
|
97
|
+
if (exactError === void 0) continue;
|
|
98
|
+
const payload = this.payloads[0];
|
|
99
|
+
payload.failures += 1;
|
|
100
|
+
if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) throw exactError;
|
|
101
|
+
const coveredPayloads = this.payloads.length;
|
|
102
|
+
const fallback = await write([await exportFlockFileSnapshot(this.flock)]);
|
|
103
|
+
if (fallback.error !== void 0) throw new AggregateError([exactError, fallback.error], "Failed to persist a Flock update and its snapshot fallback");
|
|
104
|
+
this.payloads.splice(0, coveredPayloads);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
async stageChangedKeys() {
|
|
108
|
+
const capturedKeys = [...this.keys];
|
|
109
|
+
for (const key of capturedKeys) this.keys.delete(key);
|
|
110
|
+
try {
|
|
111
|
+
const bundle = {
|
|
112
|
+
version: 0,
|
|
113
|
+
entries: {}
|
|
114
|
+
};
|
|
115
|
+
let useSnapshot = false;
|
|
116
|
+
for (const key of capturedKeys) {
|
|
117
|
+
let entry;
|
|
118
|
+
try {
|
|
119
|
+
entry = this.flock.getEntry(JSON.parse(key));
|
|
120
|
+
} catch {
|
|
121
|
+
useSnapshot = true;
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
if (!entry) {
|
|
125
|
+
useSnapshot = true;
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
const { physicalTime, logicalCounter, peerId } = entry.clock;
|
|
129
|
+
bundle.entries[key] = {
|
|
130
|
+
c: `${physicalTime},${logicalCounter},${peerId}`,
|
|
131
|
+
...entry.data === void 0 ? {} : { d: entry.data },
|
|
132
|
+
...Object.keys(entry.metadata).length ? { m: entry.metadata } : {}
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
const bytes = useSnapshot ? await exportFlockFileSnapshot(this.flock) : encodeMetaJsonUpdate(bundle);
|
|
136
|
+
this.payloads.push({
|
|
137
|
+
bytes,
|
|
138
|
+
failures: 0
|
|
139
|
+
});
|
|
140
|
+
} catch (error) {
|
|
141
|
+
for (const key of capturedKeys) this.keys.add(key);
|
|
142
|
+
throw error;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
destroy() {
|
|
146
|
+
this.unsubscribe();
|
|
147
|
+
if (this.flock[journalKey] === this) delete this.flock[journalKey];
|
|
148
|
+
}
|
|
149
|
+
};
|
|
150
|
+
/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */
|
|
151
|
+
async function receiveFlockPayloads(flock, payloads, apply) {
|
|
152
|
+
const journal = flock[journalKey];
|
|
153
|
+
return journal ? journal.receive(payloads, apply) : await apply();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
//#endregion
|
|
157
|
+
export { receiveFlockPayloads as n, FlockPersistenceJournal as t };
|
|
158
|
+
//# sourceMappingURL=flock-persistence-journal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flock-persistence-journal.js","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","payload","bundle: ExportBundle"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\n/**\n * Persists `updates` in order. Reports how many leading updates are durable\n * and, if not all of them are, the failure. It must not reject.\n */\nexport type JournalWrite = (\n updates: readonly Uint8Array[],\n) => Promise<{ readonly durable: number; readonly error?: unknown }>;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n /**\n * An inclusive version vector observed with no Flock event since. Exporting\n * a version vector from wasm costs time proportional to the number of\n * peers, and Streams delivers many batches that change nothing (echoes of\n * our own appends, replayed catch-ups). Reusing the vector read after the\n * previous import keeps such a batch to a single read.\n */\n private knownVersion?: VersionVector;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (batch.events.length) this.knownVersion = undefined;\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n // The inclusive vector only grows during the process lifetime, so a\n // cached vector that is stale because of an untracked import can only\n // cause a redundant save, never a skipped one.\n const previousVersion = this.knownVersion ?? this.flock.inclusiveVersion();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes. An import can also advance the version without\n // emitting events (an entry that loses last-writer-wins), so compare.\n if (remoteKeys.size > 0 || !this.versionUnchangedSince(previousVersion)) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private versionUnchangedSince(previous: VersionVector): boolean {\n const current = this.flock.inclusiveVersion();\n this.knownVersion = current;\n return versionsEqual(previous, current);\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /**\n * Called within the owner's serialized persistence queue.\n *\n * Each round hands every pending payload to `write` at once, so a storage\n * adapter can commit them in one transaction. Only the prefix `write`\n * reports durable is acknowledged.\n */\n async flush(write: JournalWrite): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const { durable, error: exactError } = await write(\n this.payloads.map((payload) => payload.bytes),\n );\n // Only this loop removes payloads, so the written ones are still the\n // queue's prefix. New imports/local changes during the write remain\n // behind them.\n this.payloads.splice(0, durable);\n if (exactError === undefined) continue;\n\n const payload = this.payloads[0]!;\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n const fallback = await write([snapshot]);\n if (fallback.error !== undefined) {\n throw new AggregateError(\n [exactError, fallback.error],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAUxC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;;;;;;;;CAQR,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,MAAM,OAAO,OAAQ,MAAK,eAAe;AAC7C,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EAIpC,MAAM,kBAAkB,KAAK,gBAAgB,KAAK,MAAM,kBAAkB;AAC1E,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAQR,MAAI,WAAW,OAAO,KAAK,CAAC,KAAK,sBAAsB,gBAAgB,EAAE;AACvE,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,sBAAsB,UAAkC;EAC9D,MAAM,UAAU,KAAK,MAAM,kBAAkB;AAC7C,OAAK,eAAe;AACpB,SAAO,cAAc,UAAU,QAAQ;;CAGzC,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;;;;;;;CAU/B,MAAM,MAAM,OAAoC;AAC9C,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,EAAE,SAAS,OAAO,eAAe,MAAM,MAC3C,KAAK,SAAS,KAAK,cAAYC,UAAQ,MAAM,CAC9C;AAID,QAAK,SAAS,OAAO,GAAG,QAAQ;AAChC,OAAI,eAAe,OAAW;GAE9B,MAAM,UAAU,KAAK,SAAS;AAC9B,WAAQ,YAAY;AAIpB,OAAI,QAAQ,WAAW,gCACrB,OAAM;GAUR,MAAM,kBAAkB,KAAK,SAAS;GAEtC,MAAM,WAAW,MAAM,MAAM,CADZ,MAAM,wBAAwB,KAAK,MAAM,CACnB,CAAC;AACxC,OAAI,SAAS,UAAU,OACrB,OAAM,IAAI,eACR,CAAC,YAAY,SAAS,MAAM,EAC5B,6DACD;AAEH,QAAK,SAAS,OAAO,GAAG,gBAAgB;;;CAI5C,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAM,wBAAwB,KAAK,MAAM,GACzC,qBAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|