loro-repo 0.19.1 → 0.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +228 -37
- package/dist/flock-persistence-journal.cjs +156 -0
- package/dist/flock-persistence-journal.cjs.map +1 -0
- package/dist/flock-persistence-journal.js +145 -0
- package/dist/flock-persistence-journal.js.map +1 -0
- package/dist/flock-snapshot.cjs +91 -7
- package/dist/flock-snapshot.cjs.map +1 -1
- package/dist/flock-snapshot.js +86 -8
- package/dist/flock-snapshot.js.map +1 -1
- package/dist/index.cjs +1259 -365
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +151 -30
- package/dist/index.d.ts +151 -30
- package/dist/index.js +1249 -360
- package/dist/index.js.map +1 -1
- package/dist/storage/filesystem.cjs +15 -7
- package/dist/storage/filesystem.cjs.map +1 -1
- package/dist/storage/filesystem.d.cts +1 -1
- package/dist/storage/filesystem.d.ts +1 -1
- package/dist/storage/filesystem.js +16 -8
- package/dist/storage/filesystem.js.map +1 -1
- package/dist/storage/indexeddb.cjs +389 -79
- package/dist/storage/indexeddb.cjs.map +1 -1
- package/dist/storage/indexeddb.d.cts +51 -6
- package/dist/storage/indexeddb.d.ts +51 -6
- package/dist/storage/indexeddb.js +389 -81
- package/dist/storage/indexeddb.js.map +1 -1
- package/dist/storage/sqlite.cjs +24 -5
- package/dist/storage/sqlite.cjs.map +1 -1
- package/dist/storage/sqlite.d.cts +3 -2
- package/dist/storage/sqlite.d.ts +2 -1
- package/dist/storage/sqlite.js +25 -6
- package/dist/storage/sqlite.js.map +1 -1
- package/dist/transport/broadcast-channel.d.cts +1 -1
- package/dist/transport/broadcast-channel.d.ts +1 -1
- package/dist/transport/streams.cjs +1139 -163
- package/dist/transport/streams.cjs.map +1 -1
- package/dist/transport/streams.d.cts +397 -15
- package/dist/transport/streams.d.ts +397 -15
- package/dist/transport/streams.js +1137 -165
- package/dist/transport/streams.js.map +1 -1
- package/dist/transport/websocket.d.cts +1 -1
- package/dist/transport/websocket.d.ts +1 -1
- package/dist/types.cjs +131 -0
- package/dist/types.cjs.map +1 -0
- package/dist/types.d.cts +547 -29
- package/dist/types.d.ts +547 -29
- package/dist/types.js +95 -0
- package/dist/types.js.map +1 -0
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -43,9 +43,16 @@ await repo.unloadDoc("note:welcome");
|
|
|
43
43
|
## Using the API
|
|
44
44
|
|
|
45
45
|
- **Create a repo** with `await LoroRepo.create<Meta>({ transportAdapter?, storageAdapter? })`.
|
|
46
|
-
- **
|
|
46
|
+
- **Add transports later** with `await repo.addTransport(id, adapter, options?)`.
|
|
47
47
|
- **Check adapter availability** via `repo.hasTransport()` and `repo.hasStorage()`.
|
|
48
48
|
- **Choose sync lanes** using `repo.sync({ scope: "meta" | "doc" | "full", docIds? })`.
|
|
49
|
+
The default is `"meta"` — metadata is bounded, document bodies are not, so
|
|
50
|
+
pulling bodies is opt-in. Passing `docIds`/`flockDocIds` widens the scope on
|
|
51
|
+
its own. Bodies planned automatically skip soft-deleted documents and are
|
|
52
|
+
released after the sync unless `retainLoadedDocs` says otherwise.
|
|
53
|
+
- **Acquire replicas** with `repo.acquireDoc(id)` / `repo.acquireFlockDoc(id)`
|
|
54
|
+
when a document is used across awaits; the lease keeps the repository from
|
|
55
|
+
evicting it, and must be `release()`d.
|
|
49
56
|
- **Work with docs** through `openPersistedDoc`, `openDetachedDoc`, `joinDocRoom`, `unloadDoc`, and `flush`.
|
|
50
57
|
- **React to changes** with `repo.watch(listener, { docIds, kinds, metadataFields, by })`.
|
|
51
58
|
- **Shutdown cleanly** by calling `await repo.destroy()`.
|
|
@@ -91,8 +98,9 @@ await repo.removeTransport("cloud");
|
|
|
91
98
|
Key points:
|
|
92
99
|
|
|
93
100
|
- `addTransport(id, adapter, options?)` / `removeTransport(id, { close? })`
|
|
94
|
-
register and unregister named transports; `
|
|
95
|
-
|
|
101
|
+
register and unregister named transports; the `transportAdapter`
|
|
102
|
+
construction option registers a single transport under
|
|
103
|
+
`DEFAULT_TRANSPORT_ID`.
|
|
96
104
|
- `resolveRoomTransports({ kind: "meta" | "doc" | "flock-doc", id })` decides
|
|
97
105
|
which transports join each room. It may return ids that are not registered
|
|
98
106
|
yet (they attach when added) or an empty list (the room stays pending until
|
|
@@ -140,13 +148,204 @@ Key points:
|
|
|
140
148
|
- `FileSystemStorageAdaptor` (`src/storage/filesystem.ts`)
|
|
141
149
|
- `SqliteRepoStore` (`src/storage/sqlite.ts`, Node.js only)
|
|
142
150
|
|
|
151
|
+
## Classifying storage failures
|
|
152
|
+
|
|
153
|
+
`IndexedDBStorageAdaptor` wraps backend failures in a `RepoStorageError`
|
|
154
|
+
carrying a stable `code`, so callers do not have to pattern-match browser error
|
|
155
|
+
text (which differs across engines, versions, and UI locales). Misusing the
|
|
156
|
+
adaptor — an unsupported payload type, a call after `close()` — still throws a
|
|
157
|
+
plain `Error`: those are bugs in the caller, not storage failures.
|
|
158
|
+
|
|
159
|
+
```ts
|
|
160
|
+
import { RepoStorageError } from "loro-repo";
|
|
161
|
+
|
|
162
|
+
try {
|
|
163
|
+
await repo.flush();
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if (error instanceof RepoStorageError) {
|
|
166
|
+
switch (error.code) {
|
|
167
|
+
case "quota":
|
|
168
|
+
// Out of disk / origin quota. Retrying will keep failing until space
|
|
169
|
+
// is freed — tell the user, and stop accepting writes.
|
|
170
|
+
break;
|
|
171
|
+
case "unavailable":
|
|
172
|
+
// The store is not usable right now: connection closing, backing
|
|
173
|
+
// store unreachable, open blocked by another connection, or the
|
|
174
|
+
// context is not allowed to use IndexedDB (sandboxed iframe,
|
|
175
|
+
// opaque origin).
|
|
176
|
+
break;
|
|
177
|
+
case "unknown":
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
`error.cause` is the original `DOMException` and `error.causeName` is its
|
|
185
|
+
`name` (`"QuotaExceededError"`, `"InvalidStateError"`, …), which the wrapped
|
|
186
|
+
`message` does not include.
|
|
187
|
+
|
|
188
|
+
Three caveats worth knowing:
|
|
189
|
+
|
|
190
|
+
- **`code` is best-effort for two error names.** `QuotaExceededError` and
|
|
191
|
+
`SecurityError` mean one thing each, so the name alone classifies them.
|
|
192
|
+
`InvalidStateError` and `UnknownError` are overloaded, so the adaptor also
|
|
193
|
+
probes the message — and that text is Chromium's. On other engines those two
|
|
194
|
+
fall back to `"unknown"` rather than being guessed into an actionable bucket.
|
|
195
|
+
- **Background persistence logs, it does not throw.** Debounced auto-saves
|
|
196
|
+
report failures through `console.error` and stay dirty for the next attempt.
|
|
197
|
+
To act on a `code`, `await repo.flush()` (or a load) where the error reaches
|
|
198
|
+
you.
|
|
199
|
+
- **Loading never fails for a write-related reason.** Folding queued updates
|
|
200
|
+
back into a snapshot is an optimization; if that write fails, the document is
|
|
201
|
+
still returned and the updates stay queued for the next load. The compaction
|
|
202
|
+
entry points (`compactMeta`, `compactFlockDoc`) are the opposite: the write
|
|
203
|
+
*is* the request, so they reject.
|
|
204
|
+
|
|
205
|
+
A failed *open* is retried on the next call, so a transient `blocked` (another
|
|
206
|
+
tab still holding the previous version) does not permanently disable the
|
|
207
|
+
adaptor. A *live* connection that has started failing is a different matter: the
|
|
208
|
+
adaptor deliberately does **not** reopen it on its own, because a reopen usually
|
|
209
|
+
fails for the same reason and silent self-healing makes it harder for an
|
|
210
|
+
application to fail closed. That recovery is the caller's decision.
|
|
211
|
+
|
|
212
|
+
## Per-room Streams payload protection
|
|
213
|
+
|
|
214
|
+
`StreamsTransportAdapter` can select one streams-crdt provider config for each
|
|
215
|
+
durable room. The resolver receives only logical room identity:
|
|
216
|
+
|
|
217
|
+
- Meta Flock: `{ kind: "meta" }`
|
|
218
|
+
- Loro document: `{ kind: "doc", docId }`
|
|
219
|
+
- named Flock document: `{ kind: "flock", flockDocId }`
|
|
220
|
+
|
|
221
|
+
The resolver never receives the Durable Streams bucket or stream ID. The
|
|
222
|
+
adapter privately maps its structured `{ bucketId, streamId }` address to the
|
|
223
|
+
opaque `streamUrl` expected by streams-crdt. That URL is transport routing, not
|
|
224
|
+
cryptographic identity. The provider returned for a logical room owns key
|
|
225
|
+
lookup and AEAD implementation, while the adapter always binds
|
|
226
|
+
streams-crdt's envelope AAD to a caller-supplied stable application namespace
|
|
227
|
+
plus the logical room identity. The binding is versioned, domain-separated,
|
|
228
|
+
and length-prefixed; it never contains the bucket, stream ID, or URL.
|
|
229
|
+
|
|
230
|
+
```ts
|
|
231
|
+
const transport = new StreamsTransportAdapter({
|
|
232
|
+
bucketId: "workspace-data",
|
|
233
|
+
metaStreamId: "workspace:123:meta",
|
|
234
|
+
docStreamId: (docId) => `workspace:123:doc:${docId}`,
|
|
235
|
+
flockDocStreamId: (flockDocId) => `workspace:123:flock:${flockDocId}`,
|
|
236
|
+
auth: yourAuthProvider,
|
|
237
|
+
// Stable application/workspace identity, not a physical stream route.
|
|
238
|
+
payloadProtectionNamespace: "acme-notes/workspace-123",
|
|
239
|
+
payloadProtection: (room) => ({
|
|
240
|
+
mode: "protected",
|
|
241
|
+
config: lodyCrypto.providerConfigForRoom(room),
|
|
242
|
+
}),
|
|
243
|
+
});
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
Once `payloadProtection` is present, the resolver is total. Every room must
|
|
247
|
+
return either `{ mode: "protected", config }` or `{ mode: "plaintext" }`.
|
|
248
|
+
Returning `undefined`, throwing, or returning an invalid selection fails that
|
|
249
|
+
room before a `StreamsCrdt` is created. Protected selections also set upstream
|
|
250
|
+
`payloadProtectionRequired` and force `readPolicy: "encrypted-only"` plus
|
|
251
|
+
`writePolicy: "encrypt"`. A protected config cannot opt into
|
|
252
|
+
`allow-plaintext` reads or plaintext writes. Use the honest top-level
|
|
253
|
+
`{ mode: "plaintext" }` selection for a public room. Omit `payloadProtection`
|
|
254
|
+
entirely to preserve the existing plaintext API and wire behavior.
|
|
255
|
+
|
|
256
|
+
The resolved config and the provider instance used by `seal()` are fixed for
|
|
257
|
+
one room session. In particular, `seal()` must keep using that session's write
|
|
258
|
+
epoch; it must not switch dynamically through a mutable vault handle. `open()`
|
|
259
|
+
may use the opaque authenticated header to look up historical read keys.
|
|
260
|
+
|
|
261
|
+
To replace the write provider safely without recreating the whole repo, the
|
|
262
|
+
application owns a grouped barrier:
|
|
263
|
+
|
|
264
|
+
1. stop producing writes for the affected room set;
|
|
265
|
+
2. await one `Promise.all` of `waitUntilSynced()` for every active subscription;
|
|
266
|
+
3. only after every wait succeeds, unsubscribe every lease in that set;
|
|
267
|
+
4. await `restartRoomSessions(rooms)` to join any in-progress teardown, then
|
|
268
|
+
update the application-owned config and rejoin.
|
|
269
|
+
|
|
270
|
+
The final live lease already starts that room's teardown, so
|
|
271
|
+
`restartRoomSessions` is not the atomic drain barrier. It refuses any room that
|
|
272
|
+
still has a live lease, pending live acquisition, or in-flight one-shot sync;
|
|
273
|
+
it also fences new joins and one-shot syncs until the selected teardown
|
|
274
|
+
completes. A successful return therefore leaves no old-generation session that
|
|
275
|
+
can reappear before the caller switches config. The restart preserves CRDT
|
|
276
|
+
state and remote cursors and makes the next sync/join call the resolver again.
|
|
277
|
+
If any `waitUntilSynced()` fails, keep all leases and the old write providers
|
|
278
|
+
in place; do not partially release the room set.
|
|
279
|
+
|
|
280
|
+
Use a non-owning provider handle into an application-owned vault. The adapter
|
|
281
|
+
wraps the provider only to add its logical-room AAD and fixed encrypted-only
|
|
282
|
+
policies. Resolver-configured one-shot sessions release that wrapper after each
|
|
283
|
+
operation, and live sessions release it after their final lease closes. The
|
|
284
|
+
adapter does not inspect or dispose provider state, and it never stores it in
|
|
285
|
+
repo metadata, documents, cursor storage, logs, or surfaced error causes.
|
|
286
|
+
|
|
287
|
+
The same protected `StreamsCrdt` instance handles updates, snapshots,
|
|
288
|
+
bootstrap/catch-up, and live reads. A protection failure therefore occurs before
|
|
289
|
+
CRDT import and before `beforeRemoteCursorSave`; the adapter keeps the existing
|
|
290
|
+
"persist CRDT state, then advance cursor" boundary. Each room owns an independent
|
|
291
|
+
instance, so one room's failure does not stop other rooms.
|
|
292
|
+
|
|
293
|
+
This API requires `@loro-dev/streams-crdt@^0.15.0`, specifically its common
|
|
294
|
+
protected-payload boundary and `payloadProtectionRequired` guard. No 0.12.x
|
|
295
|
+
encryption compatibility shim is retained; the adapter passes the published
|
|
296
|
+
0.15 opaque `streamUrl` transport target directly.
|
|
297
|
+
|
|
298
|
+
## Replica-bound Streams checkpoints (IndexedDB)
|
|
299
|
+
|
|
300
|
+
For browser Flock rooms, let the repo's IndexedDB storage own checkpoints:
|
|
301
|
+
|
|
302
|
+
```ts
|
|
303
|
+
import { LoroRepo } from "loro-repo";
|
|
304
|
+
import { IndexedDBStorageAdaptor } from "loro-repo/storage/indexeddb";
|
|
305
|
+
import {
|
|
306
|
+
StreamsTransportAdapter,
|
|
307
|
+
createRepoStreamsPersistence,
|
|
308
|
+
} from "loro-repo/transport/streams";
|
|
309
|
+
|
|
310
|
+
const repo = await LoroRepo.create({
|
|
311
|
+
storageAdapter: new IndexedDBStorageAdaptor({ dbName: "workspace-42" }),
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
await repo.addTransport(
|
|
315
|
+
"streams",
|
|
316
|
+
new StreamsTransportAdapter({
|
|
317
|
+
bucketId: "your-bucket",
|
|
318
|
+
auth: yourAuthProvider,
|
|
319
|
+
persistence: createRepoStreamsPersistence(repo),
|
|
320
|
+
}),
|
|
321
|
+
);
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
The storage adapter restores each Meta/named Flock snapshot, update log and
|
|
325
|
+
Streams checkpoint from one capture transaction. Existing generations use a
|
|
326
|
+
readonly transaction; a missing generation is created while recapturing the
|
|
327
|
+
resource in one readwrite transaction. A live Flock only sees the cursor
|
|
328
|
+
captured with its own restored bytes, even when another tab writes a newer
|
|
329
|
+
cursor to the same database later. Strict Flock data commits precede the strict
|
|
330
|
+
checkpoint transaction, while LoroDoc body writes retain normal IndexedDB
|
|
331
|
+
durability. Named Flock
|
|
332
|
+
deletion removes its checkpoint record with the data and invalidates outstanding
|
|
333
|
+
cursor stores. Version changes from another tab are handled by reopening the
|
|
334
|
+
database before subsequent checkpoint mutations.
|
|
335
|
+
|
|
336
|
+
LoroDoc persistence keeps its existing storage path. Its cursor is memory-only
|
|
337
|
+
in the example above. Pass `{ documentRemoteCursorStore }` as the factory's
|
|
338
|
+
second argument when an application explicitly wants its existing LoroDoc
|
|
339
|
+
cursor behavior; this does not extend the Flock recovery guarantee to LoroDoc.
|
|
340
|
+
|
|
143
341
|
## SQLite storage (Node.js / Electron)
|
|
144
342
|
|
|
145
|
-
`SqliteRepoStore` packs
|
|
146
|
-
file
|
|
147
|
-
use. Compared to `FileSystemStorageAdaptor`, it avoids creating many small
|
|
343
|
+
`SqliteRepoStore` packs repo data and cursor rows into a single SQLite database
|
|
344
|
+
file. Compared to `FileSystemStorageAdaptor`, it avoids creating many small
|
|
148
345
|
files for incremental updates, and it lets remote-cursor advances become
|
|
149
|
-
single-row `UPDATE`s instead of rewriting a large JSON blob.
|
|
346
|
+
single-row `UPDATE`s instead of rewriting a large JSON blob. It currently
|
|
347
|
+
provides ordered cursor persistence, not the atomic Flock recovery capability
|
|
348
|
+
described above.
|
|
150
349
|
|
|
151
350
|
A single `SqliteRepoStore` instance exposes two adapters that share one
|
|
152
351
|
database connection:
|
|
@@ -154,53 +353,45 @@ database connection:
|
|
|
154
353
|
- `store.storage` — a `StorageAdapter` for doc snapshots, doc updates, and
|
|
155
354
|
metadata (Flock) snapshots/updates.
|
|
156
355
|
- `store.cursorStore` — a `RemoteCursorStore` for the streams transport's
|
|
157
|
-
`
|
|
356
|
+
`persistence` bundle.
|
|
158
357
|
|
|
159
358
|
```ts
|
|
160
359
|
import { LoroRepo } from "loro-repo";
|
|
161
360
|
import { SqliteRepoStore } from "loro-repo/storage/sqlite";
|
|
162
|
-
import {
|
|
361
|
+
import {
|
|
362
|
+
StreamsTransportAdapter,
|
|
363
|
+
createRepoStreamsPersistence,
|
|
364
|
+
} from "loro-repo/transport/streams";
|
|
163
365
|
|
|
164
366
|
const store = new SqliteRepoStore({ path: "./data/repo.db" });
|
|
165
367
|
|
|
166
368
|
const repo = await LoroRepo.create({
|
|
167
369
|
storageAdapter: store.storage,
|
|
168
|
-
|
|
370
|
+
});
|
|
371
|
+
|
|
372
|
+
await repo.addTransport(
|
|
373
|
+
"streams",
|
|
374
|
+
new StreamsTransportAdapter({
|
|
169
375
|
bucketId: "your-bucket",
|
|
170
376
|
auth: yourAuthProvider,
|
|
171
|
-
|
|
172
|
-
//
|
|
173
|
-
|
|
174
|
-
// durable *before* the cursor advances. Skipping them risks a crash that
|
|
175
|
-
// leaves the cursor pointing past data that was never persisted locally.
|
|
176
|
-
onPersistDoc: async (docId, doc) => {
|
|
177
|
-
await store.storage.save({
|
|
178
|
-
type: "doc-snapshot",
|
|
179
|
-
docId,
|
|
180
|
-
snapshot: doc.export({ mode: "snapshot" }),
|
|
181
|
-
});
|
|
182
|
-
},
|
|
183
|
-
onPersistMeta: async (flock) => {
|
|
184
|
-
await store.storage.save({
|
|
185
|
-
type: "meta-snapshot",
|
|
186
|
-
snapshot: flock.exportFile(),
|
|
187
|
-
});
|
|
188
|
-
},
|
|
377
|
+
// Compatibility path: this guarantees data-before-cursor ordering, but
|
|
378
|
+
// SqliteRepoStore does not atomically restore a Flock and checkpoint.
|
|
379
|
+
persistence: createRepoStreamsPersistence(repo, store.cursorStore),
|
|
189
380
|
}),
|
|
190
|
-
|
|
381
|
+
);
|
|
191
382
|
|
|
192
383
|
// Shutdown
|
|
193
384
|
await repo.destroy();
|
|
194
385
|
store.close();
|
|
195
386
|
```
|
|
196
387
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
388
|
+
The two-argument factory is deprecated for Flock rooms because a separate
|
|
389
|
+
cursor store cannot prove that a newly loaded in-memory replica contains the
|
|
390
|
+
checkpointed state. It retains data-before-cursor ordering for compatibility.
|
|
391
|
+
SQLite, filesystem and custom stores must implement the optional atomic
|
|
392
|
+
replica-loading capability before they can use the no-argument safe factory.
|
|
393
|
+
Sessions that keep nothing across restarts can use `persistence: { mode:
|
|
394
|
+
"ephemeral" }`.
|
|
204
395
|
|
|
205
396
|
`SqliteRepoStoreOptions`:
|
|
206
397
|
|
|
@@ -298,7 +489,7 @@ pulls native bindings into a browser bundle.
|
|
|
298
489
|
- `await repo.upsertDocMeta(docId, patch)`
|
|
299
490
|
- `await repo.getDocMeta(docId)`
|
|
300
491
|
- `await repo.listDoc(query?)`
|
|
301
|
-
- `repo.getMeta()`
|
|
492
|
+
- `repo.getMeta()`
|
|
302
493
|
|
|
303
494
|
**Documents**
|
|
304
495
|
- `await repo.openPersistedDoc(docId)`
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
const require_flock_snapshot = require('./flock-snapshot.cjs');
|
|
2
|
+
|
|
3
|
+
//#region src/internal/flock-persistence-journal.ts
|
|
4
|
+
const journalKey = Symbol.for("loro-repo.flock-persistence-journal.v1");
|
|
5
|
+
const MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;
|
|
6
|
+
function versionsEqual(left, right) {
|
|
7
|
+
const leftPeers = Object.keys(left);
|
|
8
|
+
const rightPeers = Object.keys(right);
|
|
9
|
+
if (leftPeers.length !== rightPeers.length) return false;
|
|
10
|
+
return leftPeers.every((peer) => {
|
|
11
|
+
const a = left[peer];
|
|
12
|
+
const b = right[peer];
|
|
13
|
+
return a?.physicalTime === b?.physicalTime && a?.logicalCounter === b?.logicalCounter;
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
/** Pending exact records and received payloads, acknowledged only after save. */
|
|
17
|
+
var FlockPersistenceJournal = class {
|
|
18
|
+
keys = /* @__PURE__ */ new Set();
|
|
19
|
+
payloads = [];
|
|
20
|
+
unsubscribe;
|
|
21
|
+
remoteKeys;
|
|
22
|
+
constructor(flock, onDirty) {
|
|
23
|
+
this.flock = flock;
|
|
24
|
+
this.onDirty = onDirty;
|
|
25
|
+
flock[journalKey] = this;
|
|
26
|
+
this.unsubscribe = flock.subscribe((batch) => {
|
|
27
|
+
const keys = batch.source !== "local" && this.remoteKeys ? this.remoteKeys : this.keys;
|
|
28
|
+
for (const event of batch.events) keys.add(JSON.stringify(event.key));
|
|
29
|
+
if (keys === this.keys && batch.events.length) this.onDirty();
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
get hasPending() {
|
|
33
|
+
return this.keys.size > 0 || this.payloads.length > 0;
|
|
34
|
+
}
|
|
35
|
+
async receive(payloads, apply) {
|
|
36
|
+
const copies = payloads.filter((bytes) => bytes.length > 0).map((bytes) => bytes.slice());
|
|
37
|
+
let result;
|
|
38
|
+
const previousKeys = this.remoteKeys;
|
|
39
|
+
const remoteKeys = /* @__PURE__ */ new Set();
|
|
40
|
+
const previousVersion = this.flock.version();
|
|
41
|
+
this.remoteKeys = remoteKeys;
|
|
42
|
+
try {
|
|
43
|
+
result = apply();
|
|
44
|
+
} catch (error) {
|
|
45
|
+
this.retainChangedKeys(remoteKeys);
|
|
46
|
+
throw error;
|
|
47
|
+
} finally {
|
|
48
|
+
this.remoteKeys = previousKeys;
|
|
49
|
+
}
|
|
50
|
+
let outcome;
|
|
51
|
+
try {
|
|
52
|
+
outcome = await result;
|
|
53
|
+
} catch (error) {
|
|
54
|
+
this.retainChangedKeys(remoteKeys);
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
if (remoteKeys.size > 0 || !versionsEqual(previousVersion, this.flock.version())) {
|
|
58
|
+
this.payloads.push(...copies.map((bytes) => ({
|
|
59
|
+
bytes,
|
|
60
|
+
failures: 0
|
|
61
|
+
})));
|
|
62
|
+
if (copies.length) this.onDirty();
|
|
63
|
+
}
|
|
64
|
+
return outcome;
|
|
65
|
+
}
|
|
66
|
+
retainChangedKeys(keys) {
|
|
67
|
+
for (const key of keys) this.keys.add(key);
|
|
68
|
+
if (keys.size) this.onDirty();
|
|
69
|
+
}
|
|
70
|
+
/** Called within the owner's serialized persistence queue. */
|
|
71
|
+
async flush(save) {
|
|
72
|
+
this.flock.commit();
|
|
73
|
+
while (this.hasPending) {
|
|
74
|
+
if (this.keys.size) await this.stageChangedKeys();
|
|
75
|
+
const payload = this.payloads[0];
|
|
76
|
+
try {
|
|
77
|
+
await save(payload.bytes);
|
|
78
|
+
} catch (exactError) {
|
|
79
|
+
payload.failures += 1;
|
|
80
|
+
if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) throw exactError;
|
|
81
|
+
const coveredPayloads = this.payloads.length;
|
|
82
|
+
const snapshot = await require_flock_snapshot.exportFlockFileSnapshot(this.flock);
|
|
83
|
+
try {
|
|
84
|
+
await save(snapshot);
|
|
85
|
+
} catch (snapshotError) {
|
|
86
|
+
throw new AggregateError([exactError, snapshotError], "Failed to persist a Flock update and its snapshot fallback");
|
|
87
|
+
}
|
|
88
|
+
this.payloads.splice(0, coveredPayloads);
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
this.payloads.shift();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
async stageChangedKeys() {
|
|
95
|
+
const capturedKeys = [...this.keys];
|
|
96
|
+
for (const key of capturedKeys) this.keys.delete(key);
|
|
97
|
+
try {
|
|
98
|
+
const bundle = {
|
|
99
|
+
version: 0,
|
|
100
|
+
entries: {}
|
|
101
|
+
};
|
|
102
|
+
let useSnapshot = false;
|
|
103
|
+
for (const key of capturedKeys) {
|
|
104
|
+
let entry;
|
|
105
|
+
try {
|
|
106
|
+
entry = this.flock.getEntry(JSON.parse(key));
|
|
107
|
+
} catch {
|
|
108
|
+
useSnapshot = true;
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
if (!entry) {
|
|
112
|
+
useSnapshot = true;
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
const { physicalTime, logicalCounter, peerId } = entry.clock;
|
|
116
|
+
bundle.entries[key] = {
|
|
117
|
+
c: `${physicalTime},${logicalCounter},${peerId}`,
|
|
118
|
+
...entry.data === void 0 ? {} : { d: entry.data },
|
|
119
|
+
...Object.keys(entry.metadata).length ? { m: entry.metadata } : {}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
const bytes = useSnapshot ? await require_flock_snapshot.exportFlockFileSnapshot(this.flock) : require_flock_snapshot.encodeMetaJsonUpdate(bundle);
|
|
123
|
+
this.payloads.push({
|
|
124
|
+
bytes,
|
|
125
|
+
failures: 0
|
|
126
|
+
});
|
|
127
|
+
} catch (error) {
|
|
128
|
+
for (const key of capturedKeys) this.keys.add(key);
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
destroy() {
|
|
133
|
+
this.unsubscribe();
|
|
134
|
+
if (this.flock[journalKey] === this) delete this.flock[journalKey];
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */
|
|
138
|
+
async function receiveFlockPayloads(flock, payloads, apply) {
|
|
139
|
+
const journal = flock[journalKey];
|
|
140
|
+
return journal ? journal.receive(payloads, apply) : await apply();
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
//#endregion
|
|
144
|
+
Object.defineProperty(exports, 'FlockPersistenceJournal', {
|
|
145
|
+
enumerable: true,
|
|
146
|
+
get: function () {
|
|
147
|
+
return FlockPersistenceJournal;
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
Object.defineProperty(exports, 'receiveFlockPayloads', {
|
|
151
|
+
enumerable: true,
|
|
152
|
+
get: function () {
|
|
153
|
+
return receiveFlockPayloads;
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
//# sourceMappingURL=flock-persistence-journal.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flock-persistence-journal.cjs","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","exportFlockFileSnapshot","bundle: ExportBundle","encodeMetaJsonUpdate"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n const previousVersion = this.flock.version();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes.\n if (\n remoteKeys.size > 0 ||\n !versionsEqual(previousVersion, this.flock.version())\n ) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /** Called within the owner's serialized persistence queue. */\n async flush(save: (update: Uint8Array) => Promise<void>): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const payload = this.payloads[0]!;\n try {\n await save(payload.bytes);\n } catch (exactError) {\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n try {\n await save(snapshot);\n } catch (snapshotError) {\n throw new AggregateError(\n [exactError, snapshotError],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n continue;\n }\n\n // New imports/local changes during the awaits remain behind this item.\n this.payloads.shift();\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAExC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EACpC,MAAM,kBAAkB,KAAK,MAAM,SAAS;AAC5C,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAOR,MACE,WAAW,OAAO,KAClB,CAAC,cAAc,iBAAiB,KAAK,MAAM,SAAS,CAAC,EACrD;AACA,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;CAI/B,MAAM,MAAM,MAA4D;AACtE,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,UAAU,KAAK,SAAS;AAC9B,OAAI;AACF,UAAM,KAAK,QAAQ,MAAM;YAClB,YAAY;AACnB,YAAQ,YAAY;AAIpB,QAAI,QAAQ,WAAW,gCACrB,OAAM;IAUR,MAAM,kBAAkB,KAAK,SAAS;IACtC,MAAM,WAAW,MAAMC,+CAAwB,KAAK,MAAM;AAC1D,QAAI;AACF,WAAM,KAAK,SAAS;aACb,eAAe;AACtB,WAAM,IAAI,eACR,CAAC,YAAY,cAAc,EAC3B,6DACD;;AAEH,SAAK,SAAS,OAAO,GAAG,gBAAgB;AACxC;;AAIF,QAAK,SAAS,OAAO;;;CAIzB,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAMD,+CAAwB,KAAK,MAAM,GACzCE,4CAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { r as encodeMetaJsonUpdate, t as exportFlockFileSnapshot } from "./flock-snapshot.js";
|
|
2
|
+
|
|
3
|
+
//#region src/internal/flock-persistence-journal.ts
|
|
4
|
+
const journalKey = Symbol.for("loro-repo.flock-persistence-journal.v1");
|
|
5
|
+
const MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;
|
|
6
|
+
function versionsEqual(left, right) {
|
|
7
|
+
const leftPeers = Object.keys(left);
|
|
8
|
+
const rightPeers = Object.keys(right);
|
|
9
|
+
if (leftPeers.length !== rightPeers.length) return false;
|
|
10
|
+
return leftPeers.every((peer) => {
|
|
11
|
+
const a = left[peer];
|
|
12
|
+
const b = right[peer];
|
|
13
|
+
return a?.physicalTime === b?.physicalTime && a?.logicalCounter === b?.logicalCounter;
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
/** Pending exact records and received payloads, acknowledged only after save. */
|
|
17
|
+
var FlockPersistenceJournal = class {
|
|
18
|
+
keys = /* @__PURE__ */ new Set();
|
|
19
|
+
payloads = [];
|
|
20
|
+
unsubscribe;
|
|
21
|
+
remoteKeys;
|
|
22
|
+
constructor(flock, onDirty) {
|
|
23
|
+
this.flock = flock;
|
|
24
|
+
this.onDirty = onDirty;
|
|
25
|
+
flock[journalKey] = this;
|
|
26
|
+
this.unsubscribe = flock.subscribe((batch) => {
|
|
27
|
+
const keys = batch.source !== "local" && this.remoteKeys ? this.remoteKeys : this.keys;
|
|
28
|
+
for (const event of batch.events) keys.add(JSON.stringify(event.key));
|
|
29
|
+
if (keys === this.keys && batch.events.length) this.onDirty();
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
get hasPending() {
|
|
33
|
+
return this.keys.size > 0 || this.payloads.length > 0;
|
|
34
|
+
}
|
|
35
|
+
async receive(payloads, apply) {
|
|
36
|
+
const copies = payloads.filter((bytes) => bytes.length > 0).map((bytes) => bytes.slice());
|
|
37
|
+
let result;
|
|
38
|
+
const previousKeys = this.remoteKeys;
|
|
39
|
+
const remoteKeys = /* @__PURE__ */ new Set();
|
|
40
|
+
const previousVersion = this.flock.version();
|
|
41
|
+
this.remoteKeys = remoteKeys;
|
|
42
|
+
try {
|
|
43
|
+
result = apply();
|
|
44
|
+
} catch (error) {
|
|
45
|
+
this.retainChangedKeys(remoteKeys);
|
|
46
|
+
throw error;
|
|
47
|
+
} finally {
|
|
48
|
+
this.remoteKeys = previousKeys;
|
|
49
|
+
}
|
|
50
|
+
let outcome;
|
|
51
|
+
try {
|
|
52
|
+
outcome = await result;
|
|
53
|
+
} catch (error) {
|
|
54
|
+
this.retainChangedKeys(remoteKeys);
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
if (remoteKeys.size > 0 || !versionsEqual(previousVersion, this.flock.version())) {
|
|
58
|
+
this.payloads.push(...copies.map((bytes) => ({
|
|
59
|
+
bytes,
|
|
60
|
+
failures: 0
|
|
61
|
+
})));
|
|
62
|
+
if (copies.length) this.onDirty();
|
|
63
|
+
}
|
|
64
|
+
return outcome;
|
|
65
|
+
}
|
|
66
|
+
retainChangedKeys(keys) {
|
|
67
|
+
for (const key of keys) this.keys.add(key);
|
|
68
|
+
if (keys.size) this.onDirty();
|
|
69
|
+
}
|
|
70
|
+
/** Called within the owner's serialized persistence queue. */
|
|
71
|
+
async flush(save) {
|
|
72
|
+
this.flock.commit();
|
|
73
|
+
while (this.hasPending) {
|
|
74
|
+
if (this.keys.size) await this.stageChangedKeys();
|
|
75
|
+
const payload = this.payloads[0];
|
|
76
|
+
try {
|
|
77
|
+
await save(payload.bytes);
|
|
78
|
+
} catch (exactError) {
|
|
79
|
+
payload.failures += 1;
|
|
80
|
+
if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) throw exactError;
|
|
81
|
+
const coveredPayloads = this.payloads.length;
|
|
82
|
+
const snapshot = await exportFlockFileSnapshot(this.flock);
|
|
83
|
+
try {
|
|
84
|
+
await save(snapshot);
|
|
85
|
+
} catch (snapshotError) {
|
|
86
|
+
throw new AggregateError([exactError, snapshotError], "Failed to persist a Flock update and its snapshot fallback");
|
|
87
|
+
}
|
|
88
|
+
this.payloads.splice(0, coveredPayloads);
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
this.payloads.shift();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
async stageChangedKeys() {
|
|
95
|
+
const capturedKeys = [...this.keys];
|
|
96
|
+
for (const key of capturedKeys) this.keys.delete(key);
|
|
97
|
+
try {
|
|
98
|
+
const bundle = {
|
|
99
|
+
version: 0,
|
|
100
|
+
entries: {}
|
|
101
|
+
};
|
|
102
|
+
let useSnapshot = false;
|
|
103
|
+
for (const key of capturedKeys) {
|
|
104
|
+
let entry;
|
|
105
|
+
try {
|
|
106
|
+
entry = this.flock.getEntry(JSON.parse(key));
|
|
107
|
+
} catch {
|
|
108
|
+
useSnapshot = true;
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
if (!entry) {
|
|
112
|
+
useSnapshot = true;
|
|
113
|
+
break;
|
|
114
|
+
}
|
|
115
|
+
const { physicalTime, logicalCounter, peerId } = entry.clock;
|
|
116
|
+
bundle.entries[key] = {
|
|
117
|
+
c: `${physicalTime},${logicalCounter},${peerId}`,
|
|
118
|
+
...entry.data === void 0 ? {} : { d: entry.data },
|
|
119
|
+
...Object.keys(entry.metadata).length ? { m: entry.metadata } : {}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
const bytes = useSnapshot ? await exportFlockFileSnapshot(this.flock) : encodeMetaJsonUpdate(bundle);
|
|
123
|
+
this.payloads.push({
|
|
124
|
+
bytes,
|
|
125
|
+
failures: 0
|
|
126
|
+
});
|
|
127
|
+
} catch (error) {
|
|
128
|
+
for (const key of capturedKeys) this.keys.add(key);
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
destroy() {
|
|
133
|
+
this.unsubscribe();
|
|
134
|
+
if (this.flock[journalKey] === this) delete this.flock[journalKey];
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */
|
|
138
|
+
async function receiveFlockPayloads(flock, payloads, apply) {
|
|
139
|
+
const journal = flock[journalKey];
|
|
140
|
+
return journal ? journal.receive(payloads, apply) : await apply();
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
//#endregion
|
|
144
|
+
export { receiveFlockPayloads as n, FlockPersistenceJournal as t };
|
|
145
|
+
//# sourceMappingURL=flock-persistence-journal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flock-persistence-journal.js","names":["flock: Flock","onDirty: () => void","result: T","outcome: Awaited<T>","bundle: ExportBundle"],"sources":["../src/internal/flock-persistence-journal.ts"],"sourcesContent":["import type {\n ExportBundle,\n Flock,\n KeyPart,\n VersionVector,\n} from \"@loro-dev/flock-wasm\";\nimport { encodeMetaJsonUpdate } from \"../storage/meta-snapshot\";\nimport { exportFlockFileSnapshot } from \"./flock-snapshot\";\n\n// The repo and transport are separate package entry points (and may use\n// different ESM/CJS module instances). Attach the journal to the replica,\n// rather than relying on a module-local WeakMap shared by both entry points.\nconst journalKey = Symbol.for(\"loro-repo.flock-persistence-journal.v1\");\ntype JournalledFlock = Flock & { [journalKey]?: FlockPersistenceJournal };\n\ninterface PendingPayload {\n readonly bytes: Uint8Array;\n failures: number;\n}\n\nconst MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS = 3;\n\nfunction versionsEqual(left: VersionVector, right: VersionVector): boolean {\n const leftPeers = Object.keys(left);\n const rightPeers = Object.keys(right);\n if (leftPeers.length !== rightPeers.length) return false;\n return leftPeers.every((peer) => {\n const a = left[peer];\n const b = right[peer];\n return (\n a?.physicalTime === b?.physicalTime &&\n a?.logicalCounter === b?.logicalCounter\n );\n });\n}\n\n/** Pending exact records and received payloads, acknowledged only after save. */\nexport class FlockPersistenceJournal {\n private readonly keys = new Set<string>();\n private readonly payloads: PendingPayload[] = [];\n private readonly unsubscribe: () => void;\n private remoteKeys?: Set<string>;\n\n constructor(\n private readonly flock: Flock,\n private readonly onDirty: () => void,\n ) {\n (flock as JournalledFlock)[journalKey] = this;\n this.unsubscribe = flock.subscribe((batch) => {\n // Local commits can be emitted by beforeImport(), even inside a remote\n // import. They must survive alongside the received payload.\n const keys =\n batch.source !== \"local\" && this.remoteKeys ? this.remoteKeys : this.keys;\n for (const event of batch.events) {\n keys.add(JSON.stringify(event.key));\n }\n if (keys === this.keys && batch.events.length) this.onDirty();\n });\n }\n\n get hasPending(): boolean {\n return this.keys.size > 0 || this.payloads.length > 0;\n }\n\n async receive<T>(\n payloads: readonly Uint8Array[],\n apply: () => T,\n ): Promise<Awaited<T>> {\n const copies = payloads\n .filter((bytes) => bytes.length > 0)\n .map((bytes) => bytes.slice());\n // Existing Flock adapter imports are synchronous. Only suppress their\n // synchronous remote events; unrelated work after an await is observable.\n let result: T;\n const previousKeys = this.remoteKeys;\n const remoteKeys = new Set<string>();\n const previousVersion = this.flock.version();\n this.remoteKeys = remoteKeys;\n try {\n result = apply();\n } catch (error) {\n // A batch import can partially succeed before throwing. Retain its\n // accepted records without persisting a possibly malformed payload.\n this.retainChangedKeys(remoteKeys);\n throw error;\n } finally {\n this.remoteKeys = previousKeys;\n }\n let outcome: Awaited<T>;\n try {\n outcome = await result;\n } catch (error) {\n this.retainChangedKeys(remoteKeys);\n throw error;\n }\n // Streams can echo an append back to its producer, and a bootstrap can\n // repeat state already hydrated locally. Flock emits no remote events in\n // either case, so there is no newly accepted state to journal. A genuine\n // same-VV hole does emit the missing key and still records the exact\n // received bytes.\n if (\n remoteKeys.size > 0 ||\n !versionsEqual(previousVersion, this.flock.version())\n ) {\n this.payloads.push(...copies.map((bytes) => ({ bytes, failures: 0 })));\n if (copies.length) this.onDirty();\n }\n return outcome;\n }\n\n private retainChangedKeys(keys: ReadonlySet<string>): void {\n for (const key of keys) this.keys.add(key);\n if (keys.size) this.onDirty();\n }\n\n /** Called within the owner's serialized persistence queue. */\n async flush(save: (update: Uint8Array) => Promise<void>): Promise<void> {\n this.flock.commit();\n while (this.hasPending) {\n if (this.keys.size) {\n await this.stageChangedKeys();\n }\n\n const payload = this.payloads[0]!;\n try {\n await save(payload.bytes);\n } catch (exactError) {\n payload.failures += 1;\n // One failing barrier must still block its remote cursor. Repeated\n // barriers retry the exact bytes up to a bounded count before using\n // the mergeable full-state fallback below.\n if (payload.failures < MAX_EXACT_PAYLOAD_SAVE_ATTEMPTS) {\n throw exactError;\n }\n // A storage adapter can reject one otherwise accepted encoding\n // forever. A full file is another mergeable representation of the\n // current CRDT state, so it can safely cover that queue item. Never\n // advance past the item unless this fallback itself is durable.\n // The exported file covers every payload that was queued when the\n // export began, not just the failing head. Payloads received while\n // the incremental export or save awaits are appended after this\n // boundary and must remain pending.\n const coveredPayloads = this.payloads.length;\n const snapshot = await exportFlockFileSnapshot(this.flock);\n try {\n await save(snapshot);\n } catch (snapshotError) {\n throw new AggregateError(\n [exactError, snapshotError],\n \"Failed to persist a Flock update and its snapshot fallback\",\n );\n }\n this.payloads.splice(0, coveredPayloads);\n continue;\n }\n\n // New imports/local changes during the awaits remain behind this item.\n this.payloads.shift();\n }\n }\n\n private async stageChangedKeys(): Promise<void> {\n const capturedKeys = [...this.keys];\n for (const key of capturedKeys) this.keys.delete(key);\n try {\n const bundle: ExportBundle = { version: 0, entries: {} };\n let useSnapshot = false;\n for (const key of capturedKeys) {\n let entry;\n try {\n entry = this.flock.getEntry(JSON.parse(key) as KeyPart[]);\n } catch {\n useSnapshot = true;\n break;\n }\n if (!entry) {\n useSnapshot = true;\n break;\n }\n const { physicalTime, logicalCounter, peerId } = entry.clock;\n bundle.entries[key] = {\n c: `${physicalTime},${logicalCounter},${peerId}`,\n ...(entry.data === undefined ? {} : { d: entry.data }),\n ...(Object.keys(entry.metadata).length ? { m: entry.metadata } : {}),\n };\n }\n const bytes = useSnapshot\n ? await exportFlockFileSnapshot(this.flock)\n : encodeMetaJsonUpdate(bundle);\n this.payloads.push({ bytes, failures: 0 });\n } catch (error) {\n // The current state has not been represented in the payload queue.\n // Restore the captured markers without disturbing newer events.\n for (const key of capturedKeys) this.keys.add(key);\n throw error;\n }\n }\n\n destroy(): void {\n this.unsubscribe();\n if ((this.flock as JournalledFlock)[journalKey] === this) {\n delete (this.flock as JournalledFlock)[journalKey];\n }\n }\n}\n\n/** Preserve plaintext CRDT bytes at the public Streams adapter boundary. */\nexport async function receiveFlockPayloads<T>(\n flock: Flock,\n payloads: readonly Uint8Array[],\n apply: () => T,\n): Promise<Awaited<T>> {\n const journal = (flock as JournalledFlock)[journalKey];\n return journal ? journal.receive(payloads, apply) : await apply();\n}\n"],"mappings":";;;AAYA,MAAM,aAAa,OAAO,IAAI,yCAAyC;AAQvE,MAAM,kCAAkC;AAExC,SAAS,cAAc,MAAqB,OAA+B;CACzE,MAAM,YAAY,OAAO,KAAK,KAAK;CACnC,MAAM,aAAa,OAAO,KAAK,MAAM;AACrC,KAAI,UAAU,WAAW,WAAW,OAAQ,QAAO;AACnD,QAAO,UAAU,OAAO,SAAS;EAC/B,MAAM,IAAI,KAAK;EACf,MAAM,IAAI,MAAM;AAChB,SACE,GAAG,iBAAiB,GAAG,gBACvB,GAAG,mBAAmB,GAAG;GAE3B;;;AAIJ,IAAa,0BAAb,MAAqC;CACnC,AAAiB,uBAAO,IAAI,KAAa;CACzC,AAAiB,WAA6B,EAAE;CAChD,AAAiB;CACjB,AAAQ;CAER,YACE,AAAiBA,OACjB,AAAiBC,SACjB;EAFiB;EACA;AAEjB,EAAC,MAA0B,cAAc;AACzC,OAAK,cAAc,MAAM,WAAW,UAAU;GAG5C,MAAM,OACJ,MAAM,WAAW,WAAW,KAAK,aAAa,KAAK,aAAa,KAAK;AACvE,QAAK,MAAM,SAAS,MAAM,OACxB,MAAK,IAAI,KAAK,UAAU,MAAM,IAAI,CAAC;AAErC,OAAI,SAAS,KAAK,QAAQ,MAAM,OAAO,OAAQ,MAAK,SAAS;IAC7D;;CAGJ,IAAI,aAAsB;AACxB,SAAO,KAAK,KAAK,OAAO,KAAK,KAAK,SAAS,SAAS;;CAGtD,MAAM,QACJ,UACA,OACqB;EACrB,MAAM,SAAS,SACZ,QAAQ,UAAU,MAAM,SAAS,EAAE,CACnC,KAAK,UAAU,MAAM,OAAO,CAAC;EAGhC,IAAIC;EACJ,MAAM,eAAe,KAAK;EAC1B,MAAM,6BAAa,IAAI,KAAa;EACpC,MAAM,kBAAkB,KAAK,MAAM,SAAS;AAC5C,OAAK,aAAa;AAClB,MAAI;AACF,YAAS,OAAO;WACT,OAAO;AAGd,QAAK,kBAAkB,WAAW;AAClC,SAAM;YACE;AACR,QAAK,aAAa;;EAEpB,IAAIC;AACJ,MAAI;AACF,aAAU,MAAM;WACT,OAAO;AACd,QAAK,kBAAkB,WAAW;AAClC,SAAM;;AAOR,MACE,WAAW,OAAO,KAClB,CAAC,cAAc,iBAAiB,KAAK,MAAM,SAAS,CAAC,EACrD;AACA,QAAK,SAAS,KAAK,GAAG,OAAO,KAAK,WAAW;IAAE;IAAO,UAAU;IAAG,EAAE,CAAC;AACtE,OAAI,OAAO,OAAQ,MAAK,SAAS;;AAEnC,SAAO;;CAGT,AAAQ,kBAAkB,MAAiC;AACzD,OAAK,MAAM,OAAO,KAAM,MAAK,KAAK,IAAI,IAAI;AAC1C,MAAI,KAAK,KAAM,MAAK,SAAS;;;CAI/B,MAAM,MAAM,MAA4D;AACtE,OAAK,MAAM,QAAQ;AACnB,SAAO,KAAK,YAAY;AACtB,OAAI,KAAK,KAAK,KACZ,OAAM,KAAK,kBAAkB;GAG/B,MAAM,UAAU,KAAK,SAAS;AAC9B,OAAI;AACF,UAAM,KAAK,QAAQ,MAAM;YAClB,YAAY;AACnB,YAAQ,YAAY;AAIpB,QAAI,QAAQ,WAAW,gCACrB,OAAM;IAUR,MAAM,kBAAkB,KAAK,SAAS;IACtC,MAAM,WAAW,MAAM,wBAAwB,KAAK,MAAM;AAC1D,QAAI;AACF,WAAM,KAAK,SAAS;aACb,eAAe;AACtB,WAAM,IAAI,eACR,CAAC,YAAY,cAAc,EAC3B,6DACD;;AAEH,SAAK,SAAS,OAAO,GAAG,gBAAgB;AACxC;;AAIF,QAAK,SAAS,OAAO;;;CAIzB,MAAc,mBAAkC;EAC9C,MAAM,eAAe,CAAC,GAAG,KAAK,KAAK;AACnC,OAAK,MAAM,OAAO,aAAc,MAAK,KAAK,OAAO,IAAI;AACrD,MAAI;GACF,MAAMC,SAAuB;IAAE,SAAS;IAAG,SAAS,EAAE;IAAE;GACxD,IAAI,cAAc;AAClB,QAAK,MAAM,OAAO,cAAc;IAC9B,IAAI;AACJ,QAAI;AACF,aAAQ,KAAK,MAAM,SAAS,KAAK,MAAM,IAAI,CAAc;YACnD;AACN,mBAAc;AACd;;AAEF,QAAI,CAAC,OAAO;AACV,mBAAc;AACd;;IAEF,MAAM,EAAE,cAAc,gBAAgB,WAAW,MAAM;AACvD,WAAO,QAAQ,OAAO;KACpB,GAAG,GAAG,aAAa,GAAG,eAAe,GAAG;KACxC,GAAI,MAAM,SAAS,SAAY,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM;KACrD,GAAI,OAAO,KAAK,MAAM,SAAS,CAAC,SAAS,EAAE,GAAG,MAAM,UAAU,GAAG,EAAE;KACpE;;GAEH,MAAM,QAAQ,cACV,MAAM,wBAAwB,KAAK,MAAM,GACzC,qBAAqB,OAAO;AAChC,QAAK,SAAS,KAAK;IAAE;IAAO,UAAU;IAAG,CAAC;WACnC,OAAO;AAGd,QAAK,MAAM,OAAO,aAAc,MAAK,KAAK,IAAI,IAAI;AAClD,SAAM;;;CAIV,UAAgB;AACd,OAAK,aAAa;AAClB,MAAK,KAAK,MAA0B,gBAAgB,KAClD,QAAQ,KAAK,MAA0B;;;;AAM7C,eAAsB,qBACpB,OACA,UACA,OACqB;CACrB,MAAM,UAAW,MAA0B;AAC3C,QAAO,UAAU,QAAQ,QAAQ,UAAU,MAAM,GAAG,MAAM,OAAO"}
|