loro-repo 0.23.2 → 0.23.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -501,6 +501,12 @@ nothing across restarts can use `persistence: { mode: "ephemeral" }`.
501
501
  | `database` | — | Pass a pre-constructed `better-sqlite3` `Database` to share a connection; the caller owns its lifetime. |
502
502
  | `tablePrefix` | `""` | Optional prefix for all table names, useful when multiple repos share one database. |
503
503
  | `wal` | `true` | Enables `journal_mode=WAL` + `synchronous=NORMAL` for fast, durable writes. |
504
+ | `busyTimeoutMs` | `0` (`5000` for a supplied `database`) | Longest one SQLite call may block the thread for another connection's lock after opening. Opening itself may wait 5000 ms. |
505
+ | `busyRetryMs` | `5000` | How long after its first attempt an operation that hits `SQLITE_BUSY` keeps retrying, with an asynchronous backoff, before it rejects with the driver's error. Operations queued behind it start their own budget when they reach the head. |
506
+
507
+ While another connection or process holds the write lock, storage operations
508
+ wait without blocking the event loop and still reach SQLite in the order they
509
+ were requested. Closing the store rejects operations that are still waiting.
504
510
 
505
511
  The schema (created on first use) is:
506
512
 
@@ -14,7 +14,24 @@ node_crypto = require_chunk.__toESM(node_crypto);
14
14
  let better_sqlite3 = require("better-sqlite3");
15
15
  better_sqlite3 = require_chunk.__toESM(better_sqlite3);
16
16
 
17
+ //#region src/internal/storage-busy.ts
18
+ /**
19
+ * Whether a SQLite call failed only because another connection held the
20
+ * lock (`SQLITE_BUSY` and its extended codes), so the same call can be
21
+ * retried unchanged.
22
+ */
23
+ function isStorageBusyError(error) {
24
+ const code = error?.code;
25
+ return typeof code === "string" && code.startsWith("SQLITE_BUSY");
26
+ }
27
+
28
+ //#endregion
17
29
  //#region src/storage/sqlite.ts
30
+ /** `busy_timeout` while opening: schema setup is synchronous and rare. */
31
+ const OPEN_BUSY_TIMEOUT_MS = 5e3;
32
+ const DEFAULT_BUSY_RETRY_MS = 5e3;
33
+ const BUSY_RETRY_BASE_DELAY_MS = 5;
34
+ const BUSY_RETRY_MAX_DELAY_MS = 100;
18
35
  /**
19
36
  * SQLite-backed persistence for a {@link LoroRepo}. A single database
20
37
  * connection backs three logical stores:
@@ -41,6 +58,10 @@ var SqliteRepoStore = class SqliteRepoStore {
41
58
  closed = false;
42
59
  writerToken;
43
60
  writerOwner;
61
+ busyTimeoutMs;
62
+ busyRetryMs;
63
+ dbQueue = Promise.resolve();
64
+ dbQueueLength = 0;
44
65
  loadDocAtomic;
45
66
  loadFlockDocAtomic;
46
67
  loadMetaAtomic;
@@ -93,7 +114,9 @@ var SqliteRepoStore = class SqliteRepoStore {
93
114
  this.ownsDatabase = true;
94
115
  }
95
116
  this.prefix = validateTablePrefix(options.tablePrefix ?? "");
96
- this.db.pragma("busy_timeout = 5000");
117
+ this.busyTimeoutMs = validateMilliseconds("busyTimeoutMs", options.busyTimeoutMs ?? (this.ownsDatabase ? 0 : OPEN_BUSY_TIMEOUT_MS));
118
+ this.busyRetryMs = validateMilliseconds("busyRetryMs", options.busyRetryMs ?? DEFAULT_BUSY_RETRY_MS);
119
+ this.db.pragma(`busy_timeout = ${OPEN_BUSY_TIMEOUT_MS}`);
97
120
  if (options.wal !== false) {
98
121
  this.db.pragma("journal_mode = WAL");
99
122
  this.db.pragma("synchronous = NORMAL");
@@ -101,7 +124,7 @@ var SqliteRepoStore = class SqliteRepoStore {
101
124
  this.db.exec(`CREATE TABLE IF NOT EXISTS "${this.prefix}writer_owner" (id INTEGER PRIMARY KEY CHECK(id = 1), token TEXT NOT NULL)`);
102
125
  this.writerOwner = this.db.prepare(`SELECT token FROM "${this.prefix}writer_owner" WHERE id = 1`);
103
126
  try {
104
- if (options.recoverStaleWriter) this.recoverStaleWriter();
127
+ if (options.recoverStaleWriter) this.recoverStaleWriterBody();
105
128
  this.assertWriter();
106
129
  } catch (error) {
107
130
  if (this.ownsDatabase) this.db.close();
@@ -123,6 +146,7 @@ var SqliteRepoStore = class SqliteRepoStore {
123
146
  });
124
147
  this.storage = this.createStorageAdapter();
125
148
  this.cursorStore = this.createCursorStore();
149
+ this.db.pragma(`busy_timeout = ${this.busyTimeoutMs}`);
126
150
  }
127
151
  /**
128
152
  * Closes the underlying database connection if it was created by this
@@ -288,6 +312,14 @@ var SqliteRepoStore = class SqliteRepoStore {
288
312
  }
289
313
  /** Also available through the constructor when a stale fence blocks open. */
290
314
  recoverStaleWriter() {
315
+ this.db.pragma(`busy_timeout = ${OPEN_BUSY_TIMEOUT_MS}`);
316
+ try {
317
+ return this.recoverStaleWriterBody();
318
+ } finally {
319
+ this.db.pragma(`busy_timeout = ${this.busyTimeoutMs}`);
320
+ }
321
+ }
322
+ recoverStaleWriterBody() {
291
323
  return this.db.transaction(() => {
292
324
  const row = this.writerOwner.get();
293
325
  if (!row) return false;
@@ -306,24 +338,74 @@ var SqliteRepoStore = class SqliteRepoStore {
306
338
  return run(...args);
307
339
  }));
308
340
  }
341
+ /**
342
+ * Runs one synchronous section of SQLite work, in request order. While
343
+ * another connection holds the lock, the call blocks for at most
344
+ * `busyTimeoutMs`; the section is then retried after an asynchronous
345
+ * backoff until `busyRetryMs` after its first attempt, and finally rejects
346
+ * with the driver's `SQLITE_BUSY` error. Later sections queue behind a
347
+ * waiting one, so nothing commits out of turn; each gets its own budget
348
+ * once it reaches the head, as each blocking call used to. The
349
+ * writer-ownership read runs inside each attempt: without WAL it can be
350
+ * busy too.
351
+ */
352
+ runDb(section) {
353
+ const work = () => {
354
+ this.assertWriter();
355
+ return section();
356
+ };
357
+ if (this.dbQueueLength > 0) return this.queueDb(work, void 0, 0);
358
+ const deadline = Date.now() + this.busyRetryMs;
359
+ try {
360
+ return Promise.resolve(work());
361
+ } catch (error) {
362
+ if (!isStorageBusyError(error) || Date.now() >= deadline) return Promise.reject(error);
363
+ }
364
+ return this.queueDb(work, deadline, 1);
365
+ }
366
+ /** `deadline` is unset until the section's first attempt. */
367
+ queueDb(work, deadline, failedAttempts) {
368
+ this.dbQueueLength += 1;
369
+ const task = this.dbQueue.then(() => this.retryBusy(work, deadline, failedAttempts));
370
+ const settled = () => {
371
+ this.dbQueueLength -= 1;
372
+ };
373
+ this.dbQueue = task.then(settled, settled);
374
+ return task;
375
+ }
376
+ async retryBusy(work, requestedDeadline, failedAttempts) {
377
+ const deadline = requestedDeadline ?? Date.now() + this.busyRetryMs;
378
+ for (let failures = failedAttempts;; failures += 1) {
379
+ if (failures > 0) {
380
+ const delay = Math.min(BUSY_RETRY_MAX_DELAY_MS, BUSY_RETRY_BASE_DELAY_MS * 2 ** (failures - 1));
381
+ await new Promise((resolve) => setTimeout(resolve, delay));
382
+ }
383
+ if (this.closed) throw new Error("SqliteRepoStore has been closed");
384
+ try {
385
+ return work();
386
+ } catch (error) {
387
+ if (!isStorageBusyError(error) || Date.now() >= deadline) throw error;
388
+ }
389
+ }
390
+ }
309
391
  async acquireWriterLease() {
310
392
  this.ensureOpen();
311
393
  const token = await require_writer_recovery.newWriterToken();
312
- this.db.transaction(() => {
394
+ await this.runDb(() => this.db.transaction(() => {
313
395
  const owner = this.writerOwner.get();
314
396
  if (owner) throw new require_meta_snapshot.RepoWriterLocked(require_writer_recovery.writerRecoveryStatus(owner.token));
315
397
  this.db.prepare(`INSERT INTO "${this.prefix}writer_owner" (id, token) VALUES (1, ?)`).run(token);
316
- }).immediate();
398
+ }).immediate());
317
399
  this.writerToken = token;
318
400
  let released = false;
319
401
  return {
320
402
  token,
321
403
  release: async () => {
322
404
  if (released) return;
323
- this.db.transaction(() => {
405
+ await this.runDb(() => this.db.transaction(() => {
324
406
  this.assertWriter();
325
407
  this.db.prepare(`DELETE FROM "${this.prefix}writer_owner" WHERE id = 1 AND token = ?`).run(token);
326
- }).immediate();
408
+ }).immediate());
327
409
  released = true;
328
410
  this.writerToken = void 0;
329
411
  }
@@ -331,17 +413,17 @@ var SqliteRepoStore = class SqliteRepoStore {
331
413
  }
332
414
  async inventory() {
333
415
  this.ensureOpen();
334
- return this.transaction(() => ({
416
+ return this.runDb(() => this.transaction(() => ({
335
417
  docIds: this.db.prepare(`SELECT doc_id FROM "${this.prefix}docs" UNION SELECT doc_id FROM "${this.prefix}doc_updates" ORDER BY doc_id`).all().map((row) => row.doc_id),
336
418
  flockDocIds: this.db.prepare(`SELECT flock_doc_id FROM "${this.prefix}flock_docs" UNION SELECT flock_doc_id FROM "${this.prefix}flock_doc_updates" ORDER BY flock_doc_id`).all().map((row) => row.flock_doc_id),
337
419
  hasLocalState: !!this.db.prepare(`SELECT 1 FROM "${this.prefix}maintenance" UNION ALL SELECT 1 FROM "${this.prefix}replica_checkpoints" UNION ALL SELECT 1 FROM "${this.prefix}doc_cursor_urls" UNION ALL SELECT 1 FROM "${this.prefix}remote_cursors" LIMIT 1`).get(),
338
420
  hasMetadata: !!this.db.prepare(`SELECT 1 FROM "${this.prefix}meta_snapshot" UNION ALL SELECT 1 FROM "${this.prefix}meta_updates" LIMIT 1`).get()
339
- })).deferred();
421
+ })).deferred());
340
422
  }
341
423
  async discardIncompleteMigration(migrationId) {
342
424
  this.ensureOpen();
343
425
  if (!this.writerToken) throw new require_meta_snapshot.RepoWriterLocked(void 0);
344
- this.transaction(() => {
426
+ await this.runDb(() => this.transaction(() => {
345
427
  if (this.db.prepare(`SELECT value FROM "${this.prefix}maintenance" WHERE key = 'migration'`).get()?.value !== migrationId) throw new Error("Matching incomplete migration required");
346
428
  for (const table of [
347
429
  "docs",
@@ -355,7 +437,7 @@ var SqliteRepoStore = class SqliteRepoStore {
355
437
  "replica_checkpoints"
356
438
  ]) this.db.exec(`DELETE FROM "${this.prefix}${table}"`);
357
439
  this.db.exec(`DELETE FROM "${this.prefix}maintenance"`);
358
- }).immediate();
440
+ }).immediate());
359
441
  this.docSnapshotDigests.clear();
360
442
  this.metaSnapshotDigest = void 0;
361
443
  }
@@ -365,14 +447,14 @@ var SqliteRepoStore = class SqliteRepoStore {
365
447
  migrationState: {
366
448
  load: async () => {
367
449
  this.ensureOpen();
368
- return this.db.prepare(`SELECT value FROM "${this.prefix}maintenance" WHERE key = 'migration'`).get()?.value;
450
+ return this.runDb(() => this.db.prepare(`SELECT value FROM "${this.prefix}maintenance" WHERE key = 'migration'`).get()?.value);
369
451
  },
370
452
  save: async (id) => {
371
453
  this.ensureOpen();
372
- this.transaction(() => {
454
+ await this.runDb(() => this.transaction(() => {
373
455
  if (id === void 0) this.db.prepare(`DELETE FROM "${this.prefix}maintenance" WHERE key = 'migration'`).run();
374
456
  else this.db.prepare(`INSERT INTO "${this.prefix}maintenance" (key, value) VALUES ('migration', ?)`).run(id);
375
- }).immediate();
457
+ }).immediate());
376
458
  }
377
459
  },
378
460
  discardIncompleteMigration: (id) => this.discardIncompleteMigration(id),
@@ -381,15 +463,15 @@ var SqliteRepoStore = class SqliteRepoStore {
381
463
  docCursorIndex: {
382
464
  load: async (id) => {
383
465
  this.ensureOpen();
384
- return this.db.prepare(`SELECT stream_url FROM ${table} WHERE doc_id = ?`).all(id).map((row) => row.stream_url);
466
+ return this.runDb(() => this.db.prepare(`SELECT stream_url FROM ${table} WHERE doc_id = ?`).all(id).map((row) => row.stream_url));
385
467
  },
386
468
  add: async (id, url) => {
387
469
  this.ensureOpen();
388
- this.transaction(() => this.db.prepare(`INSERT OR IGNORE INTO ${table} (doc_id, stream_url) VALUES (?, ?)`).run(id, url)).immediate();
470
+ await this.runDb(() => this.transaction(() => this.db.prepare(`INSERT OR IGNORE INTO ${table} (doc_id, stream_url) VALUES (?, ?)`).run(id, url)).immediate());
389
471
  },
390
472
  delete: async (id) => {
391
473
  this.ensureOpen();
392
- this.transaction(() => this.db.prepare(`DELETE FROM ${table} WHERE doc_id = ?`).run(id)).immediate();
474
+ await this.runDb(() => this.transaction(() => this.db.prepare(`DELETE FROM ${table} WHERE doc_id = ?`).run(id)).immediate());
393
475
  }
394
476
  },
395
477
  save: (payload) => this.saveStorage(payload),
@@ -408,7 +490,7 @@ var SqliteRepoStore = class SqliteRepoStore {
408
490
  }
409
491
  async saveStorage(payload) {
410
492
  this.ensureOpen();
411
- this.savePayload(payload, Date.now());
493
+ await this.runDb(() => this.savePayload(payload, Date.now()));
412
494
  }
413
495
  /**
414
496
  * Writes every payload in one `IMMEDIATE` transaction, so a batch costs one
@@ -416,14 +498,15 @@ var SqliteRepoStore = class SqliteRepoStore {
416
498
  */
417
499
  async saveManyStorage(payloads) {
418
500
  this.ensureOpen();
419
- const now = Date.now();
420
- try {
421
- this.saveManyAtomic.immediate(payloads, now);
422
- } catch (error) {
423
- this.docSnapshotDigests.clear();
424
- this.metaSnapshotDigest = void 0;
425
- throw error;
426
- }
501
+ await this.runDb(() => {
502
+ try {
503
+ this.saveManyAtomic.immediate(payloads, Date.now());
504
+ } catch (error) {
505
+ this.docSnapshotDigests.clear();
506
+ this.metaSnapshotDigest = void 0;
507
+ throw error;
508
+ }
509
+ });
427
510
  }
428
511
  savePayload(payload, now) {
429
512
  switch (payload.type) {
@@ -453,11 +536,11 @@ var SqliteRepoStore = class SqliteRepoStore {
453
536
  }
454
537
  async loadDoc(docId) {
455
538
  this.ensureOpen();
456
- return this.loadDocAtomic.immediate(docId);
539
+ return this.runDb(() => this.loadDocAtomic.immediate(docId));
457
540
  }
458
541
  async loadFlockDoc(flockDocId) {
459
542
  this.ensureOpen();
460
- const state = this.loadFlockDocAtomic.immediate(flockDocId);
543
+ const state = await this.runDb(() => this.loadFlockDocAtomic.immediate(flockDocId));
461
544
  if (state?.needsCompaction) await this.compactLoaded(flockDocId, state);
462
545
  return state?.flock;
463
546
  }
@@ -569,8 +652,10 @@ var SqliteRepoStore = class SqliteRepoStore {
569
652
  */
570
653
  async compactLoaded(flockDocId, state) {
571
654
  const consolidated = await require_flock_snapshot.exportFlockFileSnapshot(state.flock);
572
- if (flockDocId === void 0) this.finalizeMetaCompaction(state, consolidated);
573
- else this.finalizeFlockDocCompaction(flockDocId, state, consolidated);
655
+ await this.runDb(() => {
656
+ if (flockDocId === void 0) this.finalizeMetaCompaction(state, consolidated);
657
+ else this.finalizeFlockDocCompaction(flockDocId, state, consolidated);
658
+ });
574
659
  }
575
660
  saveFlockDocSnapshotBody(flockDocId, incoming, now) {
576
661
  if (this.stmts.selectFlockDocSnapshot.get(flockDocId)) {
@@ -602,19 +687,21 @@ var SqliteRepoStore = class SqliteRepoStore {
602
687
  }
603
688
  async deleteDoc(docId) {
604
689
  this.ensureOpen();
605
- this.transaction((id) => {
690
+ const txn = this.transaction((id) => {
606
691
  this.stmts.deleteDocSnapshot.run(id);
607
692
  this.stmts.deleteDocUpdates.run(id);
608
- })(docId);
693
+ });
694
+ await this.runDb(() => txn(docId));
609
695
  this.docSnapshotDigests.delete(docId);
610
696
  }
611
697
  async deleteFlockDoc(flockDocId) {
612
698
  this.ensureOpen();
613
- this.transaction((id) => {
699
+ const txn = this.transaction((id) => {
614
700
  this.stmts.deleteFlockDocSnapshot.run(id);
615
701
  this.stmts.deleteFlockDocUpdates.run(id);
616
702
  this.stmts.deleteCheckpoint.run(checkpointResourceKey(id));
617
- }).immediate(flockDocId);
703
+ });
704
+ await this.runDb(() => txn.immediate(flockDocId));
618
705
  }
619
706
  /**
620
707
  * Loads Meta (no id) or one named Flock together with the Streams
@@ -624,13 +711,12 @@ var SqliteRepoStore = class SqliteRepoStore {
624
711
  */
625
712
  async loadReplica(flockDocId) {
626
713
  this.ensureOpen();
627
- const captured = this.captureReplicaAtomic.deferred(flockDocId, false) ?? this.captureReplicaAtomic.immediate(flockDocId, true);
714
+ const captured = await this.runDb(() => this.captureReplicaAtomic.deferred(flockDocId, false) ?? this.captureReplicaAtomic.immediate(flockDocId, true));
628
715
  const state = this.hydrateFlockRows(flockDocId, captured);
629
716
  if (state?.needsCompaction) await this.compactLoaded(flockDocId, state).catch(require_logging.logAsyncError(flockDocId === void 0 ? "metadata replica consolidation" : `Flock replica consolidation for "${flockDocId}"`));
630
- else if (state && flockDocId === void 0 && captured.snapshotRow) try {
631
- this.rememberMetaDigest(captured.snapshotRow);
632
- } catch (error) {
633
- require_logging.logAsyncError("metadata digest backfill")(error);
717
+ else if (state && flockDocId === void 0 && captured.snapshotRow) {
718
+ const snapshotRow = captured.snapshotRow;
719
+ await this.runDb(() => this.rememberMetaDigest(snapshotRow)).catch(require_logging.logAsyncError("metadata digest backfill"));
634
720
  }
635
721
  const resourceKey = checkpointResourceKey(flockDocId);
636
722
  const { generation } = captured.record;
@@ -638,7 +724,7 @@ var SqliteRepoStore = class SqliteRepoStore {
638
724
  flock: state?.flock ?? new __loro_dev_flock_wasm.Flock(),
639
725
  checkpointStore: require_replica_checkpoints.createBoundCheckpointStore(captured.record, {
640
726
  isClosed: () => this.closed,
641
- commit: (streamUrl, cursor) => this.writeCheckpointAtomic.immediate(resourceKey, generation, streamUrl, cursor)
727
+ commit: (streamUrl, cursor) => this.runDb(() => this.writeCheckpointAtomic.immediate(resourceKey, generation, streamUrl, cursor))
642
728
  })
643
729
  };
644
730
  }
@@ -664,7 +750,7 @@ var SqliteRepoStore = class SqliteRepoStore {
664
750
  }
665
751
  async loadMeta() {
666
752
  this.ensureOpen();
667
- const state = this.loadMetaAtomic.immediate();
753
+ const state = await this.runDb(() => this.loadMetaAtomic.immediate());
668
754
  if (state?.needsCompaction) await this.compactLoaded(void 0, state);
669
755
  return state?.flock;
670
756
  }
@@ -743,6 +829,9 @@ var SqliteRepoStore = class SqliteRepoStore {
743
829
  }
744
830
  async loadCursor(streamUrl) {
745
831
  this.ensureOpen();
832
+ return this.runDb(() => this.loadCursorBody(streamUrl));
833
+ }
834
+ loadCursorBody(streamUrl) {
746
835
  const row = this.stmts.selectCursor.get(streamUrl);
747
836
  if (!row) return null;
748
837
  const serverLowerBoundVersion = this.parseCursorServerLowerBoundVersion(row.server_lower_bound_version);
@@ -759,18 +848,22 @@ var SqliteRepoStore = class SqliteRepoStore {
759
848
  }
760
849
  async saveCursor(cursor) {
761
850
  this.ensureOpen();
762
- this.transaction(() => this.stmts.upsertCursor.run(cursor.streamUrl, cursor.nextOffset, JSON.stringify(cursor.serverLowerBoundVersion), cursor.updatedAtMs)).immediate();
851
+ await this.runDb(() => this.transaction(() => this.stmts.upsertCursor.run(cursor.streamUrl, cursor.nextOffset, JSON.stringify(cursor.serverLowerBoundVersion), cursor.updatedAtMs)).immediate());
763
852
  }
764
853
  async removeCursor(streamUrl) {
765
854
  this.ensureOpen();
766
- this.transaction(() => this.stmts.deleteCursor.run(streamUrl)).immediate();
855
+ await this.runDb(() => this.transaction(() => this.stmts.deleteCursor.run(streamUrl)).immediate());
767
856
  }
857
+ /** Writer ownership is checked by each `runDb` attempt, where a busy read is retried. */
768
858
  ensureOpen() {
769
- if (!this.closed) this.assertWriter();
770
859
  if (this.closed) throw new Error("SqliteRepoStore has been closed");
771
860
  }
772
861
  };
773
862
  const TABLE_PREFIX_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
863
+ function validateMilliseconds(name, value) {
864
+ if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative safe integer`);
865
+ return value;
866
+ }
774
867
  const META_RESOURCE_KEY = "meta";
775
868
  const FLOCK_RESOURCE_PREFIX = "flock:";
776
869
  function checkpointResourceKey(flockDocId) {