cojson 0.20.9 → 0.20.11

package/src/storage/sqliteAsync/client.ts

@@ -11,13 +11,16 @@ import type {
   DBTransactionInterfaceAsync,
   SessionRow,
   SignatureAfterRow,
+  StorageReconciliationLockRow,
   StoredCoValueRow,
   StoredSessionRow,
   TransactionRow,
+  StorageReconciliationAcquireResult,
 } from "../types.js";
 import { DeletedCoValueDeletionStatus } from "../types.js";
 import type { SQLiteDatabaseDriverAsync } from "./types.js";
 import type { PeerID } from "../../sync.js";
+import { STORAGE_RECONCILIATION_CONFIG } from "../../config.js";
 
 export type RawCoValueRow = {
   id: RawCoID;
@@ -153,6 +156,37 @@ export class SQLiteTransactionAsync implements DBTransactionInterfaceAsync {
       ],
     );
   }
+
+  async getStorageReconciliationLock(
+    key: string,
+  ): Promise<StorageReconciliationLockRow | undefined> {
+    return this.tx.get<StorageReconciliationLockRow>(
+      "SELECT * FROM storageReconciliationLocks WHERE key = ?",
+      [key],
+    );
+  }
+
+  async putStorageReconciliationLock(
+    entry: StorageReconciliationLockRow,
+  ): Promise<void> {
+    const {
+      key,
+      holderSessionId,
+      acquiredAt,
+      releasedAt,
+      lastProcessedOffset,
+    } = entry;
+    await this.tx.run(
+      `INSERT OR REPLACE INTO storageReconciliationLocks (key, holderSessionId, acquiredAt, releasedAt, lastProcessedOffset) VALUES (?, ?, ?, ?, ?)`,
+      [
+        key,
+        holderSessionId,
+        acquiredAt,
+        releasedAt ?? null,
+        lastProcessedOffset,
+      ],
+    );
+  }
 }
 
 export class SQLiteClientAsync implements DBClientInterfaceAsync {
@@ -334,6 +368,111 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     ]);
   }
 
+  async getCoValueIDs(
+    limit: number,
+    offset: number,
+  ): Promise<{ id: RawCoID }[]> {
+    return this.db.query<{ id: RawCoID }>(
+      "SELECT id FROM coValues WHERE rowID > ? ORDER BY rowID LIMIT ?",
+      [offset, limit],
+    );
+  }
+
+  async getCoValueCount(): Promise<number> {
+    const row = await this.db.get<{ count: number }>(
+      "SELECT COUNT(*) as count FROM coValues",
+      [],
+    );
+    return row?.count ?? 0;
+  }
+
+  async tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): Promise<StorageReconciliationAcquireResult> {
+    let result: StorageReconciliationAcquireResult = {
+      acquired: false,
+      reason: "not_due",
+    };
+    await this.transaction(async (tx) => {
+      const now = Date.now();
+      const lockKey = `lock#${peerId}`;
+
+      const lockRow = await tx.getStorageReconciliationLock(lockKey);
+      if (
+        lockRow?.releasedAt &&
+        now - lockRow.releasedAt <
+          STORAGE_RECONCILIATION_CONFIG.RECONCILIATION_INTERVAL_MS
+      ) {
+        result = { acquired: false, reason: "not_due" };
+        return;
+      }
+      const expiresAt = lockRow
+        ? lockRow.acquiredAt + STORAGE_RECONCILIATION_CONFIG.LOCK_TTL_MS
+        : 0;
+      const isLockHeldByOtherSession = lockRow?.holderSessionId !== sessionId;
+      if (
+        lockRow &&
+        !lockRow.releasedAt &&
+        expiresAt >= now &&
+        isLockHeldByOtherSession
+      ) {
+        result = { acquired: false, reason: "lock_held" };
+        return;
+      }
+
+      const lastProcessedOffset =
+        lockRow && !lockRow.releasedAt ? (lockRow.lastProcessedOffset ?? 0) : 0;
+      await tx.putStorageReconciliationLock({
+        key: lockKey,
+        holderSessionId: sessionId,
+        acquiredAt: now,
+        lastProcessedOffset,
+      });
+      result = { acquired: true, lastProcessedOffset };
+    });
+    return result;
+  }
+
+  async renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): Promise<void> {
+    await this.transaction(async (tx) => {
+      const lockKey = `lock#${peerId}`;
+      const lockRow = await tx.getStorageReconciliationLock(lockKey);
+      if (
+        lockRow &&
+        lockRow.holderSessionId === sessionId &&
+        !lockRow.releasedAt
+      ) {
+        await tx.putStorageReconciliationLock({
+          ...lockRow,
+          lastProcessedOffset: offset,
+        });
+      }
+    });
+  }
+
+  async releaseStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): Promise<void> {
+    await this.transaction(async (tx) => {
+      const lockKey = `lock#${peerId}`;
+      const releasedAt = Date.now();
+      const lockRow = await tx.getStorageReconciliationLock(lockKey);
+      if (lockRow && lockRow.holderSessionId === sessionId) {
+        await tx.putStorageReconciliationLock({
+          ...lockRow,
+          releasedAt,
+          lastProcessedOffset: 0,
+        });
+      }
+    });
+  }
+
   async getCoValueKnownState(
     coValueId: string,
   ): Promise<CoValueKnownState | undefined> {

package/src/storage/storageAsync.ts

@@ -30,6 +30,7 @@ import type {
   SignatureAfterRow,
   StoredCoValueRow,
   StoredSessionRow,
+  StorageReconciliationAcquireResult,
 } from "./types.js";
 import { isDeleteSessionID } from "../ids.js";
 
@@ -517,6 +518,40 @@ export class StorageApiAsync implements StorageAPI {
     this.dbClient.trackCoValuesSyncState(updates).then(() => done?.());
   }
 
+  getCoValueIDs(
+    limit: number,
+    offset: number,
+    callback: (batch: { id: RawCoID }[]) => void,
+  ): void {
+    this.dbClient.getCoValueIDs(limit, offset).then(callback);
+  }
+
+  getCoValueCount(callback: (count: number) => void): void {
+    this.dbClient.getCoValueCount().then(callback);
+  }
+
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    callback: (result: StorageReconciliationAcquireResult) => void,
+  ): void {
+    this.dbClient
+      .tryAcquireStorageReconciliationLock(sessionId, peerId)
+      .then(callback);
+  }
+
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): void {
+    this.dbClient.renewStorageReconciliationLock(sessionId, peerId, offset);
+  }
+
+  releaseStorageReconciliationLock(sessionId: SessionID, peerId: PeerID): void {
+    this.dbClient.releaseStorageReconciliationLock(sessionId, peerId);
+  }
+
   getUnsyncedCoValueIDs(
     callback: (unsyncedCoValueIDs: RawCoID[]) => void,
   ): void {
@@ -529,11 +564,13 @@ export class StorageApiAsync implements StorageAPI {
 
   onCoValueUnmounted(id: RawCoID): void {
     this.inMemoryCoValues.delete(id);
+    this.knownStates.deleteKnownState(id);
   }
 
   close() {
     this.deletedCoValuesEraserScheduler?.dispose();
     this.inMemoryCoValues.clear();
+    this.knownStates.clear();
     return this.storeQueue.close();
   }
 }

package/src/storage/storageSync.ts

@@ -29,6 +29,7 @@ import type {
   SignatureAfterRow,
   StoredCoValueRow,
   StoredSessionRow,
+  StorageReconciliationAcquireResult,
 } from "./types.js";
 import { DeletedCoValuesEraserScheduler } from "./DeletedCoValuesEraserScheduler.js";
 import {
@@ -68,26 +69,48 @@ export class StorageApiSync implements StorageAPI {
     return this.knownStates.getKnownState(id);
   }
 
-  loadKnownState(
-    id: string,
-    callback: (knownState: CoValueKnownState | undefined) => void,
+  getCoValueIDs(
+    limit: number,
+    offset: number,
+    callback: (batch: { id: RawCoID }[]) => void,
   ): void {
-    // Check in-memory cache first
-    const cached = this.knownStates.getCachedKnownState(id);
-    if (cached) {
-      callback(cached);
-      return;
-    }
+    const batch = this.dbClient.getCoValueIDs(limit, offset);
+    callback(batch);
+  }
 
-    // Load from database
-    const knownState = this.dbClient.getCoValueKnownState(id);
+  getCoValueCount(callback: (count: number) => void): void {
+    callback(this.dbClient.getCoValueCount());
+  }
 
-    if (knownState) {
-      // Cache for future use
-      this.knownStates.setKnownState(id, knownState);
-    }
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    callback: (result: StorageReconciliationAcquireResult) => void,
+  ): void {
+    const result = this.dbClient.tryAcquireStorageReconciliationLock(
+      sessionId,
+      peerId,
+    );
+    callback(result);
+  }
+
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): void {
+    this.dbClient.renewStorageReconciliationLock(sessionId, peerId, offset);
+  }
+
+  releaseStorageReconciliationLock(sessionId: SessionID, peerId: PeerID): void {
+    this.dbClient.releaseStorageReconciliationLock(sessionId, peerId);
+  }
 
-    callback(knownState);
+  loadKnownState(
+    id: string,
+    callback: (knownState: CoValueKnownState | undefined) => void,
+  ): void {
+    callback(this.dbClient.getCoValueKnownState(id));
   }
 
   async load(
@@ -515,11 +538,13 @@ export class StorageApiSync implements StorageAPI {
 
   onCoValueUnmounted(id: RawCoID): void {
     this.inMemoryCoValues.delete(id);
+    this.knownStates.deleteKnownState(id);
   }
 
   close() {
     this.deletedCoValuesEraserScheduler?.dispose();
     this.inMemoryCoValues.clear();
+    this.knownStates.clear();
     return undefined;
   }
 }

package/src/storage/types.ts

@@ -13,6 +13,10 @@ export type CorrectionCallback = (
   correction: CoValueKnownState,
 ) => NewContentMessage[] | undefined;
 
+export type StorageReconciliationAcquireResult =
+  | { acquired: true; lastProcessedOffset: number }
+  | { acquired: false; reason: "not_due" | "lock_held" };
+
 /**
  * Deletion work queue status for `deletedCoValues` (SQLite).
  *
@@ -86,6 +90,52 @@ export interface StorageAPI {
    */
   stopTrackingSyncState(id: RawCoID): void;
 
+  /**
+   * Get a batch of CoValue IDs from storage.
+   * Used for full storage reconciliation. Call repeatedly with increasing offset
+   * until the returned batch has length < limit (or 0) to enumerate all IDs.
+   * @param limit - Max number of IDs to return (e.g. 100).
+   * @param offset - Number of IDs to skip (0 for first batch).
+   * @param callback - Called with the batch. Ordering must be stable (e.g. by id).
+   */
+  getCoValueIDs(
+    limit: number,
+    offset: number,
+    callback: (batch: { id: RawCoID }[]) => void,
+  ): void;
+
+  /**
+   * Get the total number of CoValues in storage.
+   */
+  getCoValueCount(callback: (count: number) => void): void;
+
+  /**
+   * Try to acquire the storage reconciliation lock for a given peer.
+   * Atomically checks if reconciliation is due for this peer (lastRun older than 30 days or missing)
+   * and if no other process/tab holds the lock for this peer, then acquires it.
+   */
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    callback: (result: StorageReconciliationAcquireResult) => void,
+  ): void;
+
+  /**
+   * Update the last processed offset for the storage reconciliation lock held for this peer.
+   * Only call after a batch has been acked; used to resume from this offset on interrupt.
+   */
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): void;
+
+  /**
+   * Release the storage reconciliation lock for a peer and record completion. Only call on successful completion.
+   * On failure/interrupt, do not call; the lock expires after LOCK_TTL_MS and another process can retry for this peer.
+   */
+  releaseStorageReconciliationLock(sessionId: SessionID, peerId: PeerID): void;
+
   /**
    * Load only the knownState (header presence + session counters) for a CoValue.
    * This is more efficient than load() when we only need to check if a peer needs new content.
@@ -136,6 +186,15 @@ export type SignatureAfterRow = {
   signature: Signature;
 };
 
+export type StorageReconciliationLockRow = {
+  key: string;
+  holderSessionId: SessionID;
+  acquiredAt: number;
+  releasedAt?: number;
+  /** Offset up to which all batches have been acked; used to resume after interrupt. */
+  lastProcessedOffset: number;
+};
+
 export interface DBTransactionInterfaceAsync {
   getSingleCoValueSession(
     coValueRowId: number,
@@ -176,6 +235,14 @@ export interface DBTransactionInterfaceAsync {
   deleteCoValueContent(
     coValueRow: Pick<StoredCoValueRow, "rowID" | "id">,
   ): Promise<unknown>;
+
+  getStorageReconciliationLock(
+    key: string,
+  ): Promise<StorageReconciliationLockRow | undefined>;
+
+  putStorageReconciliationLock(
+    entry: StorageReconciliationLockRow,
+  ): Promise<void>;
 }
 
 export interface DBClientInterfaceAsync {
@@ -232,6 +299,26 @@ export interface DBClientInterfaceAsync {
   getCoValueKnownState(
     coValueId: string,
   ): Promise<CoValueKnownState | undefined>;
+
+  getCoValueIDs(limit: number, offset: number): Promise<{ id: RawCoID }[]>;
+
+  getCoValueCount(): Promise<number>;
+
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): Promise<StorageReconciliationAcquireResult>;
+
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): Promise<void>;
+
+  releaseStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): Promise<void>;
 }
 
 export interface DBTransactionInterfaceSync {
@@ -270,6 +357,12 @@ export interface DBTransactionInterfaceSync {
     idx: number;
     signature: Signature;
   }): number | undefined | unknown;
+
+  getStorageReconciliationLock(
+    key: string,
+  ): StorageReconciliationLockRow | undefined;
+
+  putStorageReconciliationLock(entry: StorageReconciliationLockRow): void;
 }
 
 export interface DBClientInterfaceSync {
@@ -317,4 +410,21 @@ export interface DBClientInterfaceSync {
    * Returns undefined if the CoValue doesn't exist.
    */
   getCoValueKnownState(coValueId: string): CoValueKnownState | undefined;
+
+  getCoValueIDs(limit: number, offset: number): { id: RawCoID }[];
+
+  getCoValueCount(): number;
+
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): StorageReconciliationAcquireResult;
+
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): void;
+
+  releaseStorageReconciliationLock(sessionId: SessionID, peerId: PeerID): void;
 }