cojson 0.20.8 → 0.20.10

package/src/queue/OutgoingLoadQueue.ts

@@ -1,3 +1,4 @@
+import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
 import { CO_VALUE_LOADING_CONFIG } from "../config.js";
 import { CoValueCore } from "../exports.js";
 import type { RawCoID } from "../ids.js";
@@ -10,6 +11,11 @@ interface PendingLoad {
   sendCallback: () => void;
 }
 
+interface InFlightLoad {
+  value: CoValueCore;
+  sentAt: number;
+}
+
 /**
  * Mode for enqueuing load requests:
  * - "high-priority" (default): high priority, processed in order
@@ -17,6 +23,7 @@ interface PendingLoad {
  * - "immediate": bypasses the queue entirely, executes immediately
  */
 export type LoadMode = "low-priority" | "immediate" | "high-priority";
+export type LoadCompletionSource = "content" | "known";
 
 /**
  * A queue that manages outgoing load requests with throttling.
@@ -28,7 +35,8 @@ export type LoadMode = "low-priority" | "immediate" | "high-priority";
  * - Manages timeouts for in-flight loads with a single timer
  */
 export class OutgoingLoadQueue {
-  private inFlightLoads: Map<CoValueCore, number> = new Map();
+  private inFlightLoads: Map<RawCoID, InFlightLoad> = new Map();
+  private inFlightCounter: UpDownCounter;
   private highPriorityPending: LinkedList<PendingLoad> = meteredList(
     "load-requests-queue",
     { priority: "high" },
@@ -49,7 +57,18 @@ export class OutgoingLoadQueue {
     new Map();
   private timeoutHandle: ReturnType<typeof setTimeout> | null = null;
 
-  constructor(private peerId: PeerID) {}
+  constructor(private peerId: PeerID) {
+    this.inFlightCounter = metrics
+      .getMeter("cojson")
+      .createUpDownCounter("jazz.loadqueue.outgoing.inflight", {
+        description: "Number of in-flight outgoing load requests",
+        unit: "1",
+        valueType: ValueType.INT,
+      });
+
+    // Emit an initial 0 value so the series appears immediately.
+    this.inFlightCounter.add(0);
+  }
 
   /**
    * Check if we can send another load request.
@@ -66,10 +85,20 @@ export class OutgoingLoadQueue {
    */
   private trackSent(coValue: CoValueCore): void {
     const now = performance.now();
-    this.inFlightLoads.set(coValue, now);
+    this.inFlightLoads.set(coValue.id, { value: coValue, sentAt: now });
+    this.inFlightCounter.add(1);
     this.scheduleTimeoutCheck(CO_VALUE_LOADING_CONFIG.TIMEOUT);
   }
 
+  private untrackInFlight(id: RawCoID): boolean {
+    if (!this.inFlightLoads.delete(id)) {
+      return false;
+    }
+
+    this.inFlightCounter.add(-1);
+    return true;
+  }
+
   /**
    * Schedule a timeout check if not already scheduled.
    * Uses a single timer to check all in-flight loads.
@@ -92,7 +121,7 @@ export class OutgoingLoadQueue {
     const now = performance.now();
 
     let nextTimeout: number | undefined;
-    for (const [coValue, sentAt] of this.inFlightLoads.entries()) {
+    for (const { value: coValue, sentAt } of this.inFlightLoads.values()) {
       const timeout = sentAt + CO_VALUE_LOADING_CONFIG.TIMEOUT;
 
       if (now >= timeout) {
@@ -101,7 +130,8 @@ export class OutgoingLoadQueue {
             id: coValue.id,
             peerId: this.peerId,
           });
-          coValue.markNotFoundInPeer(this.peerId);
+          // Re-resolve by ID to avoid mutating a stale CoValue instance.
+          coValue.node.getCoValue(coValue.id).markNotFoundInPeer(this.peerId);
         } else if (coValue.isStreaming()) {
           logger.warn(
             "Content streaming is taking more than " +
@@ -116,8 +146,9 @@ export class OutgoingLoadQueue {
           );
         }
 
-        this.inFlightLoads.delete(coValue);
-        this.processQueue();
+        if (this.untrackInFlight(coValue.id)) {
+          this.processQueue();
+        }
       } else {
         nextTimeout = Math.min(nextTimeout ?? Infinity, timeout - now);
       }
@@ -130,30 +161,37 @@ export class OutgoingLoadQueue {
   }
 
   trackUpdate(coValue: CoValueCore): void {
-    if (!this.inFlightLoads.has(coValue)) {
+    if (!this.inFlightLoads.has(coValue.id)) {
       return;
     }
 
     // Refresh the timeout for the in-flight load
-    this.inFlightLoads.set(coValue, performance.now());
+    this.inFlightLoads.set(coValue.id, {
+      value: coValue,
+      sentAt: performance.now(),
+    });
   }
 
   /**
    * Track that a load request has completed.
    * Triggers processing of pending requests.
    */
-  trackComplete(coValue: CoValueCore): void {
-    if (!this.inFlightLoads.has(coValue)) {
+  trackComplete(
+    coValue: CoValueCore,
+    source: LoadCompletionSource = "content",
+  ): void {
+    if (!this.inFlightLoads.has(coValue.id)) {
       return;
     }
 
-    if (coValue.isStreaming()) {
+    if (source === "content" && coValue.isStreaming()) {
       // wait for the next chunk
       return;
     }
 
-    this.inFlightLoads.delete(coValue);
-    this.processQueue();
+    if (this.untrackInFlight(coValue.id)) {
+      this.processQueue();
+    }
   }
 
   /**
@@ -170,7 +208,7 @@ export class OutgoingLoadQueue {
     sendCallback: () => void,
     mode: LoadMode = "high-priority",
   ): void {
-    if (this.inFlightLoads.has(value)) {
+    if (this.inFlightLoads.has(value.id)) {
       return;
     }
 
@@ -270,7 +308,11 @@ export class OutgoingLoadQueue {
       clearTimeout(this.timeoutHandle);
       this.timeoutHandle = null;
     }
+    const inFlightCount = this.inFlightLoads.size;
     this.inFlightLoads.clear();
+    if (inFlightCount > 0) {
+      this.inFlightCounter.add(-inFlightCount);
+    }
 
     // Drain existing queues to balance push/pull metrics
     while (this.highPriorityPending.shift()) {}

package/src/queue/PriorityBasedMessageQueue.ts

@@ -32,7 +32,11 @@ export class PriorityBasedMessageQueue {
   }
 
   public push(msg: SyncMessage) {
-    const priority = "priority" in msg ? msg.priority : this.defaultPriority;
+    let priority = "priority" in msg ? msg.priority : this.defaultPriority;
+
+    if (msg.action === "reconcile") {
+      priority = CO_VALUE_PRIORITY.LOW;
+    }
 
     this.getQueue(priority).push(msg);
   }
@@ -42,4 +46,14 @@ export class PriorityBasedMessageQueue {
 
     return this.queues[priority]?.shift();
   }
+
+  public trackPushPull(msg: SyncMessage) {
+    let priority = "priority" in msg ? msg.priority : this.defaultPriority;
+
+    if (msg.action === "reconcile") {
+      priority = CO_VALUE_PRIORITY.LOW;
+    }
+
+    this.getQueue(priority).trackPushPull();
+  }
 }

package/src/storage/knownState.ts

@@ -89,6 +89,20 @@ export class StorageKnownState {
       requests.add(req);
     });
   }
+
+  deleteKnownState(id: string) {
+    this.knownStates.delete(id);
+
+    for (const request of this.waitForSyncRequests.get(id) || []) {
+      request.resolve();
+    }
+    this.waitForSyncRequests.delete(id);
+  }
+
+  clear() {
+    this.knownStates.clear();
+    this.waitForSyncRequests.clear();
+  }
 }
 
 function isInSync(

package/src/storage/sqlite/client.ts

@@ -12,12 +12,15 @@ import type {
   DBTransactionInterfaceSync,
   SessionRow,
   SignatureAfterRow,
+  StorageReconciliationLockRow,
   StoredCoValueRow,
   StoredSessionRow,
   TransactionRow,
+  StorageReconciliationAcquireResult,
 } from "../types.js";
 import { DeletedCoValueDeletionStatus } from "../types.js";
 import type { SQLiteDatabaseDriver } from "./types.js";
+import { STORAGE_RECONCILIATION_CONFIG } from "../../config.js";
 
 export type RawCoValueRow = {
   id: RawCoID;
@@ -268,11 +271,55 @@ export class SQLiteClient
     );
   }
 
+  getStorageReconciliationLock(
+    key: string,
+  ): StorageReconciliationLockRow | undefined {
+    return this.db.get<StorageReconciliationLockRow>(
+      "SELECT * FROM storageReconciliationLocks WHERE key = ?",
+      [key],
+    );
+  }
+
+  putStorageReconciliationLock(entry: StorageReconciliationLockRow): void {
+    const {
+      key,
+      holderSessionId,
+      acquiredAt,
+      releasedAt,
+      lastProcessedOffset,
+    } = entry;
+    this.db.run(
+      `INSERT OR REPLACE INTO storageReconciliationLocks (key, holderSessionId, acquiredAt, releasedAt, lastProcessedOffset) VALUES (?, ?, ?, ?, ?)`,
+      [
+        key,
+        holderSessionId,
+        acquiredAt,
+        releasedAt ?? null,
+        lastProcessedOffset,
+      ],
+    );
+  }
+
   transaction(operationsCallback: (tx: DBTransactionInterfaceSync) => unknown) {
     this.db.transaction(() => operationsCallback(this));
     return undefined;
   }
 
+  getCoValueIDs(limit: number, offset: number): { id: RawCoID }[] {
+    return this.db.query<{ id: RawCoID }>(
+      "SELECT id FROM coValues WHERE rowID > ? ORDER BY rowID LIMIT ?",
+      [offset, limit],
+    );
+  }
+
+  getCoValueCount(): number {
+    const row = this.db.get<{ count: number }>(
+      "SELECT COUNT(*) as count FROM coValues",
+      [],
+    );
+    return row?.count ?? 0;
+  }
+
   getUnsyncedCoValueIDs(): RawCoID[] {
     const rows = this.db.query<{ co_value_id: RawCoID }>(
       "SELECT DISTINCT co_value_id FROM unsynced_covalues",
@@ -303,6 +350,87 @@ export class SQLiteClient
     this.db.run("DELETE FROM unsynced_covalues WHERE co_value_id = ?", [id]);
   }
 
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): StorageReconciliationAcquireResult {
+    let result: StorageReconciliationAcquireResult = {
+      acquired: false,
+      reason: "not_due",
+    };
+    this.transaction(() => {
+      const now = Date.now();
+      const lockKey = `lock#${peerId}`;
+      const lockRow = this.getStorageReconciliationLock(lockKey);
+      if (
+        lockRow?.releasedAt &&
+        now - lockRow.releasedAt <
+          STORAGE_RECONCILIATION_CONFIG.RECONCILIATION_INTERVAL_MS
+      ) {
+        result = { acquired: false, reason: "not_due" };
+        return;
+      }
+      const expiresAt = lockRow
+        ? lockRow.acquiredAt + STORAGE_RECONCILIATION_CONFIG.LOCK_TTL_MS
+        : 0;
+      const isLockHeldByOtherSession = lockRow?.holderSessionId !== sessionId;
+      if (
+        lockRow &&
+        !lockRow.releasedAt &&
+        expiresAt >= now &&
+        isLockHeldByOtherSession
+      ) {
+        result = { acquired: false, reason: "lock_held" };
+        return;
+      }
+
+      const lastProcessedOffset =
+        lockRow && !lockRow.releasedAt ? (lockRow.lastProcessedOffset ?? 0) : 0;
+      this.putStorageReconciliationLock({
+        key: lockKey,
+        holderSessionId: sessionId,
+        acquiredAt: now,
+        lastProcessedOffset,
+      });
+      result = { acquired: true, lastProcessedOffset };
+    });
+    return result;
+  }
+
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): void {
+    const lockKey = `lock#${peerId}`;
+    const lockRow = this.getStorageReconciliationLock(lockKey);
+    if (
+      lockRow &&
+      lockRow.holderSessionId === sessionId &&
+      !lockRow.releasedAt
+    ) {
+      this.putStorageReconciliationLock({
+        ...lockRow,
+        lastProcessedOffset: offset,
+      });
+    }
+  }
+
+  releaseStorageReconciliationLock(sessionId: SessionID, peerId: PeerID): void {
+    this.transaction(() => {
+      const lockKey = `lock#${peerId}`;
+      const releasedAt = Date.now();
+      const lockRow = this.getStorageReconciliationLock(lockKey);
+      if (lockRow?.holderSessionId === sessionId) {
+        this.putStorageReconciliationLock({
+          ...lockRow,
+          releasedAt,
+          lastProcessedOffset: 0,
+        });
+      }
+    });
+  }
+
   getCoValueKnownState(coValueId: string): CoValueKnownState | undefined {
     // First check if the CoValue exists
     const coValueRow = this.db.get<{ rowID: number }>(

package/src/storage/sqlite/sqliteMigrations.ts

@@ -47,6 +47,17 @@ export const migrations: Record<number, string[]> = {
     ) WITHOUT ROWID;`,
     "CREATE INDEX IF NOT EXISTS deletedCoValuesByStatus ON deletedCoValues (status);",
   ],
+  6: [
+    `CREATE TABLE IF NOT EXISTS storageReconciliationLocks (
+      key TEXT PRIMARY KEY,
+      holderSessionId TEXT NOT NULL,
+      acquiredAt INTEGER NOT NULL,
+      expiresAt INTEGER NOT NULL,
+      lastProcessedOffset INTEGER NOT NULL DEFAULT 0,
+      releasedAt INTEGER
+    ) WITHOUT ROWID;`,
+  ],
+  7: ["ALTER TABLE storageReconciliationLocks DROP COLUMN expiresAt;"],
 };
 
 type Migration = {

package/src/storage/sqliteAsync/client.ts

@@ -11,13 +11,16 @@ import type {
   DBTransactionInterfaceAsync,
   SessionRow,
   SignatureAfterRow,
+  StorageReconciliationLockRow,
   StoredCoValueRow,
   StoredSessionRow,
   TransactionRow,
+  StorageReconciliationAcquireResult,
 } from "../types.js";
 import { DeletedCoValueDeletionStatus } from "../types.js";
 import type { SQLiteDatabaseDriverAsync } from "./types.js";
 import type { PeerID } from "../../sync.js";
+import { STORAGE_RECONCILIATION_CONFIG } from "../../config.js";
 
 export type RawCoValueRow = {
   id: RawCoID;
@@ -153,6 +156,37 @@ export class SQLiteTransactionAsync implements DBTransactionInterfaceAsync {
       ],
     );
   }
+
+  async getStorageReconciliationLock(
+    key: string,
+  ): Promise<StorageReconciliationLockRow | undefined> {
+    return this.tx.get<StorageReconciliationLockRow>(
+      "SELECT * FROM storageReconciliationLocks WHERE key = ?",
+      [key],
+    );
+  }
+
+  async putStorageReconciliationLock(
+    entry: StorageReconciliationLockRow,
+  ): Promise<void> {
+    const {
+      key,
+      holderSessionId,
+      acquiredAt,
+      releasedAt,
+      lastProcessedOffset,
+    } = entry;
+    await this.tx.run(
+      `INSERT OR REPLACE INTO storageReconciliationLocks (key, holderSessionId, acquiredAt, releasedAt, lastProcessedOffset) VALUES (?, ?, ?, ?, ?)`,
+      [
+        key,
+        holderSessionId,
+        acquiredAt,
+        releasedAt ?? null,
+        lastProcessedOffset,
+      ],
+    );
+  }
 }
 
 export class SQLiteClientAsync implements DBClientInterfaceAsync {
@@ -334,6 +368,111 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     ]);
   }
 
+  async getCoValueIDs(
+    limit: number,
+    offset: number,
+  ): Promise<{ id: RawCoID }[]> {
+    return this.db.query<{ id: RawCoID }>(
+      "SELECT id FROM coValues WHERE rowID > ? ORDER BY rowID LIMIT ?",
+      [offset, limit],
+    );
+  }
+
+  async getCoValueCount(): Promise<number> {
+    const row = await this.db.get<{ count: number }>(
+      "SELECT COUNT(*) as count FROM coValues",
+      [],
+    );
+    return row?.count ?? 0;
+  }
+
+  async tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): Promise<StorageReconciliationAcquireResult> {
+    let result: StorageReconciliationAcquireResult = {
+      acquired: false,
+      reason: "not_due",
+    };
+    await this.transaction(async (tx) => {
+      const now = Date.now();
+      const lockKey = `lock#${peerId}`;
+
+      const lockRow = await tx.getStorageReconciliationLock(lockKey);
+      if (
+        lockRow?.releasedAt &&
+        now - lockRow.releasedAt <
+          STORAGE_RECONCILIATION_CONFIG.RECONCILIATION_INTERVAL_MS
+      ) {
+        result = { acquired: false, reason: "not_due" };
+        return;
+      }
+      const expiresAt = lockRow
+        ? lockRow.acquiredAt + STORAGE_RECONCILIATION_CONFIG.LOCK_TTL_MS
+        : 0;
+      const isLockHeldByOtherSession = lockRow?.holderSessionId !== sessionId;
+      if (
+        lockRow &&
+        !lockRow.releasedAt &&
+        expiresAt >= now &&
+        isLockHeldByOtherSession
+      ) {
+        result = { acquired: false, reason: "lock_held" };
+        return;
+      }
+
+      const lastProcessedOffset =
+        lockRow && !lockRow.releasedAt ? (lockRow.lastProcessedOffset ?? 0) : 0;
+      await tx.putStorageReconciliationLock({
+        key: lockKey,
+        holderSessionId: sessionId,
+        acquiredAt: now,
+        lastProcessedOffset,
+      });
+      result = { acquired: true, lastProcessedOffset };
+    });
+    return result;
+  }
+
+  async renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): Promise<void> {
+    await this.transaction(async (tx) => {
+      const lockKey = `lock#${peerId}`;
+      const lockRow = await tx.getStorageReconciliationLock(lockKey);
+      if (
+        lockRow &&
+        lockRow.holderSessionId === sessionId &&
+        !lockRow.releasedAt
+      ) {
+        await tx.putStorageReconciliationLock({
+          ...lockRow,
+          lastProcessedOffset: offset,
+        });
+      }
+    });
+  }
+
+  async releaseStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): Promise<void> {
+    await this.transaction(async (tx) => {
+      const lockKey = `lock#${peerId}`;
+      const releasedAt = Date.now();
+      const lockRow = await tx.getStorageReconciliationLock(lockKey);
+      if (lockRow && lockRow.holderSessionId === sessionId) {
+        await tx.putStorageReconciliationLock({
+          ...lockRow,
+          releasedAt,
+          lastProcessedOffset: 0,
+        });
+      }
+    });
+  }
+
   async getCoValueKnownState(
     coValueId: string,
   ): Promise<CoValueKnownState | undefined> {

package/src/storage/storageAsync.ts

@@ -30,6 +30,7 @@ import type {
   SignatureAfterRow,
   StoredCoValueRow,
   StoredSessionRow,
+  StorageReconciliationAcquireResult,
 } from "./types.js";
 import { isDeleteSessionID } from "../ids.js";
 
@@ -517,6 +518,40 @@ export class StorageApiAsync implements StorageAPI {
     this.dbClient.trackCoValuesSyncState(updates).then(() => done?.());
   }
 
+  getCoValueIDs(
+    limit: number,
+    offset: number,
+    callback: (batch: { id: RawCoID }[]) => void,
+  ): void {
+    this.dbClient.getCoValueIDs(limit, offset).then(callback);
+  }
+
+  getCoValueCount(callback: (count: number) => void): void {
+    this.dbClient.getCoValueCount().then(callback);
+  }
+
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    callback: (result: StorageReconciliationAcquireResult) => void,
+  ): void {
+    this.dbClient
+      .tryAcquireStorageReconciliationLock(sessionId, peerId)
+      .then(callback);
+  }
+
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): void {
+    this.dbClient.renewStorageReconciliationLock(sessionId, peerId, offset);
+  }
+
+  releaseStorageReconciliationLock(sessionId: SessionID, peerId: PeerID): void {
+    this.dbClient.releaseStorageReconciliationLock(sessionId, peerId);
+  }
+
   getUnsyncedCoValueIDs(
     callback: (unsyncedCoValueIDs: RawCoID[]) => void,
   ): void {
@@ -529,11 +564,13 @@ export class StorageApiAsync implements StorageAPI {
 
   onCoValueUnmounted(id: RawCoID): void {
     this.inMemoryCoValues.delete(id);
+    this.knownStates.deleteKnownState(id);
   }
 
   close() {
     this.deletedCoValuesEraserScheduler?.dispose();
     this.inMemoryCoValues.clear();
+    this.knownStates.clear();
     return this.storeQueue.close();
   }
 }