cojson 0.20.9 → 0.20.11

package/src/exports.ts

@@ -85,6 +85,7 @@ import {
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
 import {
   CO_VALUE_LOADING_CONFIG,
+  STORAGE_RECONCILIATION_CONFIG,
   TRANSACTION_CONFIG,
   WEBSOCKET_CONFIG,
   setCoValueLoadingMaxRetries,
@@ -94,6 +95,9 @@ import {
   setMaxInFlightLoadsPerPeer,
   setMaxOutgoingMessagesChunkBytes,
   setMaxRecommendedTxSize,
+  setStorageReconciliationBatchSize,
+  setStorageReconciliationLockTTL,
+  setStorageReconciliationInterval,
 } from "./config.js";
 import { LogLevel, logger } from "./logger.js";
 import { CO_VALUE_PRIORITY, getPriorityFromHeader } from "./priority.js";
@@ -147,6 +151,10 @@ export const cojsonInternals = {
   WEBSOCKET_CONFIG,
   setMaxOutgoingMessagesChunkBytes,
   setMaxInFlightLoadsPerPeer,
+  STORAGE_RECONCILIATION_CONFIG,
+  setStorageReconciliationBatchSize,
+  setStorageReconciliationLockTTL,
+  setStorageReconciliationInterval,
 };
 
 export {

package/src/localNode.ts

@@ -77,10 +77,14 @@ export class LocalNode {
     currentSessionID: SessionID,
     crypto: CryptoProvider,
     public readonly syncWhen?: SyncWhen,
+    enableFullStorageReconciliation?: boolean,
   ) {
     this.agentSecret = agentSecret;
     this.currentSessionID = currentSessionID;
     this.crypto = crypto;
+    if (enableFullStorageReconciliation) {
+      this.syncManager.fullStorageReconciliationEnabled = true;
+    }
   }
 
   enableGarbageCollector() {
@@ -133,6 +137,12 @@ export class LocalNode {
     );
   }
 
+  isCoValueInMemory(id: RawCoID) {
+    const state = this.coValues.get(id)?.loadingState;
+
+    return Boolean(state);
+  }
+
   getCoValue(id: RawCoID) {
     let entry = this.coValues.get(id);
 
@@ -256,6 +266,7 @@ export class LocalNode {
     peers?: Peer[];
     syncWhen?: SyncWhen;
     storage?: StorageAPI;
+    enableFullStorageReconciliation?: boolean;
   }): RawAccount {
     const {
       crypto,
@@ -274,6 +285,7 @@ export class LocalNode {
       crypto.newRandomSessionID(accountID as RawAccountID),
       crypto,
       syncWhen,
+      opts.enableFullStorageReconciliation,
     );
 
     if (opts.storage) {
@@ -320,6 +332,7 @@ export class LocalNode {
     crypto,
     initialAgentSecret = crypto.newRandomAgentSecret(),
     storage,
+    enableFullStorageReconciliation,
   }: {
     creationProps: { name: string };
     peers?: Peer[];
@@ -328,6 +341,7 @@ export class LocalNode {
     crypto: CryptoProvider;
     initialAgentSecret?: AgentSecret;
     storage?: StorageAPI;
+    enableFullStorageReconciliation?: boolean;
   }): Promise<{
     node: LocalNode;
     accountID: RawAccountID;
@@ -340,6 +354,7 @@ export class LocalNode {
       peers,
       syncWhen,
       storage,
+      enableFullStorageReconciliation,
     });
     const node = account.core.node;
 
@@ -385,6 +400,7 @@ export class LocalNode {
     crypto,
     migration,
     storage,
+    enableFullStorageReconciliation,
   }: {
     accountID: RawAccountID;
     accountSecret: AgentSecret;
@@ -394,6 +410,7 @@ export class LocalNode {
     crypto: CryptoProvider;
     migration?: RawAccountMigration<AccountMeta>;
     storage?: StorageAPI;
+    enableFullStorageReconciliation?: boolean;
   }): Promise<LocalNode> {
     try {
       const expectedAgentID = crypto.getAgentID(accountSecret);
@@ -403,6 +420,7 @@ export class LocalNode {
         sessionID || crypto.newRandomSessionID(accountID),
         crypto,
         syncWhen,
+        enableFullStorageReconciliation,
       );
 
       if (storage) {

package/src/permissions.ts

@@ -22,7 +22,7 @@ import { expectGroup } from "./typeUtils/expectGroup.js";
 
 export type PermissionsDef =
   | { type: "group"; initialAdmin: RawAccountID | AgentID }
-  | { type: "ownedByGroup"; group: RawCoID }
+  | { type: "ownedByGroup"; group: RawCoID; restrictDeletion?: boolean }
   | { type: "unsafeAllowAll" };
 
 export type AccountRole =

package/src/queue/LinkedList.ts

@@ -68,6 +68,10 @@ export class LinkedList<T> {
     return value;
   }
 
+  trackPushPull() {
+    this.meter?.trackPushPull();
+  }
+
   /**
    * Remove a specific node from the list in O(1) time.
    * The node must be a valid node that was returned by push().
@@ -136,7 +140,13 @@ class QueueMeter {
   public push() {
     this.pushCounter.add(1, this.attrs);
   }
+
+  public trackPushPull() {
+    this.pullCounter.add(1, this.attrs);
+    this.pushCounter.add(1, this.attrs);
+  }
 }
+
 export function meteredList<T>(
   type: "incoming" | "outgoing" | "storage-streaming" | "load-requests-queue",
   attrs?: Record<string, string | number>,

package/src/queue/OutgoingLoadQueue.ts

@@ -1,3 +1,4 @@
+import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
 import { CO_VALUE_LOADING_CONFIG } from "../config.js";
 import { CoValueCore } from "../exports.js";
 import type { RawCoID } from "../ids.js";
@@ -10,6 +11,11 @@ interface PendingLoad {
   sendCallback: () => void;
 }
 
+interface InFlightLoad {
+  value: CoValueCore;
+  sentAt: number;
+}
+
 /**
  * Mode for enqueuing load requests:
  * - "high-priority" (default): high priority, processed in order
@@ -17,6 +23,7 @@ interface PendingLoad {
  * - "immediate": bypasses the queue entirely, executes immediately
  */
 export type LoadMode = "low-priority" | "immediate" | "high-priority";
+export type LoadCompletionSource = "content" | "known";
 
 /**
  * A queue that manages outgoing load requests with throttling.
@@ -28,7 +35,8 @@ export type LoadMode = "low-priority" | "immediate" | "high-priority";
  * - Manages timeouts for in-flight loads with a single timer
  */
 export class OutgoingLoadQueue {
-  private inFlightLoads: Map<CoValueCore, number> = new Map();
+  private inFlightLoads: Map<RawCoID, InFlightLoad> = new Map();
+  private inFlightCounter: UpDownCounter;
   private highPriorityPending: LinkedList<PendingLoad> = meteredList(
     "load-requests-queue",
     { priority: "high" },
@@ -49,7 +57,18 @@ export class OutgoingLoadQueue {
     new Map();
   private timeoutHandle: ReturnType<typeof setTimeout> | null = null;
 
-  constructor(private peerId: PeerID) {}
+  constructor(private peerId: PeerID) {
+    this.inFlightCounter = metrics
+      .getMeter("cojson")
+      .createUpDownCounter("jazz.loadqueue.outgoing.inflight", {
+        description: "Number of in-flight outgoing load requests",
+        unit: "1",
+        valueType: ValueType.INT,
+      });
+
+    // Emit an initial 0 value so the series appears immediately.
+    this.inFlightCounter.add(0);
+  }
 
   /**
    * Check if we can send another load request.
@@ -66,10 +85,20 @@ export class OutgoingLoadQueue {
    */
   private trackSent(coValue: CoValueCore): void {
     const now = performance.now();
-    this.inFlightLoads.set(coValue, now);
+    this.inFlightLoads.set(coValue.id, { value: coValue, sentAt: now });
+    this.inFlightCounter.add(1);
     this.scheduleTimeoutCheck(CO_VALUE_LOADING_CONFIG.TIMEOUT);
   }
 
+  private untrackInFlight(id: RawCoID): boolean {
+    if (!this.inFlightLoads.delete(id)) {
+      return false;
+    }
+
+    this.inFlightCounter.add(-1);
+    return true;
+  }
+
   /**
    * Schedule a timeout check if not already scheduled.
    * Uses a single timer to check all in-flight loads.
@@ -92,7 +121,7 @@ export class OutgoingLoadQueue {
     const now = performance.now();
 
     let nextTimeout: number | undefined;
-    for (const [coValue, sentAt] of this.inFlightLoads.entries()) {
+    for (const { value: coValue, sentAt } of this.inFlightLoads.values()) {
       const timeout = sentAt + CO_VALUE_LOADING_CONFIG.TIMEOUT;
 
       if (now >= timeout) {
@@ -101,7 +130,8 @@ export class OutgoingLoadQueue {
             id: coValue.id,
             peerId: this.peerId,
           });
-          coValue.markNotFoundInPeer(this.peerId);
+          // Re-resolve by ID to avoid mutating a stale CoValue instance.
+          coValue.node.getCoValue(coValue.id).markNotFoundInPeer(this.peerId);
         } else if (coValue.isStreaming()) {
           logger.warn(
             "Content streaming is taking more than " +
@@ -116,8 +146,9 @@ export class OutgoingLoadQueue {
           );
         }
 
-        this.inFlightLoads.delete(coValue);
-        this.processQueue();
+        if (this.untrackInFlight(coValue.id)) {
+          this.processQueue();
+        }
       } else {
         nextTimeout = Math.min(nextTimeout ?? Infinity, timeout - now);
       }
@@ -130,30 +161,37 @@ export class OutgoingLoadQueue {
   }
 
   trackUpdate(coValue: CoValueCore): void {
-    if (!this.inFlightLoads.has(coValue)) {
+    if (!this.inFlightLoads.has(coValue.id)) {
       return;
     }
 
     // Refresh the timeout for the in-flight load
-    this.inFlightLoads.set(coValue, performance.now());
+    this.inFlightLoads.set(coValue.id, {
+      value: coValue,
+      sentAt: performance.now(),
+    });
   }
 
   /**
    * Track that a load request has completed.
    * Triggers processing of pending requests.
    */
-  trackComplete(coValue: CoValueCore): void {
-    if (!this.inFlightLoads.has(coValue)) {
+  trackComplete(
+    coValue: CoValueCore,
+    source: LoadCompletionSource = "content",
+  ): void {
+    if (!this.inFlightLoads.has(coValue.id)) {
       return;
     }
 
-    if (coValue.isStreaming()) {
+    if (source === "content" && coValue.isStreaming()) {
       // wait for the next chunk
       return;
     }
 
-    this.inFlightLoads.delete(coValue);
-    this.processQueue();
+    if (this.untrackInFlight(coValue.id)) {
+      this.processQueue();
+    }
   }
 
   /**
@@ -170,7 +208,7 @@ export class OutgoingLoadQueue {
     sendCallback: () => void,
     mode: LoadMode = "high-priority",
   ): void {
-    if (this.inFlightLoads.has(value)) {
+    if (this.inFlightLoads.has(value.id)) {
       return;
     }
 
@@ -270,7 +308,11 @@ export class OutgoingLoadQueue {
       clearTimeout(this.timeoutHandle);
       this.timeoutHandle = null;
     }
+    const inFlightCount = this.inFlightLoads.size;
     this.inFlightLoads.clear();
+    if (inFlightCount > 0) {
+      this.inFlightCounter.add(-inFlightCount);
+    }
 
     // Drain existing queues to balance push/pull metrics
     while (this.highPriorityPending.shift()) {}

package/src/queue/PriorityBasedMessageQueue.ts

@@ -32,7 +32,11 @@ export class PriorityBasedMessageQueue {
   }
 
   public push(msg: SyncMessage) {
-    const priority = "priority" in msg ? msg.priority : this.defaultPriority;
+    let priority = "priority" in msg ? msg.priority : this.defaultPriority;
+
+    if (msg.action === "reconcile") {
+      priority = CO_VALUE_PRIORITY.LOW;
+    }
 
     this.getQueue(priority).push(msg);
   }
@@ -42,4 +46,14 @@ export class PriorityBasedMessageQueue {
 
     return this.queues[priority]?.shift();
   }
+
+  public trackPushPull(msg: SyncMessage) {
+    let priority = "priority" in msg ? msg.priority : this.defaultPriority;
+
+    if (msg.action === "reconcile") {
+      priority = CO_VALUE_PRIORITY.LOW;
+    }
+
+    this.getQueue(priority).trackPushPull();
+  }
 }

package/src/storage/knownState.ts

@@ -89,6 +89,20 @@ export class StorageKnownState {
       requests.add(req);
     });
   }
+
+  deleteKnownState(id: string) {
+    this.knownStates.delete(id);
+
+    for (const request of this.waitForSyncRequests.get(id) || []) {
+      request.resolve();
+    }
+    this.waitForSyncRequests.delete(id);
+  }
+
+  clear() {
+    this.knownStates.clear();
+    this.waitForSyncRequests.clear();
+  }
 }
 
 function isInSync(

package/src/storage/sqlite/client.ts

@@ -12,12 +12,15 @@ import type {
   DBTransactionInterfaceSync,
   SessionRow,
   SignatureAfterRow,
+  StorageReconciliationLockRow,
   StoredCoValueRow,
   StoredSessionRow,
   TransactionRow,
+  StorageReconciliationAcquireResult,
 } from "../types.js";
 import { DeletedCoValueDeletionStatus } from "../types.js";
 import type { SQLiteDatabaseDriver } from "./types.js";
+import { STORAGE_RECONCILIATION_CONFIG } from "../../config.js";
 
 export type RawCoValueRow = {
   id: RawCoID;
@@ -268,11 +271,55 @@ export class SQLiteClient
     );
   }
 
+  getStorageReconciliationLock(
+    key: string,
+  ): StorageReconciliationLockRow | undefined {
+    return this.db.get<StorageReconciliationLockRow>(
+      "SELECT * FROM storageReconciliationLocks WHERE key = ?",
+      [key],
+    );
+  }
+
+  putStorageReconciliationLock(entry: StorageReconciliationLockRow): void {
+    const {
+      key,
+      holderSessionId,
+      acquiredAt,
+      releasedAt,
+      lastProcessedOffset,
+    } = entry;
+    this.db.run(
+      `INSERT OR REPLACE INTO storageReconciliationLocks (key, holderSessionId, acquiredAt, releasedAt, lastProcessedOffset) VALUES (?, ?, ?, ?, ?)`,
+      [
+        key,
+        holderSessionId,
+        acquiredAt,
+        releasedAt ?? null,
+        lastProcessedOffset,
+      ],
+    );
+  }
+
   transaction(operationsCallback: (tx: DBTransactionInterfaceSync) => unknown) {
     this.db.transaction(() => operationsCallback(this));
     return undefined;
   }
 
+  getCoValueIDs(limit: number, offset: number): { id: RawCoID }[] {
+    return this.db.query<{ id: RawCoID }>(
+      "SELECT id FROM coValues WHERE rowID > ? ORDER BY rowID LIMIT ?",
+      [offset, limit],
+    );
+  }
+
+  getCoValueCount(): number {
+    const row = this.db.get<{ count: number }>(
+      "SELECT COUNT(*) as count FROM coValues",
+      [],
+    );
+    return row?.count ?? 0;
+  }
+
   getUnsyncedCoValueIDs(): RawCoID[] {
     const rows = this.db.query<{ co_value_id: RawCoID }>(
       "SELECT DISTINCT co_value_id FROM unsynced_covalues",
@@ -303,6 +350,87 @@ export class SQLiteClient
     this.db.run("DELETE FROM unsynced_covalues WHERE co_value_id = ?", [id]);
   }
 
+  tryAcquireStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+  ): StorageReconciliationAcquireResult {
+    let result: StorageReconciliationAcquireResult = {
+      acquired: false,
+      reason: "not_due",
+    };
+    this.transaction(() => {
+      const now = Date.now();
+      const lockKey = `lock#${peerId}`;
+      const lockRow = this.getStorageReconciliationLock(lockKey);
+      if (
+        lockRow?.releasedAt &&
+        now - lockRow.releasedAt <
+          STORAGE_RECONCILIATION_CONFIG.RECONCILIATION_INTERVAL_MS
+      ) {
+        result = { acquired: false, reason: "not_due" };
+        return;
+      }
+      const expiresAt = lockRow
+        ? lockRow.acquiredAt + STORAGE_RECONCILIATION_CONFIG.LOCK_TTL_MS
+        : 0;
+      const isLockHeldByOtherSession = lockRow?.holderSessionId !== sessionId;
+      if (
+        lockRow &&
+        !lockRow.releasedAt &&
+        expiresAt >= now &&
+        isLockHeldByOtherSession
+      ) {
+        result = { acquired: false, reason: "lock_held" };
+        return;
+      }
+
+      const lastProcessedOffset =
+        lockRow && !lockRow.releasedAt ? (lockRow.lastProcessedOffset ?? 0) : 0;
+      this.putStorageReconciliationLock({
+        key: lockKey,
+        holderSessionId: sessionId,
+        acquiredAt: now,
+        lastProcessedOffset,
+      });
+      result = { acquired: true, lastProcessedOffset };
+    });
+    return result;
+  }
+
+  renewStorageReconciliationLock(
+    sessionId: SessionID,
+    peerId: PeerID,
+    offset: number,
+  ): void {
+    const lockKey = `lock#${peerId}`;
+    const lockRow = this.getStorageReconciliationLock(lockKey);
+    if (
+      lockRow &&
+      lockRow.holderSessionId === sessionId &&
+      !lockRow.releasedAt
+    ) {
+      this.putStorageReconciliationLock({
+        ...lockRow,
+        lastProcessedOffset: offset,
+      });
+    }
+  }
+
+  releaseStorageReconciliationLock(sessionId: SessionID, peerId: PeerID): void {
+    this.transaction(() => {
+      const lockKey = `lock#${peerId}`;
+      const releasedAt = Date.now();
+      const lockRow = this.getStorageReconciliationLock(lockKey);
+      if (lockRow?.holderSessionId === sessionId) {
+        this.putStorageReconciliationLock({
+          ...lockRow,
+          releasedAt,
+          lastProcessedOffset: 0,
+        });
+      }
+    });
+  }
+
   getCoValueKnownState(coValueId: string): CoValueKnownState | undefined {
     // First check if the CoValue exists
     const coValueRow = this.db.get<{ rowID: number }>(

package/src/storage/sqlite/sqliteMigrations.ts

@@ -47,6 +47,17 @@ export const migrations: Record<number, string[]> = {
     ) WITHOUT ROWID;`,
     "CREATE INDEX IF NOT EXISTS deletedCoValuesByStatus ON deletedCoValues (status);",
   ],
+  6: [
+    `CREATE TABLE IF NOT EXISTS storageReconciliationLocks (
+      key TEXT PRIMARY KEY,
+      holderSessionId TEXT NOT NULL,
+      acquiredAt INTEGER NOT NULL,
+      expiresAt INTEGER NOT NULL,
+      lastProcessedOffset INTEGER NOT NULL DEFAULT 0,
+      releasedAt INTEGER
+    ) WITHOUT ROWID;`,
+  ],
+  7: ["ALTER TABLE storageReconciliationLocks DROP COLUMN expiresAt;"],
 };
 
 type Migration = {