cojson 0.19.21 → 0.20.0

package/src/storage/sqlite/client.ts

@@ -16,6 +16,7 @@ import type {
   StoredSessionRow,
   TransactionRow,
 } from "../types.js";
+import { DeletedCoValueDeletionStatus } from "../types.js";
 import type { SQLiteDatabaseDriver } from "./types.js";
 
 export type RawCoValueRow = {
@@ -29,6 +30,10 @@ export type RawTransactionRow = {
   tx: string;
 };
 
+type DeletedCoValueQueueRow = {
+  id: RawCoID;
+};
+
 export function getErrorMessage(error: unknown) {
   return error instanceof Error ? error.message : "Unknown error";
 }
@@ -143,6 +148,78 @@ export class SQLiteClient
     return result.rowID;
   }
 
+  markCoValueAsDeleted(id: RawCoID) {
+    // Work queue entry. Table only stores the coValueID.
+    // Idempotent by design.
+    this.db.run(
+      `INSERT INTO deletedCoValues (coValueID) VALUES (?) ON CONFLICT(coValueID) DO NOTHING`,
+      [id],
+    );
+  }
+
+  eraseCoValueButKeepTombstone(coValueId: RawCoID) {
+    const coValueRow = this.db.get<{ rowID: number }>(
+      "SELECT rowID FROM coValues WHERE id = ?",
+      [coValueId],
+    );
+
+    if (!coValueRow) {
+      logger.warn(`CoValue ${coValueId} not found, skipping deletion`);
+      return;
+    }
+
+    this.transaction(() => {
+      this.db.run(
+        `DELETE FROM transactions
+       WHERE ses IN (
+         SELECT rowID FROM sessions
+         WHERE coValue = ?
+           AND sessionID NOT LIKE '%$'
+       )`,
+        [coValueRow.rowID],
+      );
+
+      this.db.run(
+        `DELETE FROM signatureAfter
+       WHERE ses IN (
+         SELECT rowID FROM sessions
+         WHERE coValue = ?
+           AND sessionID NOT LIKE '%$'
+       )`,
+        [coValueRow.rowID],
+      );
+
+      this.db.run(
+        `DELETE FROM sessions
+       WHERE coValue = ?
+         AND sessionID NOT LIKE '%$'`,
+        [coValueRow.rowID],
+      );
+
+      // Mark the delete as done
+      this.db.run(
+        `INSERT INTO deletedCoValues (coValueID, status) VALUES (?, ?)
+       ON CONFLICT(coValueID) DO UPDATE SET status=?`,
+        [
+          coValueId,
+          DeletedCoValueDeletionStatus.Done,
+          DeletedCoValueDeletionStatus.Done,
+        ],
+      );
+    });
+  }
+
+  getAllCoValuesWaitingForDelete(): RawCoID[] {
+    return this.db
+      .query<DeletedCoValueQueueRow>(
+        `SELECT coValueID as id
+         FROM deletedCoValues
+         WHERE status = ?`,
+        [DeletedCoValueDeletionStatus.Pending],
+      )
+      .map((r) => r.id);
+  }
+
   addSessionUpdate({ sessionUpdate }: { sessionUpdate: SessionRow }): number {
     const result = this.db.get<{ rowID: number }>(
       `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)

package/src/storage/sqlite/sqliteMigrations.ts

@@ -40,6 +40,13 @@ export const migrations: Record<number, string[]> = {
     );`,
     "CREATE INDEX IF NOT EXISTS idx_unsynced_covalues_co_value_id ON unsynced_covalues(co_value_id);",
   ],
+  5: [
+    `CREATE TABLE IF NOT EXISTS deletedCoValues (
+      coValueID TEXT PRIMARY KEY,
+      status INTEGER NOT NULL DEFAULT 0
+    ) WITHOUT ROWID;`,
+    "CREATE INDEX IF NOT EXISTS deletedCoValuesByStatus ON deletedCoValues (status);",
+  ],
 };
 
 type Migration = {

package/src/storage/sqliteAsync/client.ts

@@ -15,6 +15,7 @@ import type {
   StoredSessionRow,
   TransactionRow,
 } from "../types.js";
+import { DeletedCoValueDeletionStatus } from "../types.js";
 import type { SQLiteDatabaseDriverAsync } from "./types.js";
 import type { PeerID } from "../../sync.js";
 
@@ -29,6 +30,10 @@ export type RawTransactionRow = {
   tx: string;
 };
 
+type DeletedCoValueQueueRow = {
+  id: RawCoID;
+};
+
 export function getErrorMessage(error: unknown) {
   return error instanceof Error ? error.message : "Unknown error";
 }
@@ -146,6 +151,76 @@ export class SQLiteClientAsync
     return result.rowID;
   }
 
+  async markCoValueAsDeleted(id: RawCoID) {
+    // Work queue entry. Table only stores the coValueID.
+    // Idempotent by design.
+    await this.db.run(
+      `INSERT INTO deletedCoValues (coValueID) VALUES (?) ON CONFLICT(coValueID) DO NOTHING`,
+      [id],
+    );
+  }
+
+  async eraseCoValueButKeepTombstone(coValueId: RawCoID) {
+    const coValueRow = await this.db.get<RawCoValueRow & { rowID: number }>(
+      "SELECT * FROM coValues WHERE id = ?",
+      [coValueId],
+    );
+
+    if (!coValueRow) {
+      logger.warn(`CoValue ${coValueId} not found, skipping deletion`);
+      return;
+    }
+
+    await this.transaction(async () => {
+      await this.db.run(
+        `DELETE FROM transactions
+       WHERE ses IN (
+         SELECT rowID FROM sessions
+         WHERE coValue = ?
+           AND sessionID NOT LIKE '%$'
+       )`,
+        [coValueRow.rowID],
+      );
+
+      await this.db.run(
+        `DELETE FROM signatureAfter
+       WHERE ses IN (
+         SELECT rowID FROM sessions
+         WHERE coValue = ?
+           AND sessionID NOT LIKE '%$'
+       )`,
+        [coValueRow.rowID],
+      );
+
+      await this.db.run(
+        `DELETE FROM sessions
+       WHERE coValue = ?
+         AND sessionID NOT LIKE '%$'`,
+        [coValueRow.rowID],
+      );
+
+      await this.db.run(
+        `INSERT INTO deletedCoValues (coValueID, status) VALUES (?, ?)
+       ON CONFLICT(coValueID) DO UPDATE SET status=?`,
+        [
+          coValueId,
+          DeletedCoValueDeletionStatus.Done,
+          DeletedCoValueDeletionStatus.Done,
+        ],
+      );
+    });
+  }
+
+  async getAllCoValuesWaitingForDelete(): Promise<RawCoID[]> {
+    const rows = await this.db.query<DeletedCoValueQueueRow>(
+      `SELECT coValueID as id
+       FROM deletedCoValues
+       WHERE status = ?`,
+      [DeletedCoValueDeletionStatus.Pending],
+    );
+    return rows.map((r) => r.id);
+  }
+
   async addSessionUpdate({
     sessionUpdate,
   }: {

package/src/storage/storageAsync.ts

@@ -17,6 +17,7 @@ import {
   setSessionCounter,
 } from "../knownState.js";
 import { StorageKnownState } from "./knownState.js";
+import { DeletedCoValuesEraserScheduler } from "./DeletedCoValuesEraserScheduler.js";
 import {
   collectNewTxs,
   getDependedOnCoValues,
@@ -30,11 +31,20 @@ import type {
   StoredCoValueRow,
   StoredSessionRow,
 } from "./types.js";
+import { isDeleteSessionID } from "../ids.js";
 
 export class StorageApiAsync implements StorageAPI {
   private readonly dbClient: DBClientInterfaceAsync;
 
-  private loadedCoValues = new Set<RawCoID>();
+  private deletedCoValuesEraserScheduler:
+    | DeletedCoValuesEraserScheduler
+    | undefined;
+  private eraserController: AbortController | undefined;
+  /**
+   * Keeps track of CoValues that are in memory, to avoid reloading them from storage
+   * when it isn't necessary
+   */
+  private inMemoryCoValues = new Set<RawCoID>();
 
   // Track pending loads to deduplicate concurrent requests
   private pendingKnownStateLoads = new Map<
@@ -110,6 +120,7 @@ export class StorageApiAsync implements StorageAPI {
     callback: (data: NewContentMessage) => void,
     done: (found: boolean) => void,
   ) {
+    this.interruptEraser("load");
     const coValueRow = await this.dbClient.getCoValue(id);
 
     if (!coValueRow) {
@@ -153,7 +164,7 @@ export class StorageApiAsync implements StorageAPI {
       );
     }
 
-    this.loadedCoValues.add(coValueRow.id);
+    this.inMemoryCoValues.add(coValueRow.id);
 
     let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
 
@@ -224,7 +235,7 @@ export class StorageApiAsync implements StorageAPI {
     done?.(true);
   }
 
-  async pushContentWithDependencies(
+  private async pushContentWithDependencies(
     coValueRow: StoredCoValueRow,
     contentMessage: NewContentMessage,
     pushCallback: (data: NewContentMessage) => void,
@@ -237,7 +248,7 @@ export class StorageApiAsync implements StorageAPI {
     const promises = [];
 
     for (const dependedOnCoValue of dependedOnCoValuesList) {
-      if (this.loadedCoValues.has(dependedOnCoValue)) {
+      if (this.inMemoryCoValues.has(dependedOnCoValue)) {
         continue;
       }
 
@@ -263,10 +274,35 @@ export class StorageApiAsync implements StorageAPI {
     this.storeQueue.push(msg, correctionCallback);
 
     this.storeQueue.processQueue(async (data, correctionCallback) => {
+      this.interruptEraser("store");
       return this.storeSingle(data, correctionCallback);
     });
   }
 
+  private interruptEraser(reason: string) {
+    // Cooperative cancellation: a DB transaction already in progress will complete,
+    // but the eraser loop will stop starting further work at its next abort check.
+    if (this.eraserController) {
+      this.eraserController.abort(reason);
+      this.eraserController = undefined;
+    }
+  }
+
+  async eraseAllDeletedCoValues() {
+    const ids = await this.dbClient.getAllCoValuesWaitingForDelete();
+
+    this.eraserController = new AbortController();
+    const signal = this.eraserController.signal;
+
+    for (const id of ids) {
+      if (signal.aborted) {
+        return;
+      }
+
+      await this.dbClient.eraseCoValueButKeepTombstone(id);
+    }
+  }
+
   /**
    * This function is called when the storage lacks the information required to store the incoming content.
    *
@@ -309,6 +345,7 @@ export class StorageApiAsync implements StorageAPI {
     msg: NewContentMessage,
     correctionCallback: CorrectionCallback,
   ): Promise<boolean> {
+    this.interruptEraser("store");
     if (this.storeQueue.closed) {
       return false;
     }
@@ -338,6 +375,10 @@ export class StorageApiAsync implements StorageAPI {
           sessionID,
         );
 
+        if (this.deletedValues.has(id) && isDeleteSessionID(sessionID)) {
+          await tx.markCoValueAsDeleted(id);
+        }
+
         if (sessionRow) {
           setSessionCounter(
             knownState.sessions,
@@ -364,6 +405,8 @@ export class StorageApiAsync implements StorageAPI {
       });
     }
 
+    this.inMemoryCoValues.add(id);
+
     this.knownStates.handleUpdate(id, knownState);
 
     if (invalidAssumptions) {
@@ -439,6 +482,30 @@ export class StorageApiAsync implements StorageAPI {
     return newLastIdx;
   }
 
+  deletedValues = new Set<RawCoID>();
+
+  markDeleteAsValid(id: RawCoID) {
+    this.deletedValues.add(id);
+
+    if (this.deletedCoValuesEraserScheduler) {
+      this.deletedCoValuesEraserScheduler.onEnqueueDeletedCoValue();
+    }
+  }
+
+  enableDeletedCoValuesErasure() {
+    if (this.deletedCoValuesEraserScheduler) return;
+
+    this.deletedCoValuesEraserScheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        // Async storage: no max-time budgeting; drain to completion when scheduled.
+        await this.eraseAllDeletedCoValues();
+        const remaining = await this.dbClient.getAllCoValuesWaitingForDelete();
+        return { hasMore: remaining.length > 0 };
+      },
+    });
+    this.deletedCoValuesEraserScheduler.scheduleStartupDrain();
+  }
+
   waitForSync(id: string, coValue: CoValueCore) {
     return this.knownStates.waitForSync(id, coValue);
   }
@@ -460,7 +527,13 @@ export class StorageApiAsync implements StorageAPI {
     this.dbClient.stopTrackingSyncState(id);
   }
 
+  onCoValueUnmounted(id: RawCoID): void {
+    this.inMemoryCoValues.delete(id);
+  }
+
   close() {
+    this.deletedCoValuesEraserScheduler?.dispose();
+    this.inMemoryCoValues.clear();
     return this.storeQueue.close();
   }
 }

package/src/storage/storageSync.ts

@@ -16,6 +16,7 @@ import {
   emptyKnownState,
   setSessionCounter,
 } from "../knownState.js";
+import { isDeleteSessionID } from "../ids.js";
 import {
   collectNewTxs,
   getDependedOnCoValues,
@@ -29,15 +30,26 @@ import type {
   StoredCoValueRow,
   StoredSessionRow,
 } from "./types.js";
+import { DeletedCoValuesEraserScheduler } from "./DeletedCoValuesEraserScheduler.js";
 import {
   ContentCallback,
   StorageStreamingQueue,
 } from "../queue/StorageStreamingQueue.js";
 import { getPriorityFromHeader } from "../priority.js";
 
+const MAX_DELETE_SCHEDULE_DURATION_MS = 100;
+
 export class StorageApiSync implements StorageAPI {
   private readonly dbClient: DBClientInterfaceSync;
-  private loadedCoValues = new Set<RawCoID>();
+
+  private deletedCoValuesEraserScheduler:
+    | DeletedCoValuesEraserScheduler
+    | undefined;
+  /**
+   * Keeps track of CoValues that are in memory, to avoid reloading them from storage
+   * when it isn't necessary
+   */
+  private inMemoryCoValues = new Set<RawCoID>();
 
   /**
    * Queue for streaming content that will be pulled by SyncManager.
@@ -138,7 +150,7 @@ export class StorageApiSync implements StorageAPI {
       );
     }
 
-    this.loadedCoValues.add(coValueRow.id);
+    this.inMemoryCoValues.add(coValueRow.id);
 
     const priority = getPriorityFromHeader(coValueRow.header);
     const contentMessage = createContentMessage(
@@ -244,7 +256,7 @@ export class StorageApiSync implements StorageAPI {
     });
   }
 
-  async pushContentWithDependencies(
+  private async pushContentWithDependencies(
     coValueRow: StoredCoValueRow,
     contentMessage: NewContentMessage,
     pushCallback: (data: NewContentMessage) => void,
@@ -255,7 +267,7 @@ export class StorageApiSync implements StorageAPI {
     );
 
     for (const dependedOnCoValue of dependedOnCoValuesList) {
-      if (this.loadedCoValues.has(dependedOnCoValue)) {
+      if (this.inMemoryCoValues.has(dependedOnCoValue)) {
         continue;
       }
 
@@ -325,6 +337,10 @@ export class StorageApiSync implements StorageAPI {
 
     for (const sessionID of Object.keys(msg.new) as SessionID[]) {
       this.dbClient.transaction((tx) => {
+        if (this.deletedValues.has(id) && isDeleteSessionID(sessionID)) {
+          tx.markCoValueAsDeleted(id);
+        }
+
         const sessionRow = tx.getSingleCoValueSession(
           storedCoValueRowID,
           sessionID,
@@ -353,6 +369,8 @@ export class StorageApiSync implements StorageAPI {
       });
     }
 
+    this.inMemoryCoValues.add(id);
+
     this.knownStates.handleUpdate(id, knownState);
 
     if (invalidAssumptions) {
@@ -427,6 +445,51 @@ export class StorageApiSync implements StorageAPI {
     return newLastIdx;
   }
 
+  deletedValues = new Set<RawCoID>();
+
+  markDeleteAsValid(id: RawCoID) {
+    this.deletedValues.add(id);
+
+    if (this.deletedCoValuesEraserScheduler) {
+      this.deletedCoValuesEraserScheduler.onEnqueueDeletedCoValue();
+    }
+  }
+
+  async eraseAllDeletedCoValues(): Promise<void> {
+    const ids = this.dbClient.getAllCoValuesWaitingForDelete();
+
+    for (const id of ids) {
+      this.dbClient.eraseCoValueButKeepTombstone(id);
+    }
+  }
+
+  enableDeletedCoValuesErasure() {
+    if (this.deletedCoValuesEraserScheduler) return;
+    this.deletedCoValuesEraserScheduler = new DeletedCoValuesEraserScheduler({
+      run: async () =>
+        this.eraseDeletedCoValuesOnceBudgeted(MAX_DELETE_SCHEDULE_DURATION_MS),
+    });
+    this.deletedCoValuesEraserScheduler.scheduleStartupDrain();
+  }
+
+  private eraseDeletedCoValuesOnceBudgeted(budgetMs?: number) {
+    const startedAt = Date.now();
+    const ids = this.dbClient.getAllCoValuesWaitingForDelete();
+
+    for (const id of ids) {
+      // Strict time budget for sync storage to avoid blocking.
+      if (budgetMs && Date.now() - startedAt >= budgetMs) {
+        break;
+      }
+
+      this.dbClient.eraseCoValueButKeepTombstone(id);
+    }
+
+    return {
+      hasMore: this.dbClient.getAllCoValuesWaitingForDelete().length > 0,
+    };
+  }
+
   waitForSync(id: string, coValue: CoValueCore) {
     return this.knownStates.waitForSync(id, coValue);
   }
@@ -450,7 +513,13 @@ export class StorageApiSync implements StorageAPI {
     this.dbClient.stopTrackingSyncState(id);
   }
 
+  onCoValueUnmounted(id: RawCoID): void {
+    this.inMemoryCoValues.delete(id);
+  }
+
   close() {
+    this.deletedCoValuesEraserScheduler?.dispose();
+    this.inMemoryCoValues.clear();
     return undefined;
   }
 }

package/src/storage/types.ts

@@ -13,12 +13,43 @@ export type CorrectionCallback = (
   correction: CoValueKnownState,
 ) => NewContentMessage[] | undefined;
 
+/**
+ * Deletion work queue status for `deletedCoValues` (SQLite).
+ *
+ * Stored as an INTEGER in SQLite:
+ * - 0 = pending
+ * - 1 = done
+ */
+export enum DeletedCoValueDeletionStatus {
+  Pending = 0,
+  Done = 1,
+}
+
 /**
  * The StorageAPI is the interface that the StorageSync and StorageAsync classes implement.
  *
  * It uses callbacks instead of promises to have no overhead when using the StorageSync and less overhead when using the StorageAsync.
  */
 export interface StorageAPI {
+  /**
+   * Flags that the coValue delete is valid.
+   *
+   * When the delete tx is stored, the storage will mark the coValue as deleted.
+   */
+  markDeleteAsValid(id: RawCoID): void;
+
+  /**
+   * Enable the background erasure scheduler that drains the `deletedCoValues` work queue.
+   * This is intentionally opt-in and should be activated by `LocalNode`.
+   */
+  enableDeletedCoValuesErasure(): void;
+
+  /**
+   * Batch physical deletion for coValues queued in `deletedCoValues` with status `Pending`.
+   * Must preserve tombstones (header + delete session(s) + their tx/signatures).
+   */
+  eraseAllDeletedCoValues(): Promise<void>;
+
   load(
     id: string,
     // This callback is fired when data is found, might be called multiple times if the content requires streaming (e.g when loading files)
@@ -67,6 +98,12 @@ export interface StorageAPI {
     callback: (knownState: CoValueKnownState | undefined) => void,
   ): void;
 
+  /**
+   * Called when a CoValue is unmounted from memory.
+   * Used to clean up the metadata associated with that CoValue.
+   */
+  onCoValueUnmounted(id: RawCoID): void;
+
   close(): Promise<unknown> | undefined;
 }
 
@@ -105,6 +142,13 @@ export interface DBTransactionInterfaceAsync {
     sessionID: SessionID,
   ): Promise<StoredSessionRow | undefined>;
 
+  /**
+   * Persist a "deleted coValue" marker in storage (work queue entry).
+   * This is an enqueue signal: implementations should set status to `Pending`.
+   * This is expected to be idempotent (safe to call repeatedly).
+   */
+  markCoValueAsDeleted(id: RawCoID): Promise<unknown>;
+
   addSessionUpdate({
     sessionUpdate,
     sessionRow,
@@ -140,6 +184,11 @@ export interface DBClientInterfaceAsync {
     header?: CoValueHeader,
   ): Promise<number | undefined>;
 
+  /**
+   * Enumerate all coValue IDs currently pending in the "deleted coValues" work queue.
+   */
+  getAllCoValuesWaitingForDelete(): Promise<RawCoID[]>;
+
   getCoValueSessions(coValueRowId: number): Promise<StoredSessionRow[]>;
 
   getNewTransactionInSession(
@@ -165,6 +214,13 @@ export interface DBClientInterfaceAsync {
 
   stopTrackingSyncState(id: RawCoID): Promise<void>;
 
+  /**
+   * Physical deletion primitive: erase all persisted history for a deleted coValue,
+   * while preserving the tombstone (header + delete session(s)).
+   * Must run inside a single storage transaction.
+   */
+  eraseCoValueButKeepTombstone(coValueID: RawCoID): Promise<unknown>;
+
   /**
    * Get the knownState for a CoValue without loading transactions.
    * Returns undefined if the CoValue doesn't exist.
@@ -180,6 +236,13 @@ export interface DBTransactionInterfaceSync {
     sessionID: SessionID,
   ): StoredSessionRow | undefined;
 
+  /**
+   * Persist a "deleted coValue" marker in storage (work queue entry).
+   * This is an enqueue signal: implementations should set status to `"pending"`.
+   * This is expected to be idempotent (safe to call repeatedly).
+   */
+  markCoValueAsDeleted(id: RawCoID): unknown;
+
   addSessionUpdate({
     sessionUpdate,
     sessionRow,
@@ -210,6 +273,11 @@ export interface DBClientInterfaceSync {
 
   upsertCoValue(id: string, header?: CoValueHeader): number | undefined;
 
+  /**
+   * Enumerate all coValue IDs currently pending in the "deleted coValues" work queue.
+   */
+  getAllCoValuesWaitingForDelete(): RawCoID[];
+
   getCoValueSessions(coValueRowId: number): StoredSessionRow[];
 
   getNewTransactionInSession(
@@ -233,6 +301,13 @@ export interface DBClientInterfaceSync {
 
   stopTrackingSyncState(id: RawCoID): void;
 
+  /**
+   * Physical deletion primitive: erase all persisted history for a deleted coValue,
+   * while preserving the tombstone (header + delete session(s)).
+   * Must run inside a single storage transaction.
+   */
+  eraseCoValueButKeepTombstone(coValueID: RawCoID): unknown;
+
   /**
    * Get the knownState for a CoValue without loading transactions.
    * Returns undefined if the CoValue doesn't exist.