cojson 0.16.5 → 0.16.6

package/src/coValues/coPlainText.ts

@@ -1,5 +1,6 @@
 import { splitGraphemes } from "unicode-segmenter/grapheme";
 import { AvailableCoValueCore } from "../coValueCore/coValueCore.js";
+import { TRANSACTION_CONFIG } from "../config.js";
 import { JsonObject } from "../jsonValue.js";
 import { DeletionOpPayload, OpID, RawCoList } from "./coList.js";
 
@@ -110,16 +111,34 @@ export class RawCoPlainText<
     text: string,
     privacy: "private" | "trusting" = "private",
   ) {
-    const graphemes = [...splitGraphemes(text)];
+    const graphemes = Array.from(splitGraphemes(text));
 
     if (idx === 0) {
-      // For insertions at start, prepend each character in reverse
-      for (const grapheme of graphemes.reverse()) {
-        this.prepend(grapheme, 0, privacy);
+      // For insertions at start, prepend the first char and append the rest
+      const firstChar = graphemes[0];
+
+      if (firstChar) {
+        this.prepend(firstChar, 0, privacy);
+      }
+
+      if (graphemes.length > 1) {
+        this.appendChars(graphemes.slice(1), 0, privacy);
       }
     } else {
       // For other insertions, append after the previous character
-      this.appendItems(graphemes, idx - 1, privacy);
+      this.appendChars(graphemes, idx - 1, privacy);
+    }
+  }
+
+  appendChars(
+    text: string[],
+    position: number,
+    privacy: "private" | "trusting" = "private",
+  ) {
+    const chunks = splitIntoChunks(text);
+    for (const chunk of chunks) {
+      this.appendItems(chunk, position, privacy);
+      position += chunk.length;
     }
   }
 
@@ -136,11 +155,12 @@ export class RawCoPlainText<
     text: string,
     privacy: "private" | "trusting" = "private",
   ) {
-    const graphemes = [...splitGraphemes(text)];
+    const graphemes = Array.from(splitGraphemes(text));
+
     if (idx >= this.entries().length) {
-      this.appendItems(graphemes, idx - 1, privacy);
+      this.appendChars(graphemes, idx - 1, privacy);
     } else {
-      this.appendItems(graphemes, idx, privacy);
+      this.appendChars(graphemes, idx, privacy);
     }
   }
 
@@ -178,3 +198,15 @@ export class RawCoPlainText<
     return graphemes.join("");
   }
 }
+
+function splitIntoChunks(text: string[]) {
+  const chunks: string[][] = [];
+  for (
+    let i = 0;
+    i < text.length;
+    i += TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE
+  ) {
+    chunks.push(text.slice(i, i + TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE));
+  }
+  return chunks;
+}

package/src/config.ts

@@ -5,7 +5,13 @@
     This also means that we want to keep signatures roughly after each MAX_RECOMMENDED_TX size chunk,
     to be able to verify partially loaded CoValues or CoValues that are still being created (like a video live stream).
 **/
-export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
+export const TRANSACTION_CONFIG = {
+  MAX_RECOMMENDED_TX_SIZE: 100 * 1024,
+};
+
+export function setMaxRecommendedTxSize(size: number) {
+  TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE = size;
+}
 
 export const CO_VALUE_LOADING_CONFIG = {
   MAX_RETRIES: 1,
@@ -32,3 +38,16 @@ export const SYNC_SCHEDULER_CONFIG = {
 export function setIncomingMessagesTimeBudget(budget: number) {
   SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET = budget;
 }
+
+export const GARBAGE_COLLECTOR_CONFIG = {
+  MAX_AGE: 1000 * 60 * 10, // 10 minutes
+  INTERVAL: 1000 * 60 * 5, // 5 minutes
+};
+
+export function setGarbageCollectorMaxAge(maxAge: number) {
+  GARBAGE_COLLECTOR_CONFIG.MAX_AGE = maxAge;
+}
+
+export function setGarbageCollectorInterval(interval: number) {
+  GARBAGE_COLLECTOR_CONFIG.INTERVAL = interval;
+}

package/src/exports.ts

@@ -61,20 +61,25 @@ import { disablePermissionErrors } from "./permissions.js";
 import type { Peer, SyncMessage } from "./sync.js";
 import { DisconnectedError, SyncManager, emptyKnownState } from "./sync.js";
 
-type Value = JsonValue | AnyRawCoValue;
-
-export { PriorityBasedMessageQueue } from "./queue/PriorityBasedMessageQueue.js";
+import {
+  getContentMessageSize,
+  getTransactionSize,
+} from "./coValueContentMessage.js";
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
 import {
   CO_VALUE_LOADING_CONFIG,
-  MAX_RECOMMENDED_TX_SIZE,
+  TRANSACTION_CONFIG,
   setCoValueLoadingRetryDelay,
   setIncomingMessagesTimeBudget,
+  setMaxRecommendedTxSize,
 } from "./config.js";
 import { LogLevel, logger } from "./logger.js";
 import { CO_VALUE_PRIORITY, getPriorityFromHeader } from "./priority.js";
 import { getDependedOnCoValues } from "./storage/syncUtils.js";
 
+type Value = JsonValue | AnyRawCoValue;
+
+export { PriorityBasedMessageQueue } from "./queue/PriorityBasedMessageQueue.js";
 /** @hidden */
 export const cojsonInternals = {
   connectedPeers,
@@ -106,6 +111,10 @@ export const cojsonInternals = {
   ConnectedPeerChannel,
   textEncoder,
   textDecoder,
+  getTransactionSize,
+  getContentMessageSize,
+  TRANSACTION_CONFIG,
+  setMaxRecommendedTxSize,
 };
 
 export {
@@ -132,7 +141,6 @@ export {
   Media,
   CoValueCore,
   ControlledAgent,
-  MAX_RECOMMENDED_TX_SIZE,
   JsonObject,
   JsonValue,
   Peer,

package/src/localNode.ts

@@ -1,4 +1,5 @@
 import { Result, err, ok } from "neverthrow";
+import { GarbageCollector } from "./GarbageCollector.js";
 import type { CoID } from "./coValue.js";
 import type { RawCoValue } from "./coValue.js";
 import {
@@ -30,7 +31,7 @@ import {
   type RawGroup,
   secretSeedFromInviteSecret,
 } from "./coValues/group.js";
-import { CO_VALUE_LOADING_CONFIG } from "./config.js";
+import { CO_VALUE_LOADING_CONFIG, GARBAGE_COLLECTOR_CONFIG } from "./config.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
 import { logger } from "./logger.js";
@@ -63,6 +64,7 @@ export class LocalNode {
   /** @category 3. Low-level */
   syncManager = new SyncManager(this);
 
+  garbageCollector: GarbageCollector | undefined = undefined;
   crashed: Error | undefined = undefined;
 
   storage?: StorageAPI;
@@ -78,6 +80,14 @@ export class LocalNode {
     this.crypto = crypto;
   }
 
+  enableGarbageCollector() {
+    if (this.garbageCollector) {
+      return;
+    }
+
+    this.garbageCollector = new GarbageCollector(this.coValues);
+  }
+
   setStorage(storage: StorageAPI) {
     this.storage = storage;
   }
@@ -95,6 +105,8 @@ export class LocalNode {
       this.coValues.set(id, entry);
     }
 
+    this.garbageCollector?.trackCoValueAccess(entry);
+
     return entry;
   }
 
@@ -351,6 +363,7 @@ export class LocalNode {
       new VerifiedState(id, this.crypto, header, new Map()),
     );
 
+    this.garbageCollector?.trackCoValueAccess(coValue);
     this.syncManager.syncHeader(coValue.verified);
 
     return coValue;
@@ -745,6 +758,7 @@ export class LocalNode {
    */
   gracefulShutdown(): Promise<unknown> | undefined {
     this.syncManager.gracefulShutdown();
+    this.garbageCollector?.stop();
     return this.storage?.close();
   }
 }

package/src/queue/LocalTransactionsSyncQueue.ts

@@ -42,7 +42,7 @@ export class LocalTransactionsSyncQueue {
     const lastPendingSync = this.queue.tail?.value;
     const lastSignatureIdx = coValue.getLastSignatureCheckpoint(sessionID);
     const isSignatureCheckpoint =
-      lastSignatureIdx > -1 && lastSignatureIdx === txIdx - 1;
+      lastSignatureIdx > -1 && lastSignatureIdx === txIdx;
 
     if (lastPendingSync?.id === coValue.id && !isSignatureCheckpoint) {
       addTransactionToContentMessage(

package/src/queue/StoreQueue.ts

@@ -8,7 +8,38 @@ type StoreQueueEntry = {
   correctionCallback: CorrectionCallback;
 };
 
+class StoreQueueManager {
+  private backlog = new LinkedList<{
+    queue: StoreQueue;
+    callback: () => Promise<unknown>;
+  }>();
+
+  private processing = false;
+
+  async schedule(queue: StoreQueue, callback: () => Promise<unknown>) {
+    this.backlog.push({ queue, callback });
+
+    if (this.processing) {
+      return;
+    }
+
+    this.processing = true;
+
+    while (this.backlog.head) {
+      const entry = this.backlog.head;
+
+      await entry.value.callback();
+
+      this.backlog.shift();
+    }
+
+    this.processing = false;
+  }
+}
+
 export class StoreQueue {
+  static manager = new StoreQueueManager();
+
   private queue = new LinkedList<StoreQueueEntry>();
   closed = false;
 
@@ -27,7 +58,7 @@ export class StoreQueue {
   processing = false;
   lastCallback: Promise<unknown> | undefined;
 
-  async processQueue(
+  processQueue(
     callback: (
       data: NewContentMessage,
       correctionCallback: CorrectionCallback,
@@ -39,21 +70,23 @@ export class StoreQueue {
 
     this.processing = true;
 
-    let entry: StoreQueueEntry | undefined;
+    return StoreQueue.manager.schedule(this, async () => {
+      let entry: StoreQueueEntry | undefined;
 
-    while ((entry = this.pull())) {
-      const { data, correctionCallback } = entry;
+      while ((entry = this.pull())) {
+        const { data, correctionCallback } = entry;
 
-      try {
-        this.lastCallback = callback(data, correctionCallback);
-        await this.lastCallback;
-      } catch (err) {
-        logger.error("Error processing message in store queue", { err });
+        try {
+          this.lastCallback = callback(data, correctionCallback);
+          await this.lastCallback;
+        } catch (err) {
+          logger.error("Error processing message in store queue", { err });
+        }
       }
-    }
 
-    this.lastCallback = undefined;
-    this.processing = false;
+      this.lastCallback = undefined;
+      this.processing = false;
+    });
   }
 
   close() {

package/src/storage/sqlite/client.ts

@@ -112,24 +112,34 @@ export class SQLiteClient implements DBClientInterfaceSync {
     ) as SignatureAfterRow[];
   }
 
-  addCoValue(msg: NewContentMessage): number {
+  getCoValueRowID(id: RawCoID): number | undefined {
+    const row = this.db.get<{ rowID: number }>(
+      "SELECT rowID FROM coValues WHERE id = ?",
+      [id],
+    );
+    return row?.rowID;
+  }
+
+  upsertCoValue(id: RawCoID, header?: CoValueHeader): number | undefined {
+    if (!header) {
+      return this.getCoValueRowID(id);
+    }
+
     const result = this.db.get<{ rowID: number }>(
-      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
-      [msg.id, JSON.stringify(msg.header)],
+      `INSERT INTO coValues (id, header) VALUES (?, ?) 
+       ON CONFLICT(id) DO NOTHING
+       RETURNING rowID`,
+      [id, JSON.stringify(header)],
     );
 
     if (!result) {
-      throw new Error("Failed to add coValue");
+      return this.getCoValueRowID(id);
     }
 
     return result.rowID;
   }
 
-  addSessionUpdate({
-    sessionUpdate,
-  }: {
-    sessionUpdate: SessionRow;
-  }): number {
+  addSessionUpdate({ sessionUpdate }: { sessionUpdate: SessionRow }): number {
     const result = this.db.get<{ rowID: number }>(
       `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
                             ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
@@ -166,7 +176,11 @@ export class SQLiteClient implements DBClientInterfaceSync {
     sessionRowID,
     idx,
     signature,
-  }: { sessionRowID: number; idx: number; signature: Signature }) {
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }) {
     this.db.run(
       "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
       [sessionRowID, idx, signature],

package/src/storage/sqliteAsync/client.ts

@@ -112,14 +112,31 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     );
   }
 
-  async addCoValue(msg: NewContentMessage): Promise<number> {
+  async getCoValueRowID(id: RawCoID): Promise<number | undefined> {
+    const row = await this.db.get<{ rowID: number }>(
+      "SELECT rowID FROM coValues WHERE id = ?",
+      [id],
+    );
+    return row?.rowID;
+  }
+
+  async upsertCoValue(
+    id: RawCoID,
+    header?: CoValueHeader,
+  ): Promise<number | undefined> {
+    if (!header) {
+      return this.getCoValueRowID(id);
+    }
+
     const result = await this.db.get<{ rowID: number }>(
-      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
-      [msg.id, JSON.stringify(msg.header)],
+      `INSERT INTO coValues (id, header) VALUES (?, ?) 
+       ON CONFLICT(id) DO NOTHING 
+       RETURNING rowID`,
+      [id, JSON.stringify(header)],
     );
 
     if (!result) {
-      throw new Error("Failed to add coValue");
+      return this.getCoValueRowID(id);
     }
 
     return result.rowID;
@@ -166,7 +183,11 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     sessionRowID,
     idx,
     signature,
-  }: { sessionRowID: number; idx: number; signature: Signature }) {
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }) {
     this.db.run(
       "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
       [sessionRowID, idx, signature],

package/src/storage/storageAsync.ts

@@ -254,22 +254,18 @@ export class StorageApiAsync implements StorageAPI {
     }
 
     const id = msg.id;
-    const coValueRow = await this.dbClient.getCoValue(id);
-
-    // We have no info about coValue header
-    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
+    const storedCoValueRowID = await this.dbClient.upsertCoValue(
+      id,
+      msg.header,
+    );
 
-    if (invalidAssumptionOnHeaderPresence) {
+    if (!storedCoValueRowID) {
       const knownState = emptyKnownState(id as RawCoID);
       this.knwonStates.setKnownState(id, knownState);
 
       return this.handleCorrection(knownState, correctionCallback);
     }
 
-    const storedCoValueRowID: number = coValueRow
-      ? coValueRow.rowID
-      : await this.dbClient.addCoValue(msg);
-
     const knownState = this.knwonStates.getKnownState(id);
     knownState.header = true;
 

package/src/storage/storageSync.ts

@@ -235,22 +235,15 @@ export class StorageApiSync implements StorageAPI {
     correctionCallback: CorrectionCallback,
   ): boolean {
     const id = msg.id;
-    const coValueRow = this.dbClient.getCoValue(id);
-
-    // We have no info about coValue header
-    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
+    const storedCoValueRowID = this.dbClient.upsertCoValue(id, msg.header);
 
-    if (invalidAssumptionOnHeaderPresence) {
+    if (!storedCoValueRowID) {
       const knownState = emptyKnownState(id as RawCoID);
       this.knwonStates.setKnownState(id, knownState);
 
       return this.handleCorrection(knownState, correctionCallback);
     }
 
-    const storedCoValueRowID: number = coValueRow
-      ? coValueRow.rowID
-      : this.dbClient.addCoValue(msg);
-
     const knownState = this.knwonStates.getKnownState(id);
     knownState.header = true;
 

package/src/storage/types.ts

@@ -65,6 +65,11 @@ export interface DBClientInterfaceAsync {
     coValueId: string,
   ): Promise<StoredCoValueRow | undefined> | undefined;
 
+  upsertCoValue(
+    id: string,
+    header?: CoValueHeader,
+  ): Promise<number | undefined>;
+
   getCoValueSessions(coValueRowId: number): Promise<StoredSessionRow[]>;
 
   getSingleCoValueSession(
@@ -83,8 +88,6 @@ export interface DBClientInterfaceAsync {
     firstNewTxIdx: number,
   ): Promise<SignatureAfterRow[]>;
 
-  addCoValue(msg: NewContentMessage): Promise<number>;
-
   addSessionUpdate({
     sessionUpdate,
     sessionRow,
@@ -115,6 +118,8 @@ export interface DBClientInterfaceAsync {
 export interface DBClientInterfaceSync {
   getCoValue(coValueId: string): StoredCoValueRow | undefined;
 
+  upsertCoValue(id: string, header?: CoValueHeader): number | undefined;
+
   getCoValueSessions(coValueRowId: number): StoredSessionRow[];
 
   getSingleCoValueSession(
@@ -133,8 +138,6 @@ export interface DBClientInterfaceSync {
     firstNewTxIdx: number,
   ): Pick<SignatureAfterRow, "idx" | "signature">[];
 
-  addCoValue(msg: NewContentMessage): number;
-
   addSessionUpdate({
     sessionUpdate,
     sessionRow,

package/src/sync.ts

@@ -461,6 +461,22 @@ export class SyncManager {
 
     if (!coValue.hasVerifiedContent()) {
       if (!msg.header) {
+        const storageKnownState = this.local.storage?.getKnownState(msg.id);
+
+        if (storageKnownState?.header) {
+          // If the CoValue has been garbage collected, we load it from the storage before handling the new content
+          coValue.loadFromStorage((found) => {
+            if (found) {
+              this.handleNewContent(msg, from);
+            } else {
+              logger.error("Known CoValue not found in storage", {
+                id: msg.id,
+              });
+            }
+          });
+          return;
+        }
+
         if (peer) {
           this.trySendToPeer(peer, {
             action: "known",
@@ -782,12 +798,12 @@ export class SyncManager {
 
     if (!storage) return;
 
+    const value = this.local.getCoValue(content.id);
+
     // Try to store the content as-is for performance
     // In case that some transactions are missing, a correction will be requested, but it's an edge case
     storage.store(content, (correction) => {
-      return this.local
-        .getCoValue(content.id)
-        .verified?.newContentSince(correction);
+      return value.verified?.newContentSince(correction);
     });
   }
 

package/src/tests/GarbageCollector.test.ts

@@ -0,0 +1,127 @@
+import { assert, beforeEach, describe, expect, test, vi } from "vitest";
+
+import { setGarbageCollectorMaxAge } from "../config";
+import { TEST_NODE_CONFIG, setupTestAccount, setupTestNode } from "./testUtils";
+
+// We want to simulate a real world communication that happens asynchronously
+TEST_NODE_CONFIG.withAsyncPeers = true;
+
+beforeEach(() => {
+  // We want to test what happens when the garbage collector kicks in and removes a coValue
+  // We set the max age to -1 to make it remove everything
+  setGarbageCollectorMaxAge(-1);
+});
+
+describe("garbage collector", () => {
+  test("coValues are garbage collected when maxAge is reached", async () => {
+    const client = setupTestNode();
+
+    client.addStorage({
+      ourName: "client",
+    });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    client.node.garbageCollector?.collect();
+
+    const coValue = client.node.getCoValue(map.id);
+
+    expect(coValue.isAvailable()).toBe(false);
+  });
+
+  test("coValues are not garbage collected if they have listeners", async () => {
+    const client = setupTestNode();
+
+    client.addStorage({
+      ourName: "client",
+    });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Add a listener to the map
+    const unsubscribe = map.subscribe(() => {
+      // This listener keeps the coValue alive
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    client.node.garbageCollector?.collect();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(true);
+
+    // Clean up the listener
+    unsubscribe();
+
+    // The coValue should be collected after the listener is removed
+    client.node.garbageCollector?.collect();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
+  });
+
+  test("coValues are not garbage collected if they are a group or account", async () => {
+    const client = await setupTestAccount();
+
+    client.addStorage({
+      ourName: "client",
+    });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    client.node.garbageCollector?.collect();
+
+    expect(client.node.getCoValue(group.id).isAvailable()).toBe(true);
+    expect(client.node.getCoValue(client.accountID).isAvailable()).toBe(true);
+  });
+
+  test("coValues are not garbage collected if the maxAge is not reached", async () => {
+    setGarbageCollectorMaxAge(1000);
+
+    const client = setupTestNode();
+
+    client.addStorage({
+      ourName: "client",
+    });
+    client.node.enableGarbageCollector();
+
+    const garbageCollector = client.node.garbageCollector;
+
+    assert(garbageCollector);
+
+    const getCurrentTime = vi.spyOn(garbageCollector, "getCurrentTime");
+
+    getCurrentTime.mockReturnValue(1);
+
+    const group = client.node.createGroup();
+    const map1 = group.createMap();
+    const map2 = group.createMap();
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    map1.set("hello", "world", "trusting");
+
+    getCurrentTime.mockReturnValue(2000);
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    garbageCollector.collect();
+
+    const coValue = client.node.getCoValue(map1.id);
+
+    expect(coValue.isAvailable()).toBe(true);
+
+    const coValue2 = client.node.getCoValue(map2.id);
+
+    expect(coValue2.isAvailable()).toBe(false);
+  });
+});