cojson 0.16.5 → 0.16.7

package/src/GarbageCollector.ts

@@ -0,0 +1,48 @@
+import { CoValueCore } from "./coValueCore/coValueCore.js";
+import { GARBAGE_COLLECTOR_CONFIG } from "./config.js";
+import { RawCoID } from "./ids.js";
+
+export class GarbageCollector {
+  private readonly interval: ReturnType<typeof setInterval>;
+
+  constructor(private readonly coValues: Map<RawCoID, CoValueCore>) {
+    this.interval = setInterval(() => {
+      this.collect();
+    }, GARBAGE_COLLECTOR_CONFIG.INTERVAL);
+  }
+
+  getCurrentTime() {
+    return performance.now();
+  }
+
+  trackCoValueAccess({ verified }: CoValueCore) {
+    if (verified) {
+      verified.lastAccessed = this.getCurrentTime();
+    }
+  }
+
+  collect() {
+    const currentTime = this.getCurrentTime();
+    for (const coValue of this.coValues.values()) {
+      const { verified } = coValue;
+
+      if (!verified?.lastAccessed) {
+        continue;
+      }
+
+      const timeSinceLastAccessed = currentTime - verified.lastAccessed;
+
+      if (timeSinceLastAccessed > GARBAGE_COLLECTOR_CONFIG.MAX_AGE) {
+        const unmounted = coValue.unmount();
+
+        if (unmounted) {
+          this.coValues.delete(coValue.id);
+        }
+      }
+    }
+  }
+
+  stop() {
+    clearInterval(this.interval);
+  }
+}

package/src/coValueContentMessage.ts

@@ -3,7 +3,7 @@ import {
   Transaction,
   VerifiedState,
 } from "./coValueCore/verifiedState.js";
-import { MAX_RECOMMENDED_TX_SIZE } from "./config.js";
+import { TRANSACTION_CONFIG } from "./config.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID } from "./ids.js";
 import { getPriorityFromHeader } from "./priority.js";
@@ -55,10 +55,12 @@ export function exceedsRecommendedSize(
   transactionSize?: number,
 ) {
   if (transactionSize === undefined) {
-    return baseSize > MAX_RECOMMENDED_TX_SIZE;
+    return baseSize > TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE;
   }
 
-  return baseSize + transactionSize > MAX_RECOMMENDED_TX_SIZE;
+  return (
+    baseSize + transactionSize > TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE
+  );
 }
 
 export function knownStateFromContent(content: NewContentMessage) {
@@ -71,3 +73,14 @@ export function knownStateFromContent(content: NewContentMessage) {
 
   return knownState;
 }
+
+export function getContentMessageSize(msg: NewContentMessage) {
+  return Object.values(msg.new).reduce((acc, sessionNewContent) => {
+    return (
+      acc +
+      sessionNewContent.newTransactions.reduce((acc, tx) => {
+        return acc + getTransactionSize(tx);
+      }, 0)
+    );
+  }, 0);
+}

package/src/coValueCore/coValueCore.ts

@@ -69,7 +69,9 @@ export class CoValueCore {
   }
   private readonly peers = new Map<
     PeerID,
-    | { type: "unknown" | "pending" | "available" | "unavailable" }
+    | {
+        type: "unknown" | "pending" | "available" | "unavailable";
+      }
     | {
         type: "errored";
         error: TryAddTransactionsError;
@@ -78,9 +80,8 @@ export class CoValueCore {
 
   // cached state and listeners
   private _cachedContent?: RawCoValue;
-  private readonly listeners: Set<
-    (core: CoValueCore, unsub: () => void) => void
-  > = new Set();
+  readonly listeners: Set<(core: CoValueCore, unsub: () => void) => void> =
+    new Set();
   private readonly _decryptionCache: {
     [key: Encrypted<JsonValue[], JsonValue>]: JsonValue[] | undefined;
   } = {};
@@ -201,6 +202,26 @@ export class CoValueCore {
     }
   }
 
+  unmount() {
+    // For simplicity, we don't unmount groups and accounts
+    if (this.verified?.header.ruleset.type === "group") {
+      return false;
+    }
+
+    if (this.listeners.size > 0) {
+      return false; // The coValue is still in use
+    }
+
+    this.counter.add(-1, { state: this.loadingState });
+
+    if (this.groupInvalidationSubscription) {
+      this.groupInvalidationSubscription();
+      this.groupInvalidationSubscription = undefined;
+    }
+
+    return true;
+  }
+
   markNotFoundInPeer(peerId: PeerID) {
     const previousState = this.loadingState;
     this.peers.set(peerId, { type: "unavailable" });
@@ -609,9 +630,7 @@ export class CoValueCore {
     return success;
   }
 
-  getCurrentContent(options?: {
-    ignorePrivateTransactions: true;
-  }): RawCoValue {
+  getCurrentContent(options?: { ignorePrivateTransactions: true }): RawCoValue {
     if (!this.verified) {
       throw new Error(
         "CoValueCore: getCurrentContent called on coValue without verified state",
@@ -851,9 +870,7 @@ export class CoValueCore {
     }
   }
 
-  waitForSync(options?: {
-    timeout?: number;
-  }) {
+  waitForSync(options?: { timeout?: number }) {
     return this.node.syncManager.waitForSync(this.id, options?.timeout);
   }
 

package/src/coValueCore/utils.ts

@@ -2,6 +2,7 @@ import { getGroupDependentKey } from "../ids.js";
 import { RawCoID, SessionID } from "../ids.js";
 import { Stringified, parseJSON } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { NewContentMessage } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isAccountID } from "../typeUtils/isAccountID.js";
 import { CoValueHeader, Transaction } from "./verifiedState.js";

package/src/coValueCore/verifiedState.ts

@@ -65,6 +65,7 @@ export class VerifiedState {
   private _cachedKnownState?: CoValueKnownState;
   private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   private streamingKnownState?: CoValueKnownState["sessions"];
+  public lastAccessed: number | undefined;
 
   constructor(
     id: RawCoID,

package/src/coValues/coPlainText.ts

@@ -1,5 +1,6 @@
 import { splitGraphemes } from "unicode-segmenter/grapheme";
 import { AvailableCoValueCore } from "../coValueCore/coValueCore.js";
+import { TRANSACTION_CONFIG } from "../config.js";
 import { JsonObject } from "../jsonValue.js";
 import { DeletionOpPayload, OpID, RawCoList } from "./coList.js";
 
@@ -110,16 +111,34 @@ export class RawCoPlainText<
     text: string,
     privacy: "private" | "trusting" = "private",
   ) {
-    const graphemes = [...splitGraphemes(text)];
+    const graphemes = Array.from(splitGraphemes(text));
 
     if (idx === 0) {
-      // For insertions at start, prepend each character in reverse
-      for (const grapheme of graphemes.reverse()) {
-        this.prepend(grapheme, 0, privacy);
+      // For insertions at start, prepend the first char and append the rest
+      const firstChar = graphemes[0];
+
+      if (firstChar) {
+        this.prepend(firstChar, 0, privacy);
+      }
+
+      if (graphemes.length > 1) {
+        this.appendChars(graphemes.slice(1), 0, privacy);
       }
     } else {
       // For other insertions, append after the previous character
-      this.appendItems(graphemes, idx - 1, privacy);
+      this.appendChars(graphemes, idx - 1, privacy);
+    }
+  }
+
+  appendChars(
+    text: string[],
+    position: number,
+    privacy: "private" | "trusting" = "private",
+  ) {
+    const chunks = splitIntoChunks(text);
+    for (const chunk of chunks) {
+      this.appendItems(chunk, position, privacy);
+      position += chunk.length;
     }
   }
 
@@ -136,11 +155,12 @@ export class RawCoPlainText<
     text: string,
     privacy: "private" | "trusting" = "private",
   ) {
-    const graphemes = [...splitGraphemes(text)];
+    const graphemes = Array.from(splitGraphemes(text));
+
     if (idx >= this.entries().length) {
-      this.appendItems(graphemes, idx - 1, privacy);
+      this.appendChars(graphemes, idx - 1, privacy);
     } else {
-      this.appendItems(graphemes, idx, privacy);
+      this.appendChars(graphemes, idx, privacy);
     }
   }
 
@@ -178,3 +198,15 @@ export class RawCoPlainText<
     return graphemes.join("");
   }
 }
+
+function splitIntoChunks(text: string[]) {
+  const chunks: string[][] = [];
+  for (
+    let i = 0;
+    i < text.length;
+    i += TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE
+  ) {
+    chunks.push(text.slice(i, i + TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE));
+  }
+  return chunks;
+}

package/src/config.ts

@@ -5,7 +5,13 @@
     This also means that we want to keep signatures roughly after each MAX_RECOMMENDED_TX size chunk,
     to be able to verify partially loaded CoValues or CoValues that are still being created (like a video live stream).
 **/
-export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
+export const TRANSACTION_CONFIG = {
+  MAX_RECOMMENDED_TX_SIZE: 100 * 1024,
+};
+
+export function setMaxRecommendedTxSize(size: number) {
+  TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE = size;
+}
 
 export const CO_VALUE_LOADING_CONFIG = {
   MAX_RETRIES: 1,
@@ -32,3 +38,16 @@ export const SYNC_SCHEDULER_CONFIG = {
 export function setIncomingMessagesTimeBudget(budget: number) {
   SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET = budget;
 }
+
+export const GARBAGE_COLLECTOR_CONFIG = {
+  MAX_AGE: 1000 * 60 * 10, // 10 minutes
+  INTERVAL: 1000 * 60 * 5, // 5 minutes
+};
+
+export function setGarbageCollectorMaxAge(maxAge: number) {
+  GARBAGE_COLLECTOR_CONFIG.MAX_AGE = maxAge;
+}
+
+export function setGarbageCollectorInterval(interval: number) {
+  GARBAGE_COLLECTOR_CONFIG.INTERVAL = interval;
+}

package/src/exports.ts

@@ -61,20 +61,25 @@ import { disablePermissionErrors } from "./permissions.js";
 import type { Peer, SyncMessage } from "./sync.js";
 import { DisconnectedError, SyncManager, emptyKnownState } from "./sync.js";
 
-type Value = JsonValue | AnyRawCoValue;
-
-export { PriorityBasedMessageQueue } from "./queue/PriorityBasedMessageQueue.js";
+import {
+  getContentMessageSize,
+  getTransactionSize,
+} from "./coValueContentMessage.js";
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
 import {
   CO_VALUE_LOADING_CONFIG,
-  MAX_RECOMMENDED_TX_SIZE,
+  TRANSACTION_CONFIG,
   setCoValueLoadingRetryDelay,
   setIncomingMessagesTimeBudget,
+  setMaxRecommendedTxSize,
 } from "./config.js";
 import { LogLevel, logger } from "./logger.js";
 import { CO_VALUE_PRIORITY, getPriorityFromHeader } from "./priority.js";
 import { getDependedOnCoValues } from "./storage/syncUtils.js";
 
+type Value = JsonValue | AnyRawCoValue;
+
+export { PriorityBasedMessageQueue } from "./queue/PriorityBasedMessageQueue.js";
 /** @hidden */
 export const cojsonInternals = {
   connectedPeers,
@@ -106,6 +111,10 @@ export const cojsonInternals = {
   ConnectedPeerChannel,
   textEncoder,
   textDecoder,
+  getTransactionSize,
+  getContentMessageSize,
+  TRANSACTION_CONFIG,
+  setMaxRecommendedTxSize,
 };
 
 export {
@@ -132,7 +141,6 @@ export {
   Media,
   CoValueCore,
   ControlledAgent,
-  MAX_RECOMMENDED_TX_SIZE,
   JsonObject,
   JsonValue,
   Peer,

package/src/localNode.ts

@@ -1,4 +1,5 @@
 import { Result, err, ok } from "neverthrow";
+import { GarbageCollector } from "./GarbageCollector.js";
 import type { CoID } from "./coValue.js";
 import type { RawCoValue } from "./coValue.js";
 import {
@@ -30,7 +31,7 @@ import {
   type RawGroup,
   secretSeedFromInviteSecret,
 } from "./coValues/group.js";
-import { CO_VALUE_LOADING_CONFIG } from "./config.js";
+import { CO_VALUE_LOADING_CONFIG, GARBAGE_COLLECTOR_CONFIG } from "./config.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
 import { logger } from "./logger.js";
@@ -63,6 +64,7 @@ export class LocalNode {
   /** @category 3. Low-level */
   syncManager = new SyncManager(this);
 
+  garbageCollector: GarbageCollector | undefined = undefined;
   crashed: Error | undefined = undefined;
 
   storage?: StorageAPI;
@@ -78,6 +80,14 @@ export class LocalNode {
     this.crypto = crypto;
   }
 
+  enableGarbageCollector() {
+    if (this.garbageCollector) {
+      return;
+    }
+
+    this.garbageCollector = new GarbageCollector(this.coValues);
+  }
+
   setStorage(storage: StorageAPI) {
     this.storage = storage;
   }
@@ -95,6 +105,8 @@ export class LocalNode {
       this.coValues.set(id, entry);
     }
 
+    this.garbageCollector?.trackCoValueAccess(entry);
+
     return entry;
   }
 
@@ -351,6 +363,7 @@ export class LocalNode {
       new VerifiedState(id, this.crypto, header, new Map()),
     );
 
+    this.garbageCollector?.trackCoValueAccess(coValue);
     this.syncManager.syncHeader(coValue.verified);
 
     return coValue;
@@ -745,6 +758,7 @@ export class LocalNode {
    */
   gracefulShutdown(): Promise<unknown> | undefined {
     this.syncManager.gracefulShutdown();
+    this.garbageCollector?.stop();
     return this.storage?.close();
   }
 }

package/src/permissions.ts

@@ -434,7 +434,9 @@ function determineValidTransactionsForGroup(
     const currentAccountId = coValue.node.getCurrentAccountOrAgentID();
 
     const isSelfRevoke =
-      currentAccountId === change.key && change.value === "revoked";
+      currentAccountId === change.key &&
+      transactor === currentAccountId &&
+      change.value === "revoked";
 
     if (!isFirstSelfAppointment && !isSelfRevoke) {
       if (memberState[transactor] === "admin") {

package/src/queue/LocalTransactionsSyncQueue.ts

@@ -42,7 +42,7 @@ export class LocalTransactionsSyncQueue {
     const lastPendingSync = this.queue.tail?.value;
     const lastSignatureIdx = coValue.getLastSignatureCheckpoint(sessionID);
     const isSignatureCheckpoint =
-      lastSignatureIdx > -1 && lastSignatureIdx === txIdx - 1;
+      lastSignatureIdx > -1 && lastSignatureIdx === txIdx;
 
     if (lastPendingSync?.id === coValue.id && !isSignatureCheckpoint) {
       addTransactionToContentMessage(

package/src/queue/StoreQueue.ts

@@ -8,7 +8,38 @@ type StoreQueueEntry = {
   correctionCallback: CorrectionCallback;
 };
 
+class StoreQueueManager {
+  private backlog = new LinkedList<{
+    queue: StoreQueue;
+    callback: () => Promise<unknown>;
+  }>();
+
+  private processing = false;
+
+  async schedule(queue: StoreQueue, callback: () => Promise<unknown>) {
+    this.backlog.push({ queue, callback });
+
+    if (this.processing) {
+      return;
+    }
+
+    this.processing = true;
+
+    while (this.backlog.head) {
+      const entry = this.backlog.head;
+
+      await entry.value.callback();
+
+      this.backlog.shift();
+    }
+
+    this.processing = false;
+  }
+}
+
 export class StoreQueue {
+  static manager = new StoreQueueManager();
+
   private queue = new LinkedList<StoreQueueEntry>();
   closed = false;
 
@@ -27,7 +58,7 @@ export class StoreQueue {
   processing = false;
   lastCallback: Promise<unknown> | undefined;
 
-  async processQueue(
+  processQueue(
     callback: (
       data: NewContentMessage,
       correctionCallback: CorrectionCallback,
@@ -39,21 +70,23 @@ export class StoreQueue {
 
     this.processing = true;
 
-    let entry: StoreQueueEntry | undefined;
+    return StoreQueue.manager.schedule(this, async () => {
+      let entry: StoreQueueEntry | undefined;
 
-    while ((entry = this.pull())) {
-      const { data, correctionCallback } = entry;
+      while ((entry = this.pull())) {
+        const { data, correctionCallback } = entry;
 
-      try {
-        this.lastCallback = callback(data, correctionCallback);
-        await this.lastCallback;
-      } catch (err) {
-        logger.error("Error processing message in store queue", { err });
+        try {
+          this.lastCallback = callback(data, correctionCallback);
+          await this.lastCallback;
+        } catch (err) {
+          logger.error("Error processing message in store queue", { err });
+        }
       }
-    }
 
-    this.lastCallback = undefined;
-    this.processing = false;
+      this.lastCallback = undefined;
+      this.processing = false;
+    });
   }
 
   close() {

package/src/storage/sqlite/client.ts

@@ -112,24 +112,34 @@ export class SQLiteClient implements DBClientInterfaceSync {
     ) as SignatureAfterRow[];
   }
 
-  addCoValue(msg: NewContentMessage): number {
+  getCoValueRowID(id: RawCoID): number | undefined {
+    const row = this.db.get<{ rowID: number }>(
+      "SELECT rowID FROM coValues WHERE id = ?",
+      [id],
+    );
+    return row?.rowID;
+  }
+
+  upsertCoValue(id: RawCoID, header?: CoValueHeader): number | undefined {
+    if (!header) {
+      return this.getCoValueRowID(id);
+    }
+
     const result = this.db.get<{ rowID: number }>(
-      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
-      [msg.id, JSON.stringify(msg.header)],
+      `INSERT INTO coValues (id, header) VALUES (?, ?) 
+       ON CONFLICT(id) DO NOTHING
+       RETURNING rowID`,
+      [id, JSON.stringify(header)],
     );
 
     if (!result) {
-      throw new Error("Failed to add coValue");
+      return this.getCoValueRowID(id);
     }
 
     return result.rowID;
   }
 
-  addSessionUpdate({
-    sessionUpdate,
-  }: {
-    sessionUpdate: SessionRow;
-  }): number {
+  addSessionUpdate({ sessionUpdate }: { sessionUpdate: SessionRow }): number {
     const result = this.db.get<{ rowID: number }>(
       `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
                             ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
@@ -166,7 +176,11 @@ export class SQLiteClient implements DBClientInterfaceSync {
     sessionRowID,
     idx,
     signature,
-  }: { sessionRowID: number; idx: number; signature: Signature }) {
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }) {
     this.db.run(
       "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
       [sessionRowID, idx, signature],

package/src/storage/sqliteAsync/client.ts

@@ -112,14 +112,31 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     );
   }
 
-  async addCoValue(msg: NewContentMessage): Promise<number> {
+  async getCoValueRowID(id: RawCoID): Promise<number | undefined> {
+    const row = await this.db.get<{ rowID: number }>(
+      "SELECT rowID FROM coValues WHERE id = ?",
+      [id],
+    );
+    return row?.rowID;
+  }
+
+  async upsertCoValue(
+    id: RawCoID,
+    header?: CoValueHeader,
+  ): Promise<number | undefined> {
+    if (!header) {
+      return this.getCoValueRowID(id);
+    }
+
     const result = await this.db.get<{ rowID: number }>(
-      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
-      [msg.id, JSON.stringify(msg.header)],
+      `INSERT INTO coValues (id, header) VALUES (?, ?) 
+       ON CONFLICT(id) DO NOTHING 
+       RETURNING rowID`,
+      [id, JSON.stringify(header)],
     );
 
     if (!result) {
-      throw new Error("Failed to add coValue");
+      return this.getCoValueRowID(id);
     }
 
     return result.rowID;
@@ -166,7 +183,11 @@ export class SQLiteClientAsync implements DBClientInterfaceAsync {
     sessionRowID,
     idx,
     signature,
-  }: { sessionRowID: number; idx: number; signature: Signature }) {
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }) {
     this.db.run(
       "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
       [sessionRowID, idx, signature],

package/src/storage/storageAsync.ts

@@ -254,22 +254,18 @@ export class StorageApiAsync implements StorageAPI {
     }
 
     const id = msg.id;
-    const coValueRow = await this.dbClient.getCoValue(id);
-
-    // We have no info about coValue header
-    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
+    const storedCoValueRowID = await this.dbClient.upsertCoValue(
+      id,
+      msg.header,
+    );
 
-    if (invalidAssumptionOnHeaderPresence) {
+    if (!storedCoValueRowID) {
       const knownState = emptyKnownState(id as RawCoID);
       this.knwonStates.setKnownState(id, knownState);
 
       return this.handleCorrection(knownState, correctionCallback);
     }
 
-    const storedCoValueRowID: number = coValueRow
-      ? coValueRow.rowID
-      : await this.dbClient.addCoValue(msg);
-
     const knownState = this.knwonStates.getKnownState(id);
     knownState.header = true;
 

package/src/storage/storageSync.ts

@@ -235,22 +235,15 @@ export class StorageApiSync implements StorageAPI {
     correctionCallback: CorrectionCallback,
   ): boolean {
     const id = msg.id;
-    const coValueRow = this.dbClient.getCoValue(id);
-
-    // We have no info about coValue header
-    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
+    const storedCoValueRowID = this.dbClient.upsertCoValue(id, msg.header);
 
-    if (invalidAssumptionOnHeaderPresence) {
+    if (!storedCoValueRowID) {
       const knownState = emptyKnownState(id as RawCoID);
       this.knwonStates.setKnownState(id, knownState);
 
       return this.handleCorrection(knownState, correctionCallback);
     }
 
-    const storedCoValueRowID: number = coValueRow
-      ? coValueRow.rowID
-      : this.dbClient.addCoValue(msg);
-
     const knownState = this.knwonStates.getKnownState(id);
     knownState.header = true;