cojson 0.18.31 → 0.18.33

package/src/coValueCore/verifiedState.ts

@@ -4,6 +4,7 @@ import {
   createContentMessage,
   exceedsRecommendedSize,
   getTransactionSize,
+  addTransactionToContentMessage,
 } from "../coValueContentMessage.js";
 import {
   CryptoProvider,
@@ -21,7 +22,7 @@ import { NewContentMessage } from "../sync.js";
 import { TryAddTransactionsError } from "./coValueCore.js";
 import { SessionLog, SessionMap } from "./SessionMap.js";
 import { ControlledAccountOrAgent } from "../coValues/account.js";
-import { cloneKnownState, CoValueKnownState } from "../knownState.js";
+import { CoValueKnownState, KnownStateSessions } from "../knownState.js";
 
 export type CoValueHeader = {
   type: AnyRawCoValue["type"];
@@ -55,7 +56,6 @@ export class VerifiedState {
   readonly crypto: CryptoProvider;
   readonly header: CoValueHeader;
   readonly sessions: SessionMap;
-  private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   public lastAccessed: number | undefined;
   public branchSourceId?: RawCoID;
   public branchName?: string;
@@ -98,10 +98,6 @@ export class VerifiedState {
       skipVerify,
     );
 
-    if (result.isOk()) {
-      this._cachedNewContentSinceEmpty = undefined;
-    }
-
     return result;
   }
 
@@ -120,8 +116,6 @@ export class VerifiedState {
       madeAt,
     );
 
-    this._cachedNewContentSinceEmpty = undefined;
-
     return result;
   }
 
@@ -144,8 +138,6 @@ export class VerifiedState {
       madeAt,
     );
 
-    this._cachedNewContentSinceEmpty = undefined;
-
     return result;
   }
 
@@ -160,119 +152,147 @@ export class VerifiedState {
     );
   }
 
+  setStreamingKnownState(streamingKnownState: KnownStateSessions) {
+    this.sessions.setStreamingKnownState(streamingKnownState);
+  }
+
   newContentSince(
     knownState: CoValueKnownState | undefined,
   ): NewContentMessage[] | undefined {
-    const isKnownStateEmpty = !knownState?.header && !knownState?.sessions;
-
-    if (isKnownStateEmpty && this._cachedNewContentSinceEmpty) {
-      return this._cachedNewContentSinceEmpty;
-    }
-
     let currentPiece: NewContentMessage = createContentMessage(
       this.id,
       this.header,
-      !knownState?.header,
+      false,
     );
-
-    const pieces = [currentPiece];
-
-    const sentState: CoValueKnownState["sessions"] = {};
-
+    const pieces: NewContentMessage[] = [currentPiece];
     let pieceSize = 0;
 
-    let sessionsTodoAgain: Set<SessionID> | undefined | "first" = "first";
-
-    while (sessionsTodoAgain === "first" || sessionsTodoAgain?.size || 0 > 0) {
-      if (sessionsTodoAgain === "first") {
-        sessionsTodoAgain = undefined;
+    const startNewPiece = () => {
+      currentPiece = createContentMessage(this.id, this.header, false);
+      pieces.push(currentPiece);
+      pieceSize = 0;
+    };
+
+    const moveSessionContentToNewPiece = (sessionID: SessionID) => {
+      const sessionContent = currentPiece.new[sessionID];
+
+      if (!sessionContent) {
+        throw new Error("Session content not found", {
+          cause: {
+            sessionID,
+            currentPiece,
+          },
+        });
       }
-      const sessionsTodo = sessionsTodoAgain ?? this.sessions.keys();
-
-      for (const sessionIDKey of sessionsTodo) {
-        const sessionID = sessionIDKey as SessionID;
-        const log = this.sessions.get(sessionID)!;
-        const knownStateForSessionID = knownState?.sessions[sessionID];
-        const sentStateForSessionID = sentState[sessionID];
-        const nextKnownSignatureIdx = getNextKnownSignatureIdx(
-          log,
-          knownStateForSessionID,
-          sentStateForSessionID,
-        );
-
-        const firstNewTxIdx =
-          sentStateForSessionID ?? knownStateForSessionID ?? 0;
-        const afterLastNewTxIdx =
-          nextKnownSignatureIdx === undefined
-            ? log.transactions.length
-            : nextKnownSignatureIdx + 1;
-
-        const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
-
-        if (nNewTx === 0) {
-          sessionsTodoAgain?.delete(sessionID);
-          continue;
-        }
 
-        if (afterLastNewTxIdx < log.transactions.length) {
-          if (!sessionsTodoAgain) {
-            sessionsTodoAgain = new Set();
+      delete currentPiece.new[sessionID];
+
+      const newPiece = createContentMessage(this.id, this.header, false);
+      newPiece.new[sessionID] = sessionContent;
+
+      // Insert the new piece before the current piece, to ensure that the order of the new transactions is preserved
+      pieces.splice(pieces.length - 1, 0, newPiece);
+    };
+
+    const sessionSent = knownState?.sessions;
+
+    for (const [sessionID, log] of this.sessions.sessions) {
+      const startFrom = sessionSent?.[sessionID] ?? 0;
+
+      let currentSessionSize = 0;
+
+      for (let txIdx = startFrom; txIdx < log.transactions.length; txIdx++) {
+        const isLastItem = txIdx === log.transactions.length - 1;
+        const tx = log.transactions[txIdx]!;
+
+        currentSessionSize += getTransactionSize(tx);
+
+        const signatureAfter = log.signatureAfter[txIdx];
+
+        if (signatureAfter) {
+          addTransactionToContentMessage(
+            currentPiece,
+            tx,
+            sessionID,
+            signatureAfter,
+            txIdx,
+          );
+          // When we meet a signatureAfter it means that the transaction log exceeds the recommended size
+          // so we move the session content to a dedicated piece, because it must be sent in a standalone piece
+          moveSessionContentToNewPiece(sessionID);
+          currentSessionSize = 0;
+        } else if (isLastItem) {
+          if (!log.lastSignature) {
+            throw new Error(
+              "All the SessionLogs sent must have a lastSignature",
+              {
+                cause: log,
+              },
+            );
           }
-          sessionsTodoAgain.add(sessionID);
-        }
 
-        const oldPieceSize = pieceSize;
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          pieceSize += getTransactionSize(tx);
-        }
-
-        if (exceedsRecommendedSize(pieceSize)) {
-          if (!currentPiece.expectContentUntil && pieces.length === 1) {
-            currentPiece.expectContentUntil =
-              this.knownStateWithStreaming().sessions;
+          addTransactionToContentMessage(
+            currentPiece,
+            tx,
+            sessionID,
+            log.lastSignature,
+            txIdx,
+          );
+
+          // If the current session size already exceeds the recommended size, we move the session content to a dedicated piece
+          if (exceedsRecommendedSize(currentSessionSize)) {
+            assertLastSignature(sessionID, currentPiece);
+            moveSessionContentToNewPiece(sessionID);
+          } else if (exceedsRecommendedSize(pieceSize, currentSessionSize)) {
+            assertLastSignature(sessionID, currentPiece);
+            startNewPiece();
+          } else {
+            pieceSize += currentSessionSize;
           }
-
-          currentPiece = createContentMessage(this.id, this.header, false);
-          pieces.push(currentPiece);
-          pieceSize = pieceSize - oldPieceSize;
+        } else {
+          // Unsafely add the transaction to the content message, without a signature because we don't have one for this session
+          // Checks and assertions are enforced in this function to avoid that a content message gets out without a signature
+          const signature = undefined as Signature | undefined;
+          addTransactionToContentMessage(
+            currentPiece,
+            tx,
+            sessionID,
+            signature!,
+            txIdx,
+          );
         }
+      }
 
-        let sessionEntry = currentPiece.new[sessionID];
-        if (!sessionEntry) {
-          sessionEntry = {
-            after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
-            newTransactions: [],
-            lastSignature: "WILL_BE_REPLACED" as Signature,
-          };
-          currentPiece.new[sessionID] = sessionEntry;
-        }
+      assertLastSignature(sessionID, currentPiece);
+    }
 
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          sessionEntry.newTransactions.push(tx);
-        }
+    const firstPiece = pieces[0];
 
-        sessionEntry.lastSignature =
-          nextKnownSignatureIdx === undefined
-            ? log.lastSignature!
-            : log.signatureAfter[nextKnownSignatureIdx]!;
+    if (!firstPiece) {
+      throw new Error("First piece not found", {
+        cause: pieces,
+      });
+    }
 
-        sentState[sessionID] =
-          (sentStateForSessionID ?? knownStateForSessionID ?? 0) + nNewTx;
-      }
+    const includeHeader = !knownState?.header;
+
+    if (includeHeader) {
+      firstPiece.header = this.header;
     }
 
     const piecesWithContent = pieces.filter(
-      (piece) => Object.keys(piece.new).length > 0 || piece.header,
+      (piece) => piece.header || Object.keys(piece.new).length > 0,
     );
 
-    if (piecesWithContent.length === 0) {
-      return undefined;
+    if (piecesWithContent.length > 1 || this.isStreaming()) {
+      // Flag that more content is coming
+      firstPiece.expectContentUntil = {
+        ...this.knownStateWithStreaming().sessions,
+      };
     }
 
-    if (isKnownStateEmpty) {
-      this._cachedNewContentSinceEmpty = piecesWithContent;
+    if (piecesWithContent.length === 0) {
+      return undefined;
     }
 
     return piecesWithContent;
@@ -307,15 +327,10 @@ export class VerifiedState {
   }
 }
 
-function getNextKnownSignatureIdx(
-  log: SessionLog,
-  knownStateForSessionID?: number,
-  sentStateForSessionID?: number,
-) {
-  return Object.keys(log.signatureAfter)
-    .map(Number)
-    .sort((a, b) => a - b)
-    .find(
-      (idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1),
-    );
+function assertLastSignature(sessionID: SessionID, content: NewContentMessage) {
+  if (content.new[sessionID] && !content.new[sessionID].lastSignature) {
+    throw new Error("The SessionContent sent must have a lastSignature", {
+      cause: content.new[sessionID],
+    });
+  }
 }

package/src/coValues/group.ts

@@ -123,6 +123,13 @@ function healMissingKeyForEveryone(group: RawGroup) {
 }
 
 function needsKeyRotation(group: RawGroup) {
+  const myRole = group.myRole();
+
+  // Checking only direct membership because inside the migrations we can't navigate the parent groups
+  if (myRole !== "admin" && myRole !== "manager") {
+    return false;
+  }
+
   const currentReadKeyId = group.get("readKey");
 
   if (!currentReadKeyId) {
@@ -190,11 +197,27 @@ export class RawGroup<
   ) {
     super(core, options);
     this.crypto = core.node.crypto;
+    this.migrate();
+  }
 
-    // Checks if this is not an account
-    if (core.isGroup() && !core.verified.isStreaming()) {
-      // rotateReadKeyIfNeeded(this);
+  migrate() {
+    if (!this.core.isGroup()) {
+      return;
+    }
+
+    const runMigrations = () => {
+      rotateReadKeyIfNeeded(this);
       healMissingKeyForEveryone(this);
+    };
+
+    // We need the group and their parents to be completely downloaded to correctly handle the migrations
+    if (!this.core.isCompletelyDownloaded()) {
+      this.core.waitFor({
+        predicate: (core) => core.isCompletelyDownloaded(),
+        onSuccess: runMigrations,
+      });
+    } else {
+      runMigrations();
     }
   }
 
@@ -318,7 +341,7 @@ export class RawGroup<
    * @category 1. Role reading
    */
   myRole(): Role | undefined {
-    return this.roleOfInternal(this.core.node.getCurrentAgent().id);
+    return this.roleOfInternal(this.core.node.getCurrentAccountOrAgentID());
   }
 
   /**

package/src/crypto/PureJSCrypto.ts

@@ -379,17 +379,11 @@ export class PureJSSessionLog implements SessionLogImpl {
         tx: { sessionID: this.sessionID, txIndex: txIndex },
       };
 
-      const nOnce = this.crypto.generateJsonNonce(nOnceMaterial);
-
-      const ciphertext = base64URLtoBytes(
-        tx.encryptedChanges.substring("encrypted_U".length),
-      );
-      const keySecretBytes = base58.decode(
-        keySecret.substring("keySecret_z".length),
+      return this.crypto.decryptRaw(
+        tx.encryptedChanges,
+        keySecret,
+        nOnceMaterial,
       );
-      const plaintext = xsalsa20(keySecretBytes, nOnce, ciphertext);
-
-      return textDecoder.decode(plaintext);
     } else {
       return tx.changes;
     }
@@ -415,17 +409,7 @@ export class PureJSSessionLog implements SessionLogImpl {
         tx: { sessionID: this.sessionID, txIndex: txIndex },
       };
 
-      const nOnce = this.crypto.generateJsonNonce(nOnceMaterial);
-
-      const ciphertext = base64URLtoBytes(
-        tx.meta.substring("encrypted_U".length),
-      );
-      const keySecretBytes = base58.decode(
-        keySecret.substring("keySecret_z".length),
-      );
-      const plaintext = xsalsa20(keySecretBytes, nOnce, ciphertext);
-
-      return textDecoder.decode(plaintext);
+      return this.crypto.decryptRaw(tx.meta, keySecret, nOnceMaterial);
     } else {
       return tx.meta;
     }

package/src/knownState.ts

@@ -109,7 +109,7 @@ export function cloneKnownState(knownState: CoValueKnownState) {
 /**
  * Checks if all the local sessions have the same counters as in remote.
  */
-export function areLocalSessionsUploaded(
+export function isKnownStateSubsetOf(
   local: Record<string, number>,
   remote: Record<string, number>,
 ) {

package/src/localNode.ts

@@ -650,8 +650,7 @@ export class LocalNode {
     );
 
     const contentPieces =
-      groupAsInvite.core.verified.newContentSince(group.core.knownState()) ??
-      [];
+      groupAsInvite.core.newContentSince(group.core.knownState()) ?? [];
 
     // Import the new transactions to the current localNode
     for (const contentPiece of contentPieces) {

package/src/queue/LocalTransactionsSyncQueue.ts

@@ -1,10 +1,15 @@
 import {
   addTransactionToContentMessage,
   createContentMessage,
+  knownStateFromContent,
 } from "../coValueContentMessage.js";
 import { Transaction, VerifiedState } from "../coValueCore/verifiedState.js";
 import { Signature } from "../crypto/crypto.js";
 import { RawCoID, SessionID } from "../ids.js";
+import {
+  combineKnownStateSessions,
+  KnownStateSessions,
+} from "../knownState.js";
 import { NewContentMessage } from "../sync.js";
 import { LinkedList } from "./LinkedList.js";
 
@@ -113,9 +118,29 @@ export class LocalTransactionsSyncQueue {
     this.processingSyncs = true;
 
     queueMicrotask(() => {
+      const firstContentPieceMap = new Map<RawCoID, NewContentMessage>();
+
       while (this.queue.head) {
         const content = this.queue.head.value;
 
+        const firstContentPiece = firstContentPieceMap.get(content.id);
+
+        if (!firstContentPiece) {
+          firstContentPieceMap.set(content.id, content);
+        } else {
+          // There is already a content piece for this coValue, so this means that we need to flag
+          // that this content is going to be streamed
+          if (!firstContentPiece.expectContentUntil) {
+            firstContentPiece.expectContentUntil =
+              knownStateFromContent(firstContentPiece).sessions;
+          }
+
+          combineKnownStateSessions(
+            firstContentPiece.expectContentUntil,
+            knownStateFromContent(content).sessions,
+          );
+        }
+
         this.sync(content);
 
         this.queue.shift();

package/src/storage/knownState.ts

@@ -3,7 +3,7 @@ import { RawCoID } from "../ids.js";
 import {
   CoValueKnownState,
   emptyKnownState,
-  areLocalSessionsUploaded,
+  isKnownStateSubsetOf,
 } from "../knownState.js";
 
 /**
@@ -87,7 +87,7 @@ function isInSync(
     return false;
   }
 
-  return areLocalSessionsUploaded(
+  return isKnownStateSubsetOf(
     knownState.sessions,
     knownStateFromStorage.sessions,
   );

package/src/storage/sqlite/index.ts

@@ -6,13 +6,26 @@ import type { SQLiteDatabaseDriver } from "./types.js";
 export type { SQLiteDatabaseDriver };
 
 export function getSqliteStorage(db: SQLiteDatabaseDriver) {
-  const rows = db.query<{ user_version: string }>("PRAGMA user_version", []);
-  const userVersion = Number(rows[0]?.user_version) ?? 0;
+  let userVersion: number;
+
+  if (db.getMigrationVersion) {
+    userVersion = db.getMigrationVersion();
+  } else {
+    const rows = db.query<{ user_version: string }>("PRAGMA user_version", []);
+    userVersion = rows[0]?.user_version ? Number(rows[0].user_version) : 0;
+  }
 
   const migrations = getSQLiteMigrationQueries(userVersion);
 
   for (const migration of migrations) {
-    db.run(migration, []);
+    for (const query of migration.queries) {
+      db.run(query, []);
+    }
+    if (db.saveMigrationVersion) {
+      db.saveMigrationVersion(migration.version);
+    } else {
+      db.run(`PRAGMA user_version = ${migration.version};`, []);
+    }
   }
 
   return new StorageApiSync(new SQLiteClient(db));

package/src/storage/sqlite/sqliteMigrations.ts

@@ -21,7 +21,6 @@ export const migrations: Record<number, string[]> = {
       header TEXT NOT NULL UNIQUE
     );`,
     "CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);",
-    "PRAGMA user_version = 1;",
   ],
   3: [
     `CREATE TABLE IF NOT EXISTS signatureAfter (
@@ -31,14 +30,18 @@ export const migrations: Record<number, string[]> = {
       PRIMARY KEY (ses, idx)
     ) WITHOUT ROWID;`,
     "ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;",
-    "PRAGMA user_version = 3;",
   ],
 };
 
-export function getSQLiteMigrationQueries(version: number): string[] {
+type Migration = {
+  version: number;
+  queries: string[];
+};
+
+export function getSQLiteMigrationQueries(version: number): Migration[] {
   return Object.keys(migrations)
     .map((k) => Number.parseInt(k, 10))
     .filter((v) => v > version)
     .sort((a, b) => a - b)
-    .flatMap((v) => migrations[v] ?? []);
+    .map((v) => ({ version: v, queries: migrations[v] ?? [] }));
 }

package/src/storage/sqlite/types.ts

@@ -4,4 +4,6 @@ export interface SQLiteDatabaseDriver {
   query<T>(sql: string, params: unknown[]): T[];
   transaction(callback: () => unknown): void;
   closeDb(): void;
+  getMigrationVersion?(): number;
+  saveMigrationVersion?(version: number): void;
 }

package/src/storage/sqliteAsync/index.ts

@@ -9,16 +9,29 @@ import type { SQLiteDatabaseDriverAsync } from "./types.js";
 export async function getSqliteStorageAsync(db: SQLiteDatabaseDriverAsync) {
   await db.initialize();
 
-  const rows = await db.query<{ user_version: string }>(
-    "PRAGMA user_version",
-    [],
-  );
-  const userVersion = Number(rows[0]?.user_version) ?? 0;
+  let userVersion: number;
+
+  if (db.getMigrationVersion) {
+    userVersion = await db.getMigrationVersion();
+  } else {
+    const rows = await db.query<{ user_version: string }>(
+      "PRAGMA user_version",
+      [],
+    );
+    userVersion = rows[0]?.user_version ? Number(rows[0].user_version) : 0;
+  }
 
   const migrations = getSQLiteMigrationQueries(userVersion);
 
   for (const migration of migrations) {
-    await db.run(migration, []);
+    for (const query of migration.queries) {
+      await db.run(query, []);
+    }
+    if (db.saveMigrationVersion) {
+      await db.saveMigrationVersion(migration.version);
+    } else {
+      await db.run(`PRAGMA user_version = ${migration.version};`, []);
+    }
   }
 
   return new StorageApiAsync(new SQLiteClientAsync(db));

package/src/storage/sqliteAsync/types.ts

@@ -5,4 +5,6 @@ export interface SQLiteDatabaseDriverAsync {
   get<T>(sql: string, params: unknown[]): Promise<T | undefined>;
   transaction(callback: () => unknown): Promise<unknown>;
   closeDb(): Promise<unknown>;
+  getMigrationVersion?(): Promise<number>;
+  saveMigrationVersion?(version: number): Promise<void>;
 }

package/src/sync.ts

@@ -228,7 +228,7 @@ export class SyncManager {
       }
     }
 
-    const newContentPieces = coValue.verified.newContentSince(
+    const newContentPieces = coValue.newContentSince(
       peer.getOptimisticKnownState(id),
     );
 
@@ -585,6 +585,8 @@ export class SyncManager {
           sessions: msg.expectContentUntil,
         });
       }
+    } else if (msg.expectContentUntil) {
+      coValue.verified.setStreamingKnownState(msg.expectContentUntil);
     }
 
     // At this point the CoValue must be in memory, if not we have a bug
@@ -647,6 +649,9 @@ export class SyncManager {
             peerRole: peer.role,
             id: msg.id,
             err: result.error,
+            msgKnownState: knownStateFromContent(msg).sessions,
+            knownState: coValue.knownState().sessions,
+            newContent: validNewContent.new,
           });
           // TODO Mark only the session as errored, not the whole coValue
           coValue.markErrored(peer.id, result.error);
@@ -822,7 +827,7 @@ export class SyncManager {
         return undefined;
       }
 
-      return value.verified.newContentSince(correction);
+      return value.newContentSince(correction);
     });
   }
 

package/src/tests/PureJSCrypto.test.ts

@@ -117,8 +117,7 @@ describe("PureJSCrypto", () => {
       true,
     );
 
-    const content =
-      mapInOtherSession.core.verified.newContentSince(undefined)?.[0];
+    const content = mapInOtherSession.core.newContentSince(undefined)?.[0];
     assert(content);
 
     client.node.syncManager.handleNewContent(content, "storage");