cojson 0.9.9 → 0.9.11

package/src/crypto/WasmCrypto.ts

@@ -15,6 +15,7 @@ import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
 import { RawCoID, TransactionID } from "../ids.js";
 import { Stringified, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -240,7 +241,7 @@ export class WasmCrypto extends CryptoProvider<Uint8Array> {
     try {
       return JSON.parse(textDecoder.decode(plaintext));
     } catch (e) {
-      console.error("Failed to decrypt/parse sealed message", e);
+      logger.error("Failed to decrypt/parse sealed message", e);
       return undefined;
     }
   }

package/src/crypto/crypto.ts

@@ -4,6 +4,7 @@ import { AgentID, RawCoID, TransactionID } from "../ids.js";
 import { SessionID } from "../ids.js";
 import { Stringified, parseJSON, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 
 export type SignerSecret = `signerSecret_z${string}`;
 export type SignerID = `signer_z${string}`;
@@ -159,7 +160,7 @@ export abstract class CryptoProvider<Blake3State = any> {
     try {
       return parseJSON(this.decryptRaw(encrypted, keySecret, nOnceMaterial));
     } catch (e) {
-      console.error("Decryption error", e);
+      logger.error("Decryption error", e);
       return undefined;
     }
   }
@@ -305,10 +306,7 @@ export class StreamingHash {
 
   update(value: JsonValue): Uint8Array {
     const encoded = textEncoder.encode(stableStringify(value));
-    // const before = performance.now();
     this.state = this.crypto.blake3IncrementalUpdate(this.state, encoded);
-    // const after = performance.now();
-    // console.log(`Hashing throughput in MB/s`, 1000 * (encoded.length / (after - before)) / (1024 * 1024));
     return encoded;
   }
 

package/src/exports.ts

@@ -74,6 +74,7 @@ import {
 
 type Value = JsonValue | AnyRawCoValue;
 
+import { logger } from "./logger.js";
 import { getPriorityFromHeader } from "./priority.js";
 import { FileSystem } from "./storage/FileSystem.js";
 import { BlockFilename, LSMStorage, WalFilename } from "./storage/index.js";
@@ -141,6 +142,7 @@ export {
   emptyKnownState,
   RawCoPlainText,
   stringifyOpID,
+  logger,
 };
 
 export type {

package/src/localNode.ts

@@ -27,6 +27,7 @@ import {
 } from "./coValues/group.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
+import { logger } from "./logger.js";
 import { Peer, PeerID, SyncManager } from "./sync.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
@@ -230,7 +231,7 @@ export class LocalNode {
 
       return node;
     } catch (e) {
-      console.error("Error withLoadedAccount", e);
+      logger.error("Error withLoadedAccount", e);
       throw e;
     }
   }
@@ -269,7 +270,7 @@ export class LocalNode {
         this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
 
       await entry.loadFromPeers(peers).catch((e) => {
-        console.error("Error loading from peers", id, e);
+        logger.error("Error loading from peers", id, e);
       });
     }
 
@@ -311,8 +312,6 @@ export class LocalNode {
     let stopped = false;
     let unsubscribe!: () => void;
 
-    // console.log("Subscribing to " + id);
-
     this.load(id)
       .then((coValue) => {
         if (stopped) {
@@ -325,11 +324,10 @@ export class LocalNode {
         unsubscribe = coValue.subscribe(callback);
       })
       .catch((e) => {
-        console.error("Error subscribing to ", id, e);
+        logger.error("Error subscribing to ", id, e);
       });
 
     return () => {
-      console.log("Unsubscribing from " + id);
       stopped = true;
       unsubscribe?.();
     };
@@ -390,9 +388,7 @@ export class LocalNode {
       (existingRole === "reader" && inviteRole === "readerInvite") ||
       (existingRole && inviteRole === "writeOnlyInvite")
     ) {
-      console.debug(
-        "Not accepting invite that would replace or downgrade role",
-      );
+      logger.debug("Not accepting invite that would replace or downgrade role");
       return;
     }
 

package/src/logger.ts

@@ -0,0 +1,78 @@
+export enum LogLevel {
+  DEBUG = 0,
+  INFO = 1,
+  WARN = 2,
+  ERROR = 3,
+  NONE = 4,
+}
+
+export interface LogSystem {
+  debug(message: string, ...args: any[]): void;
+  info(message: string, ...args: any[]): void;
+  warn(message: string, ...args: any[]): void;
+  error(message: string, ...args: any[]): void;
+}
+
+// Default console-based logging system
+export class ConsoleLogSystem implements LogSystem {
+  debug(message: string, ...args: any[]) {
+    console.debug(message, ...args);
+  }
+  info(message: string, ...args: any[]) {
+    console.info(message, ...args);
+  }
+  warn(message: string, ...args: any[]) {
+    console.warn(message, ...args);
+  }
+  error(message: string, ...args: any[]) {
+    console.error(message, ...args);
+  }
+}
+
+export class Logger {
+  private level: LogLevel;
+  private logSystem: LogSystem;
+
+  constructor(
+    level: LogLevel = LogLevel.INFO,
+    logSystem: LogSystem = new ConsoleLogSystem(),
+  ) {
+    this.level = level;
+    this.logSystem = logSystem;
+  }
+
+  setLevel(level: LogLevel) {
+    this.level = level;
+  }
+
+  setLogSystem(logSystem: LogSystem) {
+    this.logSystem = logSystem;
+  }
+
+  debug(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.DEBUG) {
+      this.logSystem.debug(message, ...args);
+    }
+  }
+
+  info(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.INFO) {
+      this.logSystem.info(message, ...args);
+    }
+  }
+
+  warn(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.WARN) {
+      this.logSystem.warn(message, ...args);
+    }
+  }
+
+  error(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.ERROR) {
+      this.logSystem.error(message, ...args);
+    }
+  }
+}
+
+// Create default logger instance
+export const logger = new Logger();

package/src/permissions.ts

@@ -14,6 +14,7 @@ import {
 } from "./ids.js";
 import { parseJSON } from "./jsonStringify.js";
 import { JsonValue } from "./jsonValue.js";
+import { logger } from "./logger.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
@@ -46,7 +47,7 @@ function logPermissionError(...args: unknown[]) {
     return;
   }
 
-  console.warn(...args);
+  logger.warn("Permission error", ...args);
 }
 
 export function determineValidTransactions(
@@ -204,7 +205,6 @@ function determineValidTransactionsForGroup(
   const writeKeys = new Set<string>();
 
   for (const { sessionID, txIndex, tx } of allTransactionsSorted) {
-    // console.log("before", { memberState, validTransactions });
     const transactor = accountOrAgentIDfromSessionID(sessionID);
 
     if (tx.privacy === "private") {
@@ -458,8 +458,6 @@ function determineValidTransactionsForGroup(
 
     memberState[affectedMember] = change.value;
     validTransactions.push({ txID: { sessionID, txIndex }, tx });
-
-    // console.log("after", { memberState, validTransactions });
   }
 
   return { validTransactions, memberState };
@@ -473,7 +471,7 @@ function agentInAccountOrMemberInGroup(
     return groupAtTime.currentAgentID().match(
       (agentID) => agentID,
       (e) => {
-        console.error(
+        logger.error(
           "Error while determining current agent ID in valid transactions",
           e,
         );

package/src/storage/FileSystem.ts

@@ -87,26 +87,15 @@ export async function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
   const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
   await fs.append(file, headerBytes);
 
-  // console.log(
-  //     "full file",
-  //     yield* $(
-  //         fs.read(file as unknown as RH, 0, offset + headerBytes.length),
-  //     ),
-  // );
-
   const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
     3,
     "0",
   )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
     headerBytes.length
   }.jsonl`;
-  // console.log("renaming to" + filename);
   await fs.closeAndRename(file, filename);
 
   return filename;
-
-  // console.log("Wrote block", filename, blockHeader);
-  // console.log("IDs in block", blockHeader.map(e => e.id));
 }
 
 export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
@@ -120,6 +109,5 @@ export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
     ...chunk,
   };
   const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
-  console.log("writing to WAL", handle, id, bytes.length);
   return fs.append(handle, bytes);
 }

package/src/storage/index.ts

@@ -2,6 +2,7 @@ import { CoID, RawCoValue } from "../coValue.js";
 import { CoValueHeader, Transaction } from "../coValueCore.js";
 import { Signature } from "../crypto/crypto.js";
 import { RawCoID } from "../ids.js";
+import { logger } from "../logger.js";
 import { connectedPeers } from "../streamUtils.js";
 import {
   CoValueKnownState,
@@ -68,7 +69,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
     const processMessages = async () => {
       for await (const msg of fromLocalNode) {
-        console.log("Storage msg start", nMsg);
         try {
           if (msg === "Disconnected" || msg === "PingTimeout") {
             throw new Error("Unexpected Disconnected message");
@@ -83,32 +83,29 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
             await this.sendNewContent(msg.id, msg, undefined);
           }
         } catch (e) {
-          console.error(
-            new Error(
-              `Error reading from localNode, handling msg\n\n${JSON.stringify(
-                msg,
-                (k, v) =>
-                  k === "changes" || k === "encryptedChanges"
-                    ? v.slice(0, 20) + "..."
-                    : v,
-              )}`,
-              { cause: e },
-            ),
+          logger.error(
+            `Error reading from localNode, handling msg\n\n${JSON.stringify(
+              msg,
+              (k, v) =>
+                k === "changes" || k === "encryptedChanges"
+                  ? v.slice(0, 20) + "..."
+                  : v,
+            )}`,
+            e,
           );
         }
-        console.log("Storage msg end", nMsg);
         nMsg++;
       }
     };
 
     processMessages().catch((e) =>
-      console.error("Error in processMessages in storage", e),
+      logger.error("Error in processMessages in storage", e),
     );
 
     setTimeout(
       () =>
         this.compact().catch((e) => {
-          console.error("Error while compacting", e);
+          logger.error("Error while compacting", e);
         }),
       20000,
     );
@@ -134,7 +131,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
           sessions: {},
           asDependencyOf,
         })
-        .catch((e) => console.error("Error while pushing known", e));
+        .catch((e) => logger.error("Error while pushing known", e));
 
       return;
     }
@@ -190,13 +187,13 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         ...ourKnown,
         asDependencyOf,
       })
-      .catch((e) => console.error("Error while pushing known", e));
+      .catch((e) => logger.error("Error while pushing known", e));
 
     for (const message of newContentMessages) {
       if (Object.keys(message.new).length === 0) continue;
       this.toLocalNode
         .push(message)
-        .catch((e) => console.error("Error while pushing new content", e));
+        .catch((e) => logger.error("Error while pushing new content", e));
     }
 
     this.coValues[id] = coValue;
@@ -232,20 +229,19 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
     if (!coValue) {
       if (newContent.header) {
-        // console.log("Creating in WAL", newContent.id);
         await this.withWAL((wal) =>
           writeToWal(wal, this.fs, newContent.id, newContentAsChunk),
         );
 
         this.coValues[newContent.id] = newContentAsChunk;
       } else {
-        console.warn("Incontiguous incoming update for " + newContent.id);
+        logger.warn("Incontiguous incoming update for " + newContent.id);
         return;
       }
     } else {
       const merged = mergeChunks(coValue, newContentAsChunk);
       if (merged === "nonContigous") {
-        console.warn(
+        logger.warn(
           "Non-contigous new content for " + newContent.id,
           Object.entries(coValue.sessionEntries).map(([session, entries]) =>
             entries.map((entry) => ({
@@ -264,7 +260,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
           ),
         );
       } else {
-        // console.log("Appending to WAL", newContent.id);
         await this.withWAL((wal) =>
           writeToWal(wal, this.fs, newContent.id, newContentAsChunk),
         );
@@ -301,8 +296,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
       const { handle, size } = await this.getBlockHandle(blockFile, fs);
 
-      // console.log("Attempting to load", id, blockFile);
-
       if (!cachedHeader) {
         cachedHeader = {};
         const header = await readHeader(blockFile, handle, size, fs);
@@ -317,15 +310,13 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
       }
       const headerEntry = cachedHeader[id];
 
-      // console.log("Header entry", id, headerEntry);
-
       if (headerEntry) {
         const nextChunk = await readChunk(handle, headerEntry, fs);
         if (result) {
           const merged = mergeChunks(result, nextChunk);
 
           if (merged === "nonContigous") {
-            console.warn(
+            logger.warn(
               "Non-contigous chunks while loading " + id,
               result,
               nextChunk,
@@ -354,7 +345,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
     const coValues = new Map<RawCoID, CoValueChunk>();
 
-    console.log("Compacting WAL files", walFiles);
     if (walFiles.length === 0) return;
 
     const oldWal = this.currentWal;
@@ -385,7 +375,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         if (existingChunk) {
           const merged = mergeChunks(existingChunk, chunk);
           if (merged === "nonContigous") {
-            console.log(
+            logger.info(
               "Non-contigous chunks in " + chunk.id + ", " + fileName,
               existingChunk,
               chunk,
@@ -411,8 +401,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
       return acc;
     }, 0);
 
-    console.log([...coValues.keys()], fileNames, highestBlockNumber);
-
     await writeBlock(coValues, MAX_N_LEVELS, highestBlockNumber + 1, this.fs);
 
     for (const walFile of walFiles) {
@@ -438,15 +426,11 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
       blockFilesByLevelInOrder[level]!.push(blockFile);
     }
 
-    console.log(blockFilesByLevelInOrder);
-
     for (let level = MAX_N_LEVELS; level > 0; level--) {
       const nBlocksDesired = Math.pow(2, level);
       const blocksInLevel = blockFilesByLevelInOrder[level];
 
       if (blocksInLevel && blocksInLevel.length > nBlocksDesired) {
-        console.log("Compacting blocks in level", level, blocksInLevel);
-
         const coValues = new Map<RawCoID, CoValueChunk>();
 
         for (const blockFile of blocksInLevel) {
@@ -465,7 +449,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
             if (existingChunk) {
               const merged = mergeChunks(existingChunk, chunk);
               if (merged === "nonContigous") {
-                console.log(
+                logger.info(
                   "Non-contigous chunks in " + entry.id + ", " + blockFile,
                   existingChunk,
                   chunk,
@@ -517,7 +501,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
     setTimeout(
       () =>
         this.compact().catch((e) => {
-          console.error("Error while compacting", e);
+          logger.error("Error while compacting", e);
         }),
       5000,
     );

package/src/sync.ts

@@ -6,6 +6,7 @@ import { CoValueCore } from "./coValueCore.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
+import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 
 export type CoValueKnownState = {
@@ -150,7 +151,7 @@ export class SyncManager {
 
   async handleSyncMessage(msg: SyncMessage, peer: PeerState) {
     if (peer.erroredCoValues.has(msg.id)) {
-      console.error(
+      logger.warn(
         `Skipping message ${msg.action} on errored coValue ${msg.id} from peer ${peer.id}`,
       );
       return;
@@ -182,7 +183,7 @@ export class SyncManager {
 
     if (entry.state.type !== "available") {
       entry.loadFromPeers([peer]).catch((e: unknown) => {
-        console.error("Error sending load", e);
+        logger.error("Error sending load", e);
       });
       return;
     }
@@ -199,7 +200,7 @@ export class SyncManager {
         action: "load",
         ...coValue.knownState(),
       }).catch((e: unknown) => {
-        console.error("Error sending load", e);
+        logger.error("Error sending load", e);
       });
     }
   }
@@ -229,7 +230,7 @@ export class SyncManager {
         asDependencyOf,
         ...coValue.knownState(),
       }).catch((e: unknown) => {
-        console.error("Error sending known state", e);
+        logger.error("Error sending known state", e);
       });
 
       peer.toldKnownState.add(id);
@@ -256,15 +257,8 @@ export class SyncManager {
       const sendPieces = async () => {
         let lastYield = performance.now();
         for (const [_i, piece] of newContentPieces.entries()) {
-          // console.log(
-          //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${
-          //         newContentPieces.length
-          //     } header: ${!!piece.header}`,
-          //     // Object.values(piece.new).map((s) => s.newTransactions)
-          // );
-
           this.trySendToPeer(peer, piece).catch((e: unknown) => {
-            console.error("Error sending content piece", e);
+            logger.error("Error sending content piece", e);
           });
 
           if (performance.now() - lastYield > 10) {
@@ -277,7 +271,7 @@ export class SyncManager {
       };
 
       sendPieces().catch((e) => {
-        console.error("Error sending new content piece, retrying", e);
+        logger.error("Error sending new content piece, retrying", e);
         peer.optimisticKnownStates.dispatch({
           type: "SET",
           id,
@@ -337,7 +331,7 @@ export class SyncManager {
           return;
         }
         if (msg === "PingTimeout") {
-          console.error("Ping timeout from peer", peer.id);
+          logger.error("Ping timeout from peer", peer.id);
           return;
         }
         try {
@@ -360,13 +354,13 @@ export class SyncManager {
     processMessages()
       .then(() => {
         if (peer.crashOnClose) {
-          console.error("Unexepcted close from peer", peer.id);
+          logger.warn("Unexepcted close from peer", peer.id);
           this.local.crashed = new Error("Unexpected close from peer");
           throw new Error("Unexpected close from peer");
         }
       })
       .catch((e) => {
-        console.error("Error processing messages from peer", peer.id, e);
+        logger.error("Error processing messages from peer", peer.id, e);
         if (peer.crashOnClose) {
           this.local.crashed = e;
           throw new Error(e);
@@ -406,13 +400,13 @@ export class SyncManager {
         // where we can get informations about the coValue
         if (msg.header || Object.keys(msg.sessions).length > 0) {
           entry.loadFromPeers([peer]).catch((e) => {
-            console.error("Error loading coValue in handleLoad", e);
+            logger.error("Error loading coValue in handleLoad", e);
           });
         }
         return;
       } else {
         this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
-          console.error("Error loading coValue in handleLoad", e);
+          logger.error("Error loading coValue in handleLoad", e);
         });
       }
     }
@@ -439,7 +433,7 @@ export class SyncManager {
               header: false,
               sessions: {},
             }).catch((e) => {
-              console.error("Error sending known state back", e);
+              logger.error("Error sending known state back", e);
             });
 
             return;
@@ -449,7 +443,7 @@ export class SyncManager {
           await this.sendNewContentIncludingDependencies(msg.id, peer);
         })
         .catch((e) => {
-          console.error("Error loading coValue in handleLoad loading state", e);
+          logger.error("Error loading coValue in handleLoad loading state", e);
         });
     }
 
@@ -484,7 +478,7 @@ export class SyncManager {
               peer.role === "storage" ? undefined : peer.id,
             )
             .catch((e) => {
-              console.error(
+              logger.error(
                 `Error loading coValue ${msg.id} to create loading state, as dependency of ${msg.asDependencyOf}`,
                 e,
               );
@@ -521,7 +515,7 @@ export class SyncManager {
 
     if (entry.state.type !== "available") {
       if (!msg.header) {
-        console.error("Expected header to be sent in first message");
+        logger.error("Expected header to be sent in first message");
         return;
       }
 
@@ -584,7 +578,7 @@ export class SyncManager {
               : t.changes.length,
           )
           .reduce((a, b) => a + b, 0);
-        console.log(
+        logger.debug(
           `Adding incoming transactions took ${(after - before).toFixed(
             2,
           )}ms for ${totalTxLength} bytes = bandwidth: ${(
@@ -602,7 +596,7 @@ export class SyncManager {
       // );
 
       if (result.isErr()) {
-        console.error(
+        logger.error(
           "Failed to add transactions from",
           peer.id,
           result.error,
@@ -633,7 +627,7 @@ export class SyncManager {
         isCorrection: true,
         ...coValue.knownState(),
       }).catch((e) => {
-        console.error("Error sending known state correction", e);
+        logger.error("Error sending known state correction", e);
       });
     } else {
       /**
@@ -647,7 +641,7 @@ export class SyncManager {
         action: "known",
         ...coValue.knownState(),
       }).catch((e: unknown) => {
-        console.error("Error sending known state", e);
+        logger.error("Error sending known state", e);
       });
     }
 
@@ -681,9 +675,6 @@ export class SyncManager {
       const done = new Promise<void>((resolve) => {
         queueMicrotask(async () => {
           delete this.requestedSyncs[coValue.id];
-          // if (entry.nRequestsThisTick >= 2) {
-          //     console.log("Syncing", coValue.id, "for", entry.nRequestsThisTick, "requests");
-          // }
           await this.actuallySyncCoValue(coValue);
           resolve();
         });

package/src/tests/PeerState.test.ts

@@ -49,12 +49,9 @@ describe("PeerState", () => {
 
   test("should perform graceful shutdown", () => {
     const { mockPeer, peerState } = setup();
-    const consoleSpy = vi.spyOn(console, "debug").mockImplementation(() => {});
     peerState.gracefulShutdown();
     expect(mockPeer.outgoing.close).toHaveBeenCalled();
     expect(peerState.closed).toBe(true);
-    expect(consoleSpy).toHaveBeenCalledWith("Gracefully closing", "test-peer");
-    consoleSpy.mockRestore();
   });
 
   test("should empty the queue when closing", async () => {

package/src/tests/PriorityBasedMessageQueue.test.ts

@@ -22,7 +22,7 @@ describe("PriorityBasedMessageQueue", () => {
     const { queue } = setup();
     expect(queue["defaultPriority"]).toBe(CO_VALUE_PRIORITY.MEDIUM);
     expect(queue["queues"].length).toBe(8);
-    expect(queue["queues"].every((q) => q.length === 0)).toBe(true);
+    expect(queue["queues"].every((q) => !q.isNonEmpty())).toBe(true);
   });
 
   test("should push message with default priority", async () => {