cojson 0.9.12 → 0.9.18

package/src/exports.ts

@@ -38,7 +38,7 @@ import {
 } from "./ids.js";
 import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
 import { LocalNode } from "./localNode.js";
-import type { Role } from "./permissions.js";
+import type { AccountRole, Role } from "./permissions.js";
 import { Channel, connectedPeers } from "./streamUtils.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
@@ -74,6 +74,7 @@ import {
 
 type Value = JsonValue | AnyRawCoValue;
 
+import { logger } from "./logger.js";
 import { getPriorityFromHeader } from "./priority.js";
 import { FileSystem } from "./storage/FileSystem.js";
 import { BlockFilename, LSMStorage, WalFilename } from "./storage/index.js";
@@ -141,6 +142,7 @@ export {
   emptyKnownState,
   RawCoPlainText,
   stringifyOpID,
+  logger,
 };
 
 export type {
@@ -156,6 +158,7 @@ export type {
   Stringified,
   CoStreamItem,
   OpID,
+  AccountRole,
 };
 
 // eslint-disable-next-line @typescript-eslint/no-namespace

package/src/localNode.ts

@@ -27,6 +27,7 @@ import {
 } from "./coValues/group.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
+import { logger } from "./logger.js";
 import { Peer, PeerID, SyncManager } from "./sync.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
@@ -230,7 +231,7 @@ export class LocalNode {
 
       return node;
     } catch (e) {
-      console.error("Error withLoadedAccount", e);
+      logger.error("Error withLoadedAccount: " + (e as Error)?.message);
       throw e;
     }
   }
@@ -269,7 +270,9 @@ export class LocalNode {
         this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
 
       await entry.loadFromPeers(peers).catch((e) => {
-        console.error("Error loading from peers", id, e);
+        logger.error("Error loading from peers: " + (e as Error)?.message, {
+          id,
+        });
       });
     }
 
@@ -311,8 +314,6 @@ export class LocalNode {
     let stopped = false;
     let unsubscribe!: () => void;
 
-    // console.log("Subscribing to " + id);
-
     this.load(id)
       .then((coValue) => {
         if (stopped) {
@@ -325,11 +326,12 @@ export class LocalNode {
         unsubscribe = coValue.subscribe(callback);
       })
       .catch((e) => {
-        console.error("Error subscribing to ", id, e);
+        logger.error(
+          "Error subscribing to " + id + ": " + (e as Error)?.message,
+        );
       });
 
     return () => {
-      console.log("Unsubscribing from " + id);
       stopped = true;
       unsubscribe?.();
     };
@@ -390,9 +392,7 @@ export class LocalNode {
       (existingRole === "reader" && inviteRole === "readerInvite") ||
       (existingRole && inviteRole === "writeOnlyInvite")
     ) {
-      console.debug(
-        "Not accepting invite that would replace or downgrade role",
-      );
+      logger.debug("Not accepting invite that would replace or downgrade role");
       return;
     }
 

package/src/logger.ts

@@ -0,0 +1,80 @@
+import type { JsonValue } from "./jsonValue.js";
+
+export enum LogLevel {
+  DEBUG = 0,
+  INFO = 1,
+  WARN = 2,
+  ERROR = 3,
+  NONE = 4,
+}
+
+export interface LogSystem {
+  debug(message: string, attributes?: Record<string, JsonValue>): void;
+  info(message: string, attributes?: Record<string, JsonValue>): void;
+  warn(message: string, attributes?: Record<string, JsonValue>): void;
+  error(message: string, attributes?: Record<string, JsonValue>): void;
+}
+
+// Default console-based logging system
+export class ConsoleLogSystem implements LogSystem {
+  debug(message: string, attributes?: Record<string, JsonValue>) {
+    console.debug(message, attributes);
+  }
+  info(message: string, attributes?: Record<string, JsonValue>) {
+    console.info(message, attributes);
+  }
+  warn(message: string, attributes?: Record<string, JsonValue>) {
+    console.warn(message, attributes);
+  }
+  error(message: string, attributes?: Record<string, JsonValue>) {
+    console.error(message, attributes);
+  }
+}
+
+export class Logger {
+  private level: LogLevel;
+  private logSystem: LogSystem;
+
+  constructor(
+    level: LogLevel = LogLevel.INFO,
+    logSystem: LogSystem = new ConsoleLogSystem(),
+  ) {
+    this.level = level;
+    this.logSystem = logSystem;
+  }
+
+  setLevel(level: LogLevel) {
+    this.level = level;
+  }
+
+  setLogSystem(logSystem: LogSystem) {
+    this.logSystem = logSystem;
+  }
+
+  debug(message: string, attributes?: Record<string, JsonValue>) {
+    if (this.level <= LogLevel.DEBUG) {
+      this.logSystem.debug(message, attributes);
+    }
+  }
+
+  info(message: string, attributes?: Record<string, JsonValue>) {
+    if (this.level <= LogLevel.INFO) {
+      this.logSystem.info(message, attributes);
+    }
+  }
+
+  warn(message: string, attributes?: Record<string, JsonValue>) {
+    if (this.level <= LogLevel.WARN) {
+      this.logSystem.warn(message, attributes);
+    }
+  }
+
+  error(message: string, attributes?: Record<string, JsonValue>) {
+    if (this.level <= LogLevel.ERROR) {
+      this.logSystem.error(message, attributes);
+    }
+  }
+}
+
+// Create default logger instance
+export const logger = new Logger();

package/src/permissions.ts

@@ -14,6 +14,7 @@ import {
 } from "./ids.js";
 import { parseJSON } from "./jsonStringify.js";
 import { JsonValue } from "./jsonValue.js";
+import { logger } from "./logger.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
@@ -41,12 +42,15 @@ export function disablePermissionErrors() {
   logPermissionErrors = false;
 }
 
-function logPermissionError(...args: unknown[]) {
+function logPermissionError(
+  message: string,
+  attributes?: Record<string, JsonValue>,
+) {
   if (logPermissionErrors === false) {
     return;
   }
 
-  console.warn(...args);
+  logger.warn("Permission error: " + message, attributes);
 }
 
 export function determineValidTransactions(
@@ -204,7 +208,6 @@ function determineValidTransactionsForGroup(
   const writeKeys = new Set<string>();
 
   for (const { sessionID, txIndex, tx } of allTransactionsSorted) {
-    // console.log("before", { memberState, validTransactions });
     const transactor = accountOrAgentIDfromSessionID(sessionID);
 
     if (tx.privacy === "private") {
@@ -227,17 +230,10 @@ function determineValidTransactionsForGroup(
     try {
       changes = parseJSON(tx.changes);
     } catch (e) {
-      logPermissionError(
-        coValue.id,
-        "Invalid JSON in transaction",
-        e,
+      logPermissionError("Invalid JSON in transaction", {
+        id: coValue.id,
         tx,
-        JSON.stringify(tx.changes, (k, v) =>
-          k === "changes" || k === "encryptedChanges"
-            ? v.slice(0, 20) + "..."
-            : v,
-        ),
-      );
+      });
       continue;
     }
 
@@ -458,8 +454,6 @@ function determineValidTransactionsForGroup(
 
     memberState[affectedMember] = change.value;
     validTransactions.push({ txID: { sessionID, txIndex }, tx });
-
-    // console.log("after", { memberState, validTransactions });
   }
 
   return { validTransactions, memberState };
@@ -473,7 +467,7 @@ function agentInAccountOrMemberInGroup(
     return groupAtTime.currentAgentID().match(
       (agentID) => agentID,
       (e) => {
-        console.error(
+        logger.error(
           "Error while determining current agent ID in valid transactions",
           e,
         );

package/src/storage/FileSystem.ts

@@ -87,26 +87,15 @@ export async function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
   const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
   await fs.append(file, headerBytes);
 
-  // console.log(
-  //     "full file",
-  //     yield* $(
-  //         fs.read(file as unknown as RH, 0, offset + headerBytes.length),
-  //     ),
-  // );
-
   const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
     3,
     "0",
   )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
     headerBytes.length
   }.jsonl`;
-  // console.log("renaming to" + filename);
   await fs.closeAndRename(file, filename);
 
   return filename;
-
-  // console.log("Wrote block", filename, blockHeader);
-  // console.log("IDs in block", blockHeader.map(e => e.id));
 }
 
 export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
@@ -120,6 +109,5 @@ export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
     ...chunk,
   };
   const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
-  console.log("writing to WAL", handle, id, bytes.length);
   return fs.append(handle, bytes);
 }

package/src/storage/index.ts

@@ -2,6 +2,7 @@ import { CoID, RawCoValue } from "../coValue.js";
 import { CoValueHeader, Transaction } from "../coValueCore.js";
 import { Signature } from "../crypto/crypto.js";
 import { RawCoID } from "../ids.js";
+import { logger } from "../logger.js";
 import { connectedPeers } from "../streamUtils.js";
 import {
   CoValueKnownState,
@@ -68,7 +69,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
     const processMessages = async () => {
       for await (const msg of fromLocalNode) {
-        console.log("Storage msg start", nMsg);
         try {
           if (msg === "Disconnected" || msg === "PingTimeout") {
             throw new Error("Unexpected Disconnected message");
@@ -83,32 +83,30 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
             await this.sendNewContent(msg.id, msg, undefined);
           }
         } catch (e) {
-          console.error(
-            new Error(
-              `Error reading from localNode, handling msg\n\n${JSON.stringify(
-                msg,
-                (k, v) =>
-                  k === "changes" || k === "encryptedChanges"
-                    ? v.slice(0, 20) + "..."
-                    : v,
-              )}`,
-              { cause: e },
-            ),
+          logger.error(
+            `Error reading from localNode, handling msg\n\n${JSON.stringify(
+              msg,
+              (k, v) =>
+                k === "changes" || k === "encryptedChanges"
+                  ? v.slice(0, 20) + "..."
+                  : v,
+            )}
+            Error: ${e instanceof Error ? e.message : "Unknown error"},
+            `,
           );
         }
-        console.log("Storage msg end", nMsg);
         nMsg++;
       }
     };
 
     processMessages().catch((e) =>
-      console.error("Error in processMessages in storage", e),
+      logger.error("Error in processMessages in storage", e),
     );
 
     setTimeout(
       () =>
         this.compact().catch((e) => {
-          console.error("Error while compacting", e);
+          logger.error("Error while compacting", e);
         }),
       20000,
     );
@@ -134,7 +132,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
           sessions: {},
           asDependencyOf,
         })
-        .catch((e) => console.error("Error while pushing known", e));
+        .catch((e) => logger.error("Error while pushing known", e));
 
       return;
     }
@@ -190,13 +188,13 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         ...ourKnown,
         asDependencyOf,
       })
-      .catch((e) => console.error("Error while pushing known", e));
+      .catch((e) => logger.error("Error while pushing known", e));
 
     for (const message of newContentMessages) {
       if (Object.keys(message.new).length === 0) continue;
       this.toLocalNode
         .push(message)
-        .catch((e) => console.error("Error while pushing new content", e));
+        .catch((e) => logger.error("Error while pushing new content", e));
     }
 
     this.coValues[id] = coValue;
@@ -232,14 +230,13 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
     if (!coValue) {
       if (newContent.header) {
-        // console.log("Creating in WAL", newContent.id);
         await this.withWAL((wal) =>
           writeToWal(wal, this.fs, newContent.id, newContentAsChunk),
         );
 
         this.coValues[newContent.id] = newContentAsChunk;
       } else {
-        console.warn("Incontiguous incoming update for " + newContent.id);
+        logger.warn("Incontiguous incoming update for " + newContent.id);
         return;
       }
     } else {
@@ -264,7 +261,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
           ),
         );
       } else {
-        // console.log("Appending to WAL", newContent.id);
         await this.withWAL((wal) =>
           writeToWal(wal, this.fs, newContent.id, newContentAsChunk),
         );
@@ -301,8 +297,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
       const { handle, size } = await this.getBlockHandle(blockFile, fs);
 
-      // console.log("Attempting to load", id, blockFile);
-
       if (!cachedHeader) {
         cachedHeader = {};
         const header = await readHeader(blockFile, handle, size, fs);
@@ -317,8 +311,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
       }
       const headerEntry = cachedHeader[id];
 
-      // console.log("Header entry", id, headerEntry);
-
       if (headerEntry) {
         const nextChunk = await readChunk(handle, headerEntry, fs);
         if (result) {
@@ -354,7 +346,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
     const coValues = new Map<RawCoID, CoValueChunk>();
 
-    console.log("Compacting WAL files", walFiles);
     if (walFiles.length === 0) return;
 
     const oldWal = this.currentWal;
@@ -411,8 +402,6 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
       return acc;
     }, 0);
 
-    console.log([...coValues.keys()], fileNames, highestBlockNumber);
-
     await writeBlock(coValues, MAX_N_LEVELS, highestBlockNumber + 1, this.fs);
 
     for (const walFile of walFiles) {
@@ -438,15 +427,11 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
       blockFilesByLevelInOrder[level]!.push(blockFile);
     }
 
-    console.log(blockFilesByLevelInOrder);
-
     for (let level = MAX_N_LEVELS; level > 0; level--) {
       const nBlocksDesired = Math.pow(2, level);
       const blocksInLevel = blockFilesByLevelInOrder[level];
 
       if (blocksInLevel && blocksInLevel.length > nBlocksDesired) {
-        console.log("Compacting blocks in level", level, blocksInLevel);
-
         const coValues = new Map<RawCoID, CoValueChunk>();
 
         for (const blockFile of blocksInLevel) {
@@ -517,7 +502,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
     setTimeout(
       () =>
         this.compact().catch((e) => {
-          console.error("Error while compacting", e);
+          logger.error("Error while compacting", e);
         }),
       5000,
     );

package/src/sync.ts

@@ -6,6 +6,7 @@ import { CoValueCore } from "./coValueCore.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
+import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 
 export type CoValueKnownState = {
@@ -22,6 +23,10 @@ export function emptyKnownState(id: RawCoID): CoValueKnownState {
   };
 }
 
+function getErrorMessage(e: unknown) {
+  return e instanceof Error ? e.message : "Unknown error";
+}
+
 export type SyncMessage =
   | LoadMessage
   | KnownStateMessage
@@ -150,7 +155,7 @@ export class SyncManager {
 
   async handleSyncMessage(msg: SyncMessage, peer: PeerState) {
     if (peer.erroredCoValues.has(msg.id)) {
-      console.error(
+      logger.warn(
         `Skipping message ${msg.action} on errored coValue ${msg.id} from peer ${peer.id}`,
       );
       return;
@@ -182,7 +187,7 @@ export class SyncManager {
 
     if (entry.state.type !== "available") {
       entry.loadFromPeers([peer]).catch((e: unknown) => {
-        console.error("Error sending load", e);
+        logger.error("Error sending load: " + getErrorMessage(e));
       });
       return;
     }
@@ -199,7 +204,7 @@ export class SyncManager {
         action: "load",
         ...coValue.knownState(),
       }).catch((e: unknown) => {
-        console.error("Error sending load", e);
+        logger.error("Error sending load: " + getErrorMessage(e));
       });
     }
   }
@@ -229,7 +234,7 @@ export class SyncManager {
         asDependencyOf,
         ...coValue.knownState(),
       }).catch((e: unknown) => {
-        console.error("Error sending known state", e);
+        logger.error("Error sending known state: " + getErrorMessage(e));
       });
 
       peer.toldKnownState.add(id);
@@ -256,15 +261,8 @@ export class SyncManager {
       const sendPieces = async () => {
         let lastYield = performance.now();
         for (const [_i, piece] of newContentPieces.entries()) {
-          // console.log(
-          //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${
-          //         newContentPieces.length
-          //     } header: ${!!piece.header}`,
-          //     // Object.values(piece.new).map((s) => s.newTransactions)
-          // );
-
           this.trySendToPeer(peer, piece).catch((e: unknown) => {
-            console.error("Error sending content piece", e);
+            logger.error("Error sending content piece: " + getErrorMessage(e));
           });
 
           if (performance.now() - lastYield > 10) {
@@ -277,7 +275,7 @@ export class SyncManager {
       };
 
       sendPieces().catch((e) => {
-        console.error("Error sending new content piece, retrying", e);
+        logger.error("Error sending new content piece, retrying", e);
         peer.optimisticKnownStates.dispatch({
           type: "SET",
           id,
@@ -337,7 +335,10 @@ export class SyncManager {
           return;
         }
         if (msg === "PingTimeout") {
-          console.error("Ping timeout from peer", peer.id);
+          logger.error("Ping timeout from peer", {
+            peerId: peer.id,
+            peerRole: peer.role,
+          });
           return;
         }
         try {
@@ -360,13 +361,22 @@ export class SyncManager {
     processMessages()
       .then(() => {
         if (peer.crashOnClose) {
-          console.error("Unexepcted close from peer", peer.id);
+          logger.error("Unexepcted close from peer", {
+            peerId: peer.id,
+            peerRole: peer.role,
+          });
           this.local.crashed = new Error("Unexpected close from peer");
           throw new Error("Unexpected close from peer");
         }
       })
       .catch((e) => {
-        console.error("Error processing messages from peer", peer.id, e);
+        logger.error(
+          "Error processing messages from peer: " + getErrorMessage(e),
+          {
+            peerId: peer.id,
+            peerRole: peer.role,
+          },
+        );
         if (peer.crashOnClose) {
           this.local.crashed = e;
           throw new Error(e);
@@ -406,13 +416,13 @@ export class SyncManager {
         // where we can get informations about the coValue
         if (msg.header || Object.keys(msg.sessions).length > 0) {
           entry.loadFromPeers([peer]).catch((e) => {
-            console.error("Error loading coValue in handleLoad", e);
+            logger.error("Error loading coValue in handleLoad", e);
           });
         }
         return;
       } else {
         this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
-          console.error("Error loading coValue in handleLoad", e);
+          logger.error("Error loading coValue in handleLoad", e);
         });
       }
     }
@@ -439,7 +449,7 @@ export class SyncManager {
               header: false,
               sessions: {},
             }).catch((e) => {
-              console.error("Error sending known state back", e);
+              logger.error("Error sending known state back", e);
             });
 
             return;
@@ -449,7 +459,7 @@ export class SyncManager {
           await this.sendNewContentIncludingDependencies(msg.id, peer);
         })
         .catch((e) => {
-          console.error("Error loading coValue in handleLoad loading state", e);
+          logger.error("Error loading coValue in handleLoad loading state", e);
         });
     }
 
@@ -484,7 +494,7 @@ export class SyncManager {
               peer.role === "storage" ? undefined : peer.id,
             )
             .catch((e) => {
-              console.error(
+              logger.error(
                 `Error loading coValue ${msg.id} to create loading state, as dependency of ${msg.asDependencyOf}`,
                 e,
               );
@@ -521,7 +531,7 @@ export class SyncManager {
 
     if (entry.state.type !== "available") {
       if (!msg.header) {
-        console.error("Expected header to be sent in first message");
+        logger.error("Expected header to be sent in first message");
         return;
       }
 
@@ -584,7 +594,7 @@ export class SyncManager {
               : t.changes.length,
           )
           .reduce((a, b) => a + b, 0);
-        console.log(
+        logger.debug(
           `Adding incoming transactions took ${(after - before).toFixed(
             2,
           )}ms for ${totalTxLength} bytes = bandwidth: ${(
@@ -602,17 +612,11 @@ export class SyncManager {
       // );
 
       if (result.isErr()) {
-        console.error(
-          "Failed to add transactions from",
-          peer.id,
-          result.error,
-          msg.id,
-          newTransactions.length + " new transactions",
-          "after: " + newContentForSession.after,
-          "our last known tx idx initially: " + ourKnownTxIdx,
-          "our last known tx idx now: " +
-            coValue.sessionLogs.get(sessionID)?.transactions.length,
-        );
+        logger.error("Failed to add transactions: " + result.error.type, {
+          peerId: peer.id,
+          peerRole: peer.role,
+          id: msg.id,
+        });
         peer.erroredCoValues.set(msg.id, result.error);
         continue;
       }
@@ -633,7 +637,13 @@ export class SyncManager {
         isCorrection: true,
         ...coValue.knownState(),
       }).catch((e) => {
-        console.error("Error sending known state correction", e);
+        logger.error(
+          "Error sending known state correction: " + getErrorMessage(e),
+          {
+            peerId: peer.id,
+            peerRole: peer.role,
+          },
+        );
       });
     } else {
       /**
@@ -647,7 +657,10 @@ export class SyncManager {
         action: "known",
         ...coValue.knownState(),
       }).catch((e: unknown) => {
-        console.error("Error sending known state", e);
+        logger.error("Error sending known state: " + getErrorMessage(e), {
+          peerId: peer.id,
+          peerRole: peer.role,
+        });
       });
     }
 
@@ -681,9 +694,6 @@ export class SyncManager {
       const done = new Promise<void>((resolve) => {
         queueMicrotask(async () => {
           delete this.requestedSyncs[coValue.id];
-          // if (entry.nRequestsThisTick >= 2) {
-          //     console.log("Syncing", coValue.id, "for", entry.nRequestsThisTick, "requests");
-          // }
           await this.actuallySyncCoValue(coValue);
           resolve();
         });