cojson 0.9.10 → 0.9.11

package/src/PriorityBasedMessageQueue.ts

@@ -18,11 +18,12 @@ function promiseWithResolvers<R>() {
   };
 }
 
-export type QueueEntry = {
-  msg: SyncMessage;
+export type QueueEntry<V> = {
+  msg: V;
   promise: Promise<void>;
   resolve: () => void;
   reject: (_: unknown) => void;
+  next: QueueEntry<V> | undefined;
 };
 
 /**
@@ -33,10 +34,68 @@ type Tuple<T, N extends number, A extends unknown[] = []> = A extends {
 }
   ? A
   : Tuple<T, N, [...A, T]>;
-type QueueTuple = Tuple<QueueEntry[], 8>;
+type QueueTuple = Tuple<Queue<SyncMessage>, 8>;
+
+class Queue<V> {
+  head: QueueEntry<V> | undefined = undefined;
+  tail: QueueEntry<V> | undefined = undefined;
+
+  push(msg: V) {
+    const { promise, resolve, reject } = promiseWithResolvers<void>();
+    const entry: QueueEntry<V> = {
+      msg,
+      promise,
+      resolve,
+      reject,
+      next: undefined,
+    };
+
+    if (this.head === undefined) {
+      this.head = entry;
+    } else {
+      if (this.tail === undefined) {
+        throw new Error("Tail is null but head is not");
+      }
+
+      this.tail.next = entry;
+    }
+
+    this.tail = entry;
+
+    return entry;
+  }
+
+  pull() {
+    const entry = this.head;
+
+    if (entry) {
+      this.head = entry.next;
+    }
+
+    if (this.head === undefined) {
+      this.tail = undefined;
+    }
+
+    return entry;
+  }
+
+  isNonEmpty() {
+    return this.head !== undefined;
+  }
+}
 
 export class PriorityBasedMessageQueue {
-  private queues: QueueTuple = [[], [], [], [], [], [], [], []];
+  private queues: QueueTuple = [
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+  ];
+
   queueSizeCounter = metrics
     .getMeter("cojson")
     .createUpDownCounter("jazz.messagequeue.size", {
@@ -52,22 +111,19 @@ export class PriorityBasedMessageQueue {
   constructor(private defaultPriority: CoValuePriority) {}
 
   public push(msg: SyncMessage) {
-    const { promise, resolve, reject } = promiseWithResolvers<void>();
-    const entry: QueueEntry = { msg, promise, resolve, reject };
-
     const priority = "priority" in msg ? msg.priority : this.defaultPriority;
 
-    this.getQueue(priority).push(entry);
+    const entry = this.getQueue(priority).push(msg);
 
     this.queueSizeCounter.add(1, {
       priority,
     });
 
-    return promise;
+    return entry.promise;
   }
 
   public pull() {
-    const priority = this.queues.findIndex((queue) => queue.length > 0);
+    const priority = this.queues.findIndex((queue) => queue.isNonEmpty());
 
     if (priority === -1) {
       return;
@@ -77,6 +133,6 @@ export class PriorityBasedMessageQueue {
       priority,
     });
 
-    return this.queues[priority]?.shift();
+    return this.queues[priority]?.pull();
   }
 }

package/src/base64url.ts

@@ -43,12 +43,7 @@ export function bytesToBase64url(bytes: Uint8Array) {
   let base64 = decoder.decode(new Uint8Array(encoded.buffer, 0, n));
   if (k === 1) base64 += "==";
   if (k === 2) base64 += "=";
-  // const after = performance.now();
-  // console.log(
-  //     "bytesToBase64url bandwidth in MB/s for length",
-  //     (1000 * bytes.length / (after - before)) / (1024 * 1024),
-  //     bytes.length
-  // );
+
   return base64;
 }
 

package/src/coValueCore.ts

@@ -24,6 +24,7 @@ import {
 import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
 import { JsonObject, JsonValue } from "./jsonValue.js";
 import { LocalNode, ResolveAccountAgentError } from "./localNode.js";
+import { logger } from "./logger.js";
 import {
   PermissionsDef as RulesetDef,
   determineValidTransactions,
@@ -209,16 +210,10 @@ export class CoValueCore {
       .andThen((agent) => {
         const signerID = this.crypto.getAgentSignerID(agent);
 
-        // const beforeHash = performance.now();
         const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
           sessionID,
           newTransactions,
         );
-        // const afterHash = performance.now();
-        // console.log(
-        //     "Hashing took",
-        //     afterHash - beforeHash
-        // );
 
         if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
           return err({
@@ -391,7 +386,6 @@ export class CoValueCore {
       streamingHash.update(transaction);
       const after = performance.now();
       if (after - before > 1) {
-        // console.log("Hashing blocked for", after - before);
         await new Promise((resolve) => setTimeout(resolve, 0));
         before = performance.now();
       }
@@ -540,7 +534,7 @@ export class CoValueCore {
       }
 
       if (!decryptedChanges) {
-        console.error("Failed to decrypt transaction despite having key");
+        logger.error("Failed to decrypt transaction despite having key");
         continue;
       }
 
@@ -685,7 +679,7 @@ export class CoValueCore {
           if (secret) {
             return secret as KeySecret;
           } else {
-            console.error(
+            logger.warn(
               `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`,
             );
           }
@@ -725,7 +719,7 @@ export class CoValueCore {
               if (secret) {
                 return secret as KeySecret;
               } else {
-                console.error(
+                logger.warn(
                   `Encrypting parent ${parentKey.id} key didn't decrypt ${keyID}`,
                 );
               }

package/src/coValueState.ts

@@ -1,6 +1,7 @@
 import { PeerState } from "./PeerState.js";
 import { CoValueCore } from "./coValueCore.js";
 import { RawCoID } from "./ids.js";
+import { logger } from "./logger.js";
 import { PeerID } from "./sync.js";
 
 export const CO_VALUE_LOADING_MAX_RETRIES = 5;
@@ -282,7 +283,7 @@ async function loadCoValueFromPeers(
           ...coValueEntry.state.coValue.knownState(),
         })
         .catch((err) => {
-          console.error(`Failed to push load message to peer ${peer.id}`, err);
+          logger.warn(`Failed to push load message to peer ${peer.id}`, err);
         });
     } else {
       /**
@@ -296,14 +297,14 @@ async function loadCoValueFromPeers(
           sessions: {},
         })
         .catch((err) => {
-          console.error(`Failed to push load message to peer ${peer.id}`, err);
+          logger.warn(`Failed to push load message to peer ${peer.id}`, err);
         });
     }
 
     if (coValueEntry.state.type === "loading") {
       const timeout = setTimeout(() => {
         if (coValueEntry.state.type === "loading") {
-          console.error("Failed to load coValue from peer", peer.id);
+          logger.warn("Failed to load coValue from peer", peer.id);
           coValueEntry.dispatch({
             type: "not-found-in-peer",
             peerId: peer.id,
@@ -356,7 +357,7 @@ function sleep(ms: number) {
 function getPeersWithoutErrors(peers: PeerState[], coValueId: RawCoID) {
   return peers.filter((p) => {
     if (p.erroredCoValues.has(coValueId)) {
-      console.error(
+      logger.warn(
         `Skipping load on errored coValue ${coValueId} from peer ${p.id}`,
       );
       return false;

package/src/coValues/account.ts

@@ -16,6 +16,7 @@ import {
 import { AgentID } from "../ids.js";
 import { JsonObject } from "../jsonValue.js";
 import { LocalNode } from "../localNode.js";
+import { logger } from "../logger.js";
 import type { AccountRole } from "../permissions.js";
 import { RawCoMap } from "./coMap.js";
 import { InviteSecret, RawGroup } from "./group.js";
@@ -59,7 +60,7 @@ export class RawAccount<
       );
 
     if (agents.length !== 1) {
-      console.warn("Account has " + agents.length + " agents", this.id);
+      logger.warn("Account has " + agents.length + " agents", this.id);
     }
 
     this._cachedCurrentAgentID = agents[0];

package/src/coValues/coList.ts

@@ -133,10 +133,6 @@ export class RawCoListView<
                   change.before.txIndex
                 ]?.[change.before.changeIdx];
               if (!beforeEntry) {
-                // console.error(
-                //     "Insertion before missing op " +
-                //         change.before
-                // );
                 continue;
               }
               beforeEntry.predecessors.splice(0, 0, {
@@ -156,9 +152,6 @@ export class RawCoListView<
                   change.after.txIndex
                 ]?.[change.after.changeIdx];
               if (!afterEntry) {
-                // console.error(
-                //     "Insertion after missing op " + change.after
-                // );
                 continue;
               }
               afterEntry.successors.push({

package/src/coValues/coStream.ts

@@ -3,6 +3,7 @@ import { CoID, RawCoValue } from "../coValue.js";
 import { CoValueCore } from "../coValueCore.js";
 import { AgentID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import { CoValueKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isAccountID } from "../typeUtils/isAccountID.js";
@@ -309,7 +310,7 @@ export class RawBinaryCoStreamView<
     const start = items[0];
 
     if (start?.type !== "start") {
-      console.error("Invalid binary stream start", start);
+      logger.error("Invalid binary stream start", start);
       return;
     }
 
@@ -328,7 +329,7 @@ export class RawBinaryCoStreamView<
       }
 
       if (item.type !== "chunk") {
-        console.error("Invalid binary stream chunk", item);
+        logger.error("Invalid binary stream chunk", item);
         return undefined;
       }
 
@@ -382,7 +383,6 @@ export class RawBinaryCoStream<
     chunk: Uint8Array,
     privacy: "private" | "trusting" = "private",
   ): void {
-    // const before = performance.now();
     this.push(
       {
         type: "chunk",
@@ -391,11 +391,6 @@ export class RawBinaryCoStream<
       privacy,
       false,
     );
-    // const after = performance.now();
-    // console.log(
-    //     "pushBinaryStreamChunk bandwidth in MB/s",
-    //     (1000 * chunk.length) / (after - before) / (1024 * 1024)
-    // );
   }
 
   endBinaryStream(privacy: "private" | "trusting" = "private") {

package/src/coValues/group.ts

@@ -13,6 +13,7 @@ import {
   isParentGroupReference,
 } from "../ids.js";
 import { JsonObject } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import { AccountRole, Role } from "../permissions.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
 import {
@@ -153,7 +154,7 @@ export class RawGroup<
         child.state.type === "unavailable"
       ) {
         child.loadFromPeers(peers).catch(() => {
-          console.error(`Failed to load child group ${id}`);
+          logger.error(`Failed to load child group ${id}`);
         });
       }
 
@@ -321,7 +322,7 @@ export class RawGroup<
         const secret = this.core.getReadKey(keyID);
 
         if (!secret) {
-          console.error("Can't find key", keyID);
+          logger.error("Can't find key", keyID);
           continue;
         }
 

package/src/crypto/PureJSCrypto.ts

@@ -7,6 +7,7 @@ import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
 import { RawCoID, TransactionID } from "../ids.js";
 import { Stringified, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -192,7 +193,7 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
     try {
       return JSON.parse(textDecoder.decode(plaintext));
     } catch (e) {
-      console.error("Failed to decrypt/parse sealed message", e);
+      logger.error("Failed to decrypt/parse sealed message", e);
       return undefined;
     }
   }

package/src/crypto/WasmCrypto.ts

@@ -15,6 +15,7 @@ import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
 import { RawCoID, TransactionID } from "../ids.js";
 import { Stringified, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -240,7 +241,7 @@ export class WasmCrypto extends CryptoProvider<Uint8Array> {
     try {
       return JSON.parse(textDecoder.decode(plaintext));
     } catch (e) {
-      console.error("Failed to decrypt/parse sealed message", e);
+      logger.error("Failed to decrypt/parse sealed message", e);
       return undefined;
     }
   }

package/src/crypto/crypto.ts

@@ -4,6 +4,7 @@ import { AgentID, RawCoID, TransactionID } from "../ids.js";
 import { SessionID } from "../ids.js";
 import { Stringified, parseJSON, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 
 export type SignerSecret = `signerSecret_z${string}`;
 export type SignerID = `signer_z${string}`;
@@ -159,7 +160,7 @@ export abstract class CryptoProvider<Blake3State = any> {
     try {
       return parseJSON(this.decryptRaw(encrypted, keySecret, nOnceMaterial));
     } catch (e) {
-      console.error("Decryption error", e);
+      logger.error("Decryption error", e);
       return undefined;
     }
   }
@@ -305,10 +306,7 @@ export class StreamingHash {
 
   update(value: JsonValue): Uint8Array {
     const encoded = textEncoder.encode(stableStringify(value));
-    // const before = performance.now();
     this.state = this.crypto.blake3IncrementalUpdate(this.state, encoded);
-    // const after = performance.now();
-    // console.log(`Hashing throughput in MB/s`, 1000 * (encoded.length / (after - before)) / (1024 * 1024));
     return encoded;
   }
 

package/src/exports.ts

@@ -74,6 +74,7 @@ import {
 
 type Value = JsonValue | AnyRawCoValue;
 
+import { logger } from "./logger.js";
 import { getPriorityFromHeader } from "./priority.js";
 import { FileSystem } from "./storage/FileSystem.js";
 import { BlockFilename, LSMStorage, WalFilename } from "./storage/index.js";
@@ -141,6 +142,7 @@ export {
   emptyKnownState,
   RawCoPlainText,
   stringifyOpID,
+  logger,
 };
 
 export type {

package/src/localNode.ts

@@ -27,6 +27,7 @@ import {
 } from "./coValues/group.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
+import { logger } from "./logger.js";
 import { Peer, PeerID, SyncManager } from "./sync.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
@@ -230,7 +231,7 @@ export class LocalNode {
 
       return node;
     } catch (e) {
-      console.error("Error withLoadedAccount", e);
+      logger.error("Error withLoadedAccount", e);
       throw e;
     }
   }
@@ -269,7 +270,7 @@ export class LocalNode {
         this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
 
       await entry.loadFromPeers(peers).catch((e) => {
-        console.error("Error loading from peers", id, e);
+        logger.error("Error loading from peers", id, e);
       });
     }
 
@@ -311,8 +312,6 @@ export class LocalNode {
     let stopped = false;
     let unsubscribe!: () => void;
 
-    // console.log("Subscribing to " + id);
-
     this.load(id)
       .then((coValue) => {
         if (stopped) {
@@ -325,11 +324,10 @@ export class LocalNode {
         unsubscribe = coValue.subscribe(callback);
       })
       .catch((e) => {
-        console.error("Error subscribing to ", id, e);
+        logger.error("Error subscribing to ", id, e);
       });
 
     return () => {
-      console.log("Unsubscribing from " + id);
       stopped = true;
       unsubscribe?.();
     };
@@ -390,9 +388,7 @@ export class LocalNode {
       (existingRole === "reader" && inviteRole === "readerInvite") ||
       (existingRole && inviteRole === "writeOnlyInvite")
     ) {
-      console.debug(
-        "Not accepting invite that would replace or downgrade role",
-      );
+      logger.debug("Not accepting invite that would replace or downgrade role");
       return;
     }
 

package/src/logger.ts

@@ -0,0 +1,78 @@
+export enum LogLevel {
+  DEBUG = 0,
+  INFO = 1,
+  WARN = 2,
+  ERROR = 3,
+  NONE = 4,
+}
+
+export interface LogSystem {
+  debug(message: string, ...args: any[]): void;
+  info(message: string, ...args: any[]): void;
+  warn(message: string, ...args: any[]): void;
+  error(message: string, ...args: any[]): void;
+}
+
+// Default console-based logging system
+export class ConsoleLogSystem implements LogSystem {
+  debug(message: string, ...args: any[]) {
+    console.debug(message, ...args);
+  }
+  info(message: string, ...args: any[]) {
+    console.info(message, ...args);
+  }
+  warn(message: string, ...args: any[]) {
+    console.warn(message, ...args);
+  }
+  error(message: string, ...args: any[]) {
+    console.error(message, ...args);
+  }
+}
+
+export class Logger {
+  private level: LogLevel;
+  private logSystem: LogSystem;
+
+  constructor(
+    level: LogLevel = LogLevel.INFO,
+    logSystem: LogSystem = new ConsoleLogSystem(),
+  ) {
+    this.level = level;
+    this.logSystem = logSystem;
+  }
+
+  setLevel(level: LogLevel) {
+    this.level = level;
+  }
+
+  setLogSystem(logSystem: LogSystem) {
+    this.logSystem = logSystem;
+  }
+
+  debug(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.DEBUG) {
+      this.logSystem.debug(message, ...args);
+    }
+  }
+
+  info(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.INFO) {
+      this.logSystem.info(message, ...args);
+    }
+  }
+
+  warn(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.WARN) {
+      this.logSystem.warn(message, ...args);
+    }
+  }
+
+  error(message: string, ...args: any[]) {
+    if (this.level <= LogLevel.ERROR) {
+      this.logSystem.error(message, ...args);
+    }
+  }
+}
+
+// Create default logger instance
+export const logger = new Logger();

package/src/permissions.ts

@@ -14,6 +14,7 @@ import {
 } from "./ids.js";
 import { parseJSON } from "./jsonStringify.js";
 import { JsonValue } from "./jsonValue.js";
+import { logger } from "./logger.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
@@ -46,7 +47,7 @@ function logPermissionError(...args: unknown[]) {
     return;
   }
 
-  console.warn(...args);
+  logger.warn("Permission error", ...args);
 }
 
 export function determineValidTransactions(
@@ -204,7 +205,6 @@ function determineValidTransactionsForGroup(
   const writeKeys = new Set<string>();
 
   for (const { sessionID, txIndex, tx } of allTransactionsSorted) {
-    // console.log("before", { memberState, validTransactions });
     const transactor = accountOrAgentIDfromSessionID(sessionID);
 
     if (tx.privacy === "private") {
@@ -458,8 +458,6 @@ function determineValidTransactionsForGroup(
 
     memberState[affectedMember] = change.value;
     validTransactions.push({ txID: { sessionID, txIndex }, tx });
-
-    // console.log("after", { memberState, validTransactions });
   }
 
   return { validTransactions, memberState };
@@ -473,7 +471,7 @@ function agentInAccountOrMemberInGroup(
     return groupAtTime.currentAgentID().match(
       (agentID) => agentID,
       (e) => {
-        console.error(
+        logger.error(
           "Error while determining current agent ID in valid transactions",
           e,
         );

package/src/storage/FileSystem.ts

@@ -87,26 +87,15 @@ export async function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
   const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
   await fs.append(file, headerBytes);
 
-  // console.log(
-  //     "full file",
-  //     yield* $(
-  //         fs.read(file as unknown as RH, 0, offset + headerBytes.length),
-  //     ),
-  // );
-
   const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
     3,
     "0",
   )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
     headerBytes.length
   }.jsonl`;
-  // console.log("renaming to" + filename);
   await fs.closeAndRename(file, filename);
 
   return filename;
-
-  // console.log("Wrote block", filename, blockHeader);
-  // console.log("IDs in block", blockHeader.map(e => e.id));
 }
 
 export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
@@ -120,6 +109,5 @@ export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
     ...chunk,
   };
   const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
-  console.log("writing to WAL", handle, id, bytes.length);
   return fs.append(handle, bytes);
 }