cojson 0.9.9 → 0.9.11

package/src/PeerState.ts

@@ -5,6 +5,7 @@ import {
 } from "./PriorityBasedMessageQueue.js";
 import { TryAddTransactionsError } from "./coValueCore.js";
 import { RawCoID } from "./ids.js";
+import { logger } from "./logger.js";
 import { CO_VALUE_PRIORITY } from "./priority.js";
 import { Peer, SyncMessage } from "./sync.js";
 
@@ -92,7 +93,7 @@ export class PeerState {
 
     this.processing = true;
 
-    let entry: QueueEntry | undefined;
+    let entry: QueueEntry<SyncMessage> | undefined;
     while ((entry = this.queue.pull())) {
       // Awaiting the push to send one message at a time
       // This way when the peer is "under pressure" we can enqueue all
@@ -129,7 +130,7 @@ export class PeerState {
   }
 
   private closeQueue() {
-    let entry: QueueEntry | undefined;
+    let entry: QueueEntry<SyncMessage> | undefined;
     while ((entry = this.queue.pull())) {
       // Using resolve here to avoid unnecessary noise in the logs
       entry.resolve();
@@ -137,7 +138,7 @@ export class PeerState {
   }
 
   gracefulShutdown() {
-    console.debug("Gracefully closing", this.id);
+    logger.debug("Gracefully closing", this.id);
     this.closeQueue();
     this.peer.outgoing.close();
     this.closed = true;

package/src/PriorityBasedMessageQueue.ts

@@ -18,11 +18,12 @@ function promiseWithResolvers<R>() {
   };
 }
 
-export type QueueEntry = {
-  msg: SyncMessage;
+export type QueueEntry<V> = {
+  msg: V;
   promise: Promise<void>;
   resolve: () => void;
   reject: (_: unknown) => void;
+  next: QueueEntry<V> | undefined;
 };
 
 /**
@@ -33,10 +34,68 @@ type Tuple<T, N extends number, A extends unknown[] = []> = A extends {
 }
   ? A
   : Tuple<T, N, [...A, T]>;
-type QueueTuple = Tuple<QueueEntry[], 8>;
+type QueueTuple = Tuple<Queue<SyncMessage>, 8>;
+
+class Queue<V> {
+  head: QueueEntry<V> | undefined = undefined;
+  tail: QueueEntry<V> | undefined = undefined;
+
+  push(msg: V) {
+    const { promise, resolve, reject } = promiseWithResolvers<void>();
+    const entry: QueueEntry<V> = {
+      msg,
+      promise,
+      resolve,
+      reject,
+      next: undefined,
+    };
+
+    if (this.head === undefined) {
+      this.head = entry;
+    } else {
+      if (this.tail === undefined) {
+        throw new Error("Tail is null but head is not");
+      }
+
+      this.tail.next = entry;
+    }
+
+    this.tail = entry;
+
+    return entry;
+  }
+
+  pull() {
+    const entry = this.head;
+
+    if (entry) {
+      this.head = entry.next;
+    }
+
+    if (this.head === undefined) {
+      this.tail = undefined;
+    }
+
+    return entry;
+  }
+
+  isNonEmpty() {
+    return this.head !== undefined;
+  }
+}
 
 export class PriorityBasedMessageQueue {
-  private queues: QueueTuple = [[], [], [], [], [], [], [], []];
+  private queues: QueueTuple = [
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+    new Queue(),
+  ];
+
   queueSizeCounter = metrics
     .getMeter("cojson")
     .createUpDownCounter("jazz.messagequeue.size", {
@@ -52,22 +111,19 @@ export class PriorityBasedMessageQueue {
   constructor(private defaultPriority: CoValuePriority) {}
 
   public push(msg: SyncMessage) {
-    const { promise, resolve, reject } = promiseWithResolvers<void>();
-    const entry: QueueEntry = { msg, promise, resolve, reject };
-
     const priority = "priority" in msg ? msg.priority : this.defaultPriority;
 
-    this.getQueue(priority).push(entry);
+    const entry = this.getQueue(priority).push(msg);
 
     this.queueSizeCounter.add(1, {
       priority,
     });
 
-    return promise;
+    return entry.promise;
   }
 
   public pull() {
-    const priority = this.queues.findIndex((queue) => queue.length > 0);
+    const priority = this.queues.findIndex((queue) => queue.isNonEmpty());
 
     if (priority === -1) {
       return;
@@ -77,6 +133,6 @@ export class PriorityBasedMessageQueue {
       priority,
     });
 
-    return this.queues[priority]?.shift();
+    return this.queues[priority]?.pull();
   }
 }

package/src/base64url.ts

@@ -43,12 +43,7 @@ export function bytesToBase64url(bytes: Uint8Array) {
   let base64 = decoder.decode(new Uint8Array(encoded.buffer, 0, n));
   if (k === 1) base64 += "==";
   if (k === 2) base64 += "=";
-  // const after = performance.now();
-  // console.log(
-  //     "bytesToBase64url bandwidth in MB/s for length",
-  //     (1000 * bytes.length / (after - before)) / (1024 * 1024),
-  //     bytes.length
-  // );
+
   return base64;
 }
 

package/src/coValue.ts

@@ -35,6 +35,43 @@ export interface RawCoValue {
   subscribe(listener: (coValue: this) => void): () => void;
 }
 
+export class RawUnknownCoValue implements RawCoValue {
+  id: CoID<this>;
+  core: CoValueCore;
+
+  constructor(core: CoValueCore) {
+    this.id = core.id as CoID<this>;
+    this.core = core;
+  }
+
+  get type() {
+    return this.core.header.type;
+  }
+
+  get headerMeta() {
+    return this.core.header.meta as JsonObject;
+  }
+
+  /** @category 6. Meta */
+  get group(): RawGroup {
+    return this.core.getGroup();
+  }
+
+  toJSON() {
+    return {};
+  }
+
+  atTime() {
+    return this;
+  }
+
+  subscribe(listener: (value: this) => void): () => void {
+    return this.core.subscribe((content) => {
+      listener(content as this);
+    });
+  }
+}
+
 export type AnyRawCoValue =
   | RawCoMap
   | RawGroup

package/src/coValueCore.ts

@@ -24,6 +24,7 @@ import {
 import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
 import { JsonObject, JsonValue } from "./jsonValue.js";
 import { LocalNode, ResolveAccountAgentError } from "./localNode.js";
+import { logger } from "./logger.js";
 import {
   PermissionsDef as RulesetDef,
   determineValidTransactions,
@@ -126,10 +127,7 @@ export class CoValueCore {
         .expectCoValueLoaded(header.ruleset.group)
         .subscribe((_groupUpdate) => {
           this._cachedContent = undefined;
-          const newContent = this.getCurrentContent();
-          for (const listener of this.listeners) {
-            listener(newContent);
-          }
+          this.notifyUpdate("immediate");
         });
     }
   }
@@ -212,16 +210,10 @@ export class CoValueCore {
       .andThen((agent) => {
         const signerID = this.crypto.getAgentSignerID(agent);
 
-        // const beforeHash = performance.now();
         const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
           sessionID,
           newTransactions,
         );
-        // const afterHash = performance.now();
-        // console.log(
-        //     "Hashing took",
-        //     afterHash - beforeHash
-        // );
 
         if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
           return err({
@@ -244,11 +236,6 @@ export class CoValueCore {
             signerID,
           } satisfies InvalidSignatureError);
         }
-        // const afterVerify = performance.now();
-        // console.log(
-        //     "Verify took",
-        //     afterVerify - beforeVerify
-        // );
 
         this.doAddTransactions(
           sessionID,
@@ -263,138 +250,6 @@ export class CoValueCore {
       });
   }
 
-  /*tryAddTransactionsAsync(
-        sessionID: SessionID,
-        newTransactions: Transaction[],
-        givenExpectedNewHash: Hash | undefined,
-        newSignature: Signature,
-    ): ResultAsync<true, TryAddTransactionsError> {
-        const currentAsyncAddTransaction = this._currentAsyncAddTransaction;
-        let maybeAwaitPrevious:
-            | ResultAsync<void, TryAddTransactionsError>
-            | undefined;
-        let thisDone = () => {};
-
-        if (currentAsyncAddTransaction) {
-            // eslint-disable-next-line neverthrow/must-use-result
-            maybeAwaitPrevious = ResultAsync.fromSafePromise(
-                currentAsyncAddTransaction,
-            );
-        } else {
-            // eslint-disable-next-line neverthrow/must-use-result
-            maybeAwaitPrevious = ResultAsync.fromSafePromise(Promise.resolve());
-            this._currentAsyncAddTransaction = new Promise((resolve) => {
-                thisDone = resolve;
-            });
-        }
-
-        return maybeAwaitPrevious
-            .andThen((_previousDone) =>
-                this.node
-                    .resolveAccountAgentAsync(
-                        accountOrAgentIDfromSessionID(sessionID),
-                        "Expected to know signer of transaction",
-                    )
-                    .andThen((agent) => {
-                        const signerID = this.crypto.getAgentSignerID(agent);
-
-                        const nTxBefore =
-                            this.sessionLogs.get(sessionID)?.transactions
-                                .length ?? 0;
-
-                        // const beforeHash = performance.now();
-                        return ResultAsync.fromSafePromise(
-                            this.expectedNewHashAfterAsync(
-                                sessionID,
-                                newTransactions,
-                            ),
-                        ).andThen(({ expectedNewHash, newStreamingHash }) => {
-                            // const afterHash = performance.now();
-                            // console.log(
-                            //     "Hashing took",
-                            //     afterHash - beforeHash
-                            // );
-
-                            const nTxAfter =
-                                this.sessionLogs.get(sessionID)?.transactions
-                                    .length ?? 0;
-
-                            if (nTxAfter !== nTxBefore) {
-                                const newTransactionLengthBefore =
-                                    newTransactions.length;
-                                newTransactions = newTransactions.slice(
-                                    nTxAfter - nTxBefore,
-                                );
-                                console.warn(
-                                    "Transactions changed while async hashing",
-                                    {
-                                        nTxBefore,
-                                        nTxAfter,
-                                        newTransactionLengthBefore,
-                                        remainingNewTransactions:
-                                            newTransactions.length,
-                                    },
-                                );
-                            }
-
-                            if (
-                                givenExpectedNewHash &&
-                                givenExpectedNewHash !== expectedNewHash
-                            ) {
-                                return err({
-                                    type: "InvalidHash",
-                                    id: this.id,
-                                    expectedNewHash,
-                                    givenExpectedNewHash,
-                                } satisfies InvalidHashError);
-                            }
-
-                            performance.mark("verifyStart" + this.id);
-                            if (
-                                !this.crypto.verify(
-                                    newSignature,
-                                    expectedNewHash,
-                                    signerID,
-                                )
-                            ) {
-                                return err({
-                                    type: "InvalidSignature",
-                                    id: this.id,
-                                    newSignature,
-                                    sessionID,
-                                    signerID,
-                                } satisfies InvalidSignatureError);
-                            }
-                            performance.mark("verifyEnd" + this.id);
-                            performance.measure(
-                                "verify" + this.id,
-                                "verifyStart" + this.id,
-                                "verifyEnd" + this.id,
-                            );
-
-                            this.doAddTransactions(
-                                sessionID,
-                                newTransactions,
-                                newSignature,
-                                expectedNewHash,
-                                newStreamingHash,
-                                "deferred",
-                            );
-
-                            return ok(true as const);
-                        });
-                    }),
-            )
-            .map((trueResult) => {
-                thisDone();
-                return trueResult;
-            })
-            .mapErr((err) => {
-                thisDone();
-                return err;
-            });
-    }*/
-
   private doAddTransactions(
     sessionID: SessionID,
     newTransactions: Transaction[],
@@ -432,12 +287,6 @@ export class CoValueCore {
       );
 
     if (sizeOfTxsSinceLastInbetweenSignature > MAX_RECOMMENDED_TX_SIZE) {
-      // console.log(
-      //     "Saving inbetween signature for tx ",
-      //     sessionID,
-      //     transactions.length - 1,
-      //     sizeOfTxsSinceLastInbetweenSignature
-      // );
       signatureAfter[transactions.length - 1] = newSignature;
     }
 
@@ -463,34 +312,40 @@ export class CoValueCore {
     this._cachedDependentOn = undefined;
     this._cachedNewContentSinceEmpty = undefined;
 
-    if (this.listeners.size > 0) {
-      if (notifyMode === "immediate") {
-        const content = this.getCurrentContent();
-        for (const listener of this.listeners) {
-          listener(content);
-        }
-      } else {
-        if (!this.nextDeferredNotify) {
-          this.nextDeferredNotify = new Promise((resolve) => {
-            setTimeout(() => {
-              this.nextDeferredNotify = undefined;
-              this.deferredUpdates = 0;
-              const content = this.getCurrentContent();
-              for (const listener of this.listeners) {
-                listener(content);
-              }
-              resolve();
-            }, 0);
-          });
-        }
-        this.deferredUpdates++;
-      }
-    }
+    this.notifyUpdate(notifyMode);
   }
 
   deferredUpdates = 0;
   nextDeferredNotify: Promise<void> | undefined;
 
+  notifyUpdate(notifyMode: "immediate" | "deferred") {
+    if (this.listeners.size === 0) {
+      return;
+    }
+
+    if (notifyMode === "immediate") {
+      const content = this.getCurrentContent();
+      for (const listener of this.listeners) {
+        listener(content);
+      }
+    } else {
+      if (!this.nextDeferredNotify) {
+        this.nextDeferredNotify = new Promise((resolve) => {
+          setTimeout(() => {
+            this.nextDeferredNotify = undefined;
+            this.deferredUpdates = 0;
+            const content = this.getCurrentContent();
+            for (const listener of this.listeners) {
+              listener(content);
+            }
+            resolve();
+          }, 0);
+        });
+      }
+      this.deferredUpdates++;
+    }
+  }
+
   subscribe(listener: (content?: RawCoValue) => void): () => void {
     this.listeners.add(listener);
     listener(this.getCurrentContent());
@@ -531,7 +386,6 @@ export class CoValueCore {
       streamingHash.update(transaction);
       const after = performance.now();
       if (after - before > 1) {
-        // console.log("Hashing blocked for", after - before);
         await new Promise((resolve) => setTimeout(resolve, 0));
         before = performance.now();
       }
@@ -680,7 +534,7 @@ export class CoValueCore {
       }
 
       if (!decryptedChanges) {
-        console.error("Failed to decrypt transaction despite having key");
+        logger.error("Failed to decrypt transaction despite having key");
         continue;
       }
 
@@ -825,7 +679,7 @@ export class CoValueCore {
           if (secret) {
             return secret as KeySecret;
           } else {
-            console.error(
+            logger.warn(
               `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`,
             );
           }
@@ -865,7 +719,7 @@ export class CoValueCore {
               if (secret) {
                 return secret as KeySecret;
               } else {
-                console.error(
+                logger.warn(
                   `Encrypting parent ${parentKey.id} key didn't decrypt ${keyID}`,
                 );
               }

package/src/coValueState.ts

@@ -1,6 +1,7 @@
 import { PeerState } from "./PeerState.js";
 import { CoValueCore } from "./coValueCore.js";
 import { RawCoID } from "./ids.js";
+import { logger } from "./logger.js";
 import { PeerID } from "./sync.js";
 
 export const CO_VALUE_LOADING_MAX_RETRIES = 5;
@@ -282,7 +283,7 @@ async function loadCoValueFromPeers(
           ...coValueEntry.state.coValue.knownState(),
         })
         .catch((err) => {
-          console.error(`Failed to push load message to peer ${peer.id}`, err);
+          logger.warn(`Failed to push load message to peer ${peer.id}`, err);
         });
     } else {
       /**
@@ -296,14 +297,14 @@ async function loadCoValueFromPeers(
           sessions: {},
         })
         .catch((err) => {
-          console.error(`Failed to push load message to peer ${peer.id}`, err);
+          logger.warn(`Failed to push load message to peer ${peer.id}`, err);
         });
     }
 
     if (coValueEntry.state.type === "loading") {
       const timeout = setTimeout(() => {
         if (coValueEntry.state.type === "loading") {
-          console.error("Failed to load coValue from peer", peer.id);
+          logger.warn("Failed to load coValue from peer", peer.id);
           coValueEntry.dispatch({
             type: "not-found-in-peer",
             peerId: peer.id,
@@ -356,7 +357,7 @@ function sleep(ms: number) {
 function getPeersWithoutErrors(peers: PeerState[], coValueId: RawCoID) {
   return peers.filter((p) => {
     if (p.erroredCoValues.has(coValueId)) {
-      console.error(
+      logger.warn(
         `Skipping load on errored coValue ${coValueId} from peer ${p.id}`,
       );
       return false;

package/src/coValues/account.ts

@@ -16,6 +16,7 @@ import {
 import { AgentID } from "../ids.js";
 import { JsonObject } from "../jsonValue.js";
 import { LocalNode } from "../localNode.js";
+import { logger } from "../logger.js";
 import type { AccountRole } from "../permissions.js";
 import { RawCoMap } from "./coMap.js";
 import { InviteSecret, RawGroup } from "./group.js";
@@ -59,7 +60,7 @@ export class RawAccount<
       );
 
     if (agents.length !== 1) {
-      console.warn("Account has " + agents.length + " agents", this.id);
+      logger.warn("Account has " + agents.length + " agents", this.id);
     }
 
     this._cachedCurrentAgentID = agents[0];

package/src/coValues/coList.ts

@@ -133,10 +133,6 @@ export class RawCoListView<
                   change.before.txIndex
                 ]?.[change.before.changeIdx];
               if (!beforeEntry) {
-                // console.error(
-                //     "Insertion before missing op " +
-                //         change.before
-                // );
                 continue;
               }
               beforeEntry.predecessors.splice(0, 0, {
@@ -156,9 +152,6 @@ export class RawCoListView<
                   change.after.txIndex
                 ]?.[change.after.changeIdx];
               if (!afterEntry) {
-                // console.error(
-                //     "Insertion after missing op " + change.after
-                // );
                 continue;
               }
               afterEntry.successors.push({

package/src/coValues/coStream.ts

@@ -3,6 +3,7 @@ import { CoID, RawCoValue } from "../coValue.js";
 import { CoValueCore } from "../coValueCore.js";
 import { AgentID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import { CoValueKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isAccountID } from "../typeUtils/isAccountID.js";
@@ -309,7 +310,7 @@ export class RawBinaryCoStreamView<
     const start = items[0];
 
     if (start?.type !== "start") {
-      console.error("Invalid binary stream start", start);
+      logger.error("Invalid binary stream start", start);
       return;
     }
 
@@ -328,7 +329,7 @@ export class RawBinaryCoStreamView<
       }
 
       if (item.type !== "chunk") {
-        console.error("Invalid binary stream chunk", item);
+        logger.error("Invalid binary stream chunk", item);
         return undefined;
       }
 
@@ -382,7 +383,6 @@ export class RawBinaryCoStream<
     chunk: Uint8Array,
     privacy: "private" | "trusting" = "private",
   ): void {
-    // const before = performance.now();
     this.push(
       {
         type: "chunk",
@@ -391,11 +391,6 @@ export class RawBinaryCoStream<
       privacy,
       false,
     );
-    // const after = performance.now();
-    // console.log(
-    //     "pushBinaryStreamChunk bandwidth in MB/s",
-    //     (1000 * chunk.length) / (after - before) / (1024 * 1024)
-    // );
   }
 
   endBinaryStream(privacy: "private" | "trusting" = "private") {

package/src/coValues/group.ts

@@ -13,6 +13,7 @@ import {
   isParentGroupReference,
 } from "../ids.js";
 import { JsonObject } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import { AccountRole, Role } from "../permissions.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
 import {
@@ -153,7 +154,7 @@ export class RawGroup<
         child.state.type === "unavailable"
       ) {
         child.loadFromPeers(peers).catch(() => {
-          console.error(`Failed to load child group ${id}`);
+          logger.error(`Failed to load child group ${id}`);
         });
       }
 
@@ -321,7 +322,7 @@ export class RawGroup<
         const secret = this.core.getReadKey(keyID);
 
         if (!secret) {
-          console.error("Can't find key", keyID);
+          logger.error("Can't find key", keyID);
           continue;
         }
 

package/src/coreToCoValue.ts

@@ -1,3 +1,4 @@
+import { RawUnknownCoValue } from "./coValue.js";
 import type { CoValueCore } from "./coValueCore.js";
 import { RawAccount, RawControlledAccount } from "./coValues/account.js";
 import { RawCoList } from "./coValues/coList.js";
@@ -38,6 +39,6 @@ export function coreToCoValue(
       return new RawCoStream(core);
     }
   } else {
-    throw new Error(`Unknown coValue type ${core.header.type}`);
+    return new RawUnknownCoValue(core);
   }
 }

package/src/crypto/PureJSCrypto.ts

@@ -7,6 +7,7 @@ import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
 import { RawCoID, TransactionID } from "../ids.js";
 import { Stringified, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { logger } from "../logger.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -192,7 +193,7 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
     try {
       return JSON.parse(textDecoder.decode(plaintext));
     } catch (e) {
-      console.error("Failed to decrypt/parse sealed message", e);
+      logger.error("Failed to decrypt/parse sealed message", e);
       return undefined;
     }
   }