jazz-tools 0.18.5 → 0.18.7

package/src/tools/coValues/inbox.ts

@@ -1,5 +1,12 @@
-import { CoID, InviteSecret, RawAccount, RawCoMap, SessionID } from "cojson";
-import { CoStreamItem, RawCoStream } from "cojson";
+import {
+  CoID,
+  CoValueCore,
+  InviteSecret,
+  RawAccount,
+  RawCoMap,
+  SessionID,
+} from "cojson";
+import { type AvailableCoValueCore, type RawCoStream } from "cojson";
 import {
   type Account,
   CoValue,
@@ -16,10 +23,11 @@ export type InboxInvite = `${CoID<MessagesStream>}/${InviteSecret}`;
 type TxKey = `${SessionID}/${number}`;
 
 type MessagesStream = RawCoStream<CoID<InboxMessage<CoValue, any>>>;
-type FailedMessagesStream = RawCoStream<{
+type FailedMessagesStreamItem = {
   errors: string[];
   value: CoID<InboxMessage<CoValue, any>>;
-}>;
+};
+type FailedMessagesStream = RawCoStream<FailedMessagesStreamItem>;
 type TxKeyStream = RawCoStream<TxKey>;
 export type InboxRoot = RawCoMap<{
   messages: CoID<MessagesStream>;
@@ -55,6 +63,25 @@ export function createInboxRoot(account: Account) {
   };
 }
 
+/**
+ * An abstraction on top of CoStream to get the new items in a performant way.
+ */
+class IncrementalFeed {
+  constructor(private feed: AvailableCoValueCore) {}
+
+  private sessions = {};
+  getNewItems() {
+    const items = this.feed.getValidTransactions({
+      ignorePrivateTransactions: false,
+      from: this.sessions,
+    });
+
+    this.sessions = this.feed.knownState().sessions;
+
+    return items;
+  }
+}
+
 type InboxMessage<I extends CoValue, O extends CoValue | undefined> = RawCoMap<{
   payload: ID<I>;
   result: ID<O> | undefined;
@@ -87,13 +114,68 @@ async function createInboxMessage<
   return message;
 }
 
+class MessageQueue {
+  private queue: Array<{
+    txKey: TxKey;
+    messageId: CoID<InboxMessage<CoValue, any>>;
+  }> = [];
+  private processing = new Set<TxKey>();
+  private concurrencyLimit: number;
+  private activeCount = 0;
+
+  constructor(
+    concurrencyLimit: number = 10,
+    private processMessage: (
+      txKey: TxKey,
+      messageId: CoID<InboxMessage<CoValue, any>>,
+    ) => Promise<void>,
+    private handleError: (
+      txKey: TxKey,
+      messageId: CoID<InboxMessage<CoValue, any>>,
+      error: Error,
+    ) => void,
+  ) {
+    this.concurrencyLimit = concurrencyLimit;
+  }
+
+  enqueue(txKey: TxKey, messageId: CoID<InboxMessage<CoValue, any>>) {
+    this.queue.push({ txKey, messageId });
+    this.processNext();
+  }
+
+  private async processNext() {
+    if (this.activeCount >= this.concurrencyLimit || this.queue.length === 0) {
+      return;
+    }
+
+    const { txKey, messageId } = this.queue.shift()!;
+
+    if (this.processing.has(txKey)) {
+      this.processNext();
+      return;
+    }
+
+    this.processing.add(txKey);
+    this.activeCount++;
+
+    try {
+      await this.processMessage(txKey, messageId);
+    } catch (error) {
+      this.handleError(txKey, messageId, error as Error);
+    } finally {
+      this.processing.delete(txKey);
+      this.activeCount--;
+      this.processNext();
+    }
+  }
+}
+
 export class Inbox {
   account: Account;
   messages: MessagesStream;
   processed: TxKeyStream;
   failed: FailedMessagesStream;
   root: InboxRoot;
-  processing = new Set<`${SessionID}/${number}`>();
 
   private constructor(
     account: Account,
@@ -115,127 +197,126 @@ export class Inbox {
       message: InstanceOfSchema<M>,
       senderAccountID: ID<Account>,
     ) => Promise<O | undefined | void>,
-    options: { retries?: number } = {},
+    options?: { concurrencyLimit?: number },
   ) {
     const processed = new Set<`${SessionID}/${number}`>();
-    const failed = new Map<`${SessionID}/${number}`, string[]>();
     const node = this.account.$jazz.localNode;
 
-    this.processed.subscribe((stream) => {
-      for (const items of Object.values(stream.items)) {
-        for (const item of items) {
-          processed.add(item.value as TxKey);
-        }
+    // Create queue instance inside subscribe function
+    const concurrencyLimit = options?.concurrencyLimit ?? 10;
+
+    const processedFeed = new IncrementalFeed(this.processed.core);
+
+    // Track the already processed messages, triggered immediately so we know the messages processed in the previous sessions
+    this.processed.subscribe(() => {
+      for (const { changes } of processedFeed.getNewItems()) {
+        processed.add(changes[0] as TxKey);
       }
     });
 
     const { account } = this;
-    const { retries = 3 } = options;
 
-    let failTimer: ReturnType<typeof setTimeout> | number | undefined =
-      undefined;
+    const messagesFeed = new IncrementalFeed(this.messages.core);
+
+    // Set up the message processing handler for the queue
+    const processMessage = async (
+      txKey: TxKey,
+      messageId: CoID<InboxMessage<CoValue, any>>,
+    ) => {
+      const message = await node.load(messageId);
+      if (message === "unavailable") {
+        throw new Error(`Inbox: message ${messageId} is unavailable`);
+      }
+
+      const value = await loadCoValue(
+        coValueClassFromCoValueClassOrSchema(Schema),
+        message.get("payload")!,
+        {
+          loadAs: account,
+        },
+      );
+
+      if (!value) {
+        throw new Error(
+          `Inbox: Unable to load the payload of message ${messageId}`,
+        );
+      }
+
+      const accountID = getAccountIDfromSessionID(
+        txKey.split("/")[0] as SessionID,
+      );
+      if (!accountID) {
+        throw new Error(`Inbox: Unknown account for message ${messageId}`);
+      }
+
+      const result = await callback(value as InstanceOfSchema<M>, accountID);
+
+      const inboxMessage = node
+        .expectCoValueLoaded(messageId)
+        .getCurrentContent() as RawCoMap;
+
+      if (result) {
+        inboxMessage.set("result", result.$jazz.id);
+      }
 
-    const clearFailTimer = () => {
-      clearTimeout(failTimer);
-      failTimer = undefined;
+      inboxMessage.set("processed", true);
+      this.processed.push(txKey);
     };
 
-    const handleNewMessages = (stream: MessagesStream) => {
-      clearFailTimer(); // Stop the failure timers, we're going to process the failed entries anyway
+    const handleError = (
+      txKey: TxKey,
+      messageId: CoID<InboxMessage<CoValue, any>>,
+      error: Error,
+    ) => {
+      console.error(error);
 
-      for (const [sessionID, items] of Object.entries(stream.items) as [
-        SessionID,
-        CoStreamItem<CoID<InboxMessage<NonNullable<InstanceOfSchema<M>>, O>>>[],
-      ][]) {
-        const accountID = getAccountIDfromSessionID(sessionID);
+      const stringifiedError = String(error);
+
+      this.processed.push(txKey);
+      this.failed.push({ errors: [stringifiedError], value: messageId });
+
+      try {
+        const inboxMessage = node
+          .expectCoValueLoaded(messageId)
+          .getCurrentContent() as RawCoMap;
+
+        inboxMessage.set("error", stringifiedError);
+        inboxMessage.set("processed", true);
+      } catch (error) {}
+    };
+
+    const messageQueue = new MessageQueue(
+      concurrencyLimit,
+      processMessage,
+      handleError,
+    );
+
+    const handleNewMessages = () => {
+      for (const tx of messagesFeed.getNewItems()) {
+        const accountID = getAccountIDfromSessionID(tx.txID.sessionID);
 
         if (!accountID) {
-          console.warn("Received message from unknown account", sessionID);
+          console.warn(
+            "Received message from unknown account",
+            tx.txID.sessionID,
+          );
           continue;
         }
 
-        for (const item of items) {
-          const txKey = `${sessionID}/${item.tx.txIndex}` as const;
-
-          if (!processed.has(txKey) && !this.processing.has(txKey)) {
-            this.processing.add(txKey);
-
-            const id = item.value;
-
-            node
-              .load(id)
-              .then((message) => {
-                if (message === "unavailable") {
-                  return Promise.reject(
-                    new Error("Unable to load inbox message " + id),
-                  );
-                }
-
-                return loadCoValue(
-                  coValueClassFromCoValueClassOrSchema(Schema),
-                  message.get("payload")!,
-                  {
-                    loadAs: account,
-                  },
-                );
-              })
-              .then((value) => {
-                if (!value) {
-                  return Promise.reject(
-                    new Error("Unable to load inbox message " + id),
-                  );
-                }
-
-                return callback(value as InstanceOfSchema<M>, accountID);
-              })
-              .then((result) => {
-                const inboxMessage = node
-                  .expectCoValueLoaded(item.value)
-                  .getCurrentContent() as RawCoMap;
-
-                if (result) {
-                  inboxMessage.set("result", result.$jazz.id);
-                }
-
-                inboxMessage.set("processed", true);
-
-                this.processed.push(txKey);
-                this.processing.delete(txKey);
-              })
-              .catch((error) => {
-                console.error("Error processing inbox message", error);
-                this.processing.delete(txKey);
-                const errors = failed.get(txKey) ?? [];
-
-                const stringifiedError = String(error);
-                errors.push(stringifiedError);
-
-                let inboxMessage: RawCoMap | undefined;
-
-                try {
-                  inboxMessage = node
-                    .expectCoValueLoaded(item.value)
-                    .getCurrentContent() as RawCoMap;
-
-                  inboxMessage.set("error", stringifiedError);
-                } catch (error) {}
-
-                if (errors.length > retries) {
-                  inboxMessage?.set("processed", true);
-                  this.processed.push(txKey);
-                  this.failed.push({ errors, value: item.value });
-                } else {
-                  failed.set(txKey, errors);
-                  if (!failTimer) {
-                    failTimer = setTimeout(
-                      () => handleNewMessages(stream),
-                      100,
-                    );
-                  }
-                }
-              });
-          }
+        const id = tx.changes[0] as CoID<InboxMessage<CoValue, any>>;
+
+        if (!isCoValueId(id)) {
+          continue;
         }
+
+        const txKey = `${tx.txID.sessionID}/${tx.txID.txIndex}` as const;
+
+        if (processed.has(txKey)) {
+          continue;
+        }
+
+        // Enqueue the message for processing
+        messageQueue.enqueue(txKey, id);
       }
     };
 
@@ -243,7 +324,6 @@ export class Inbox {
 
     return () => {
       unsubscribe();
-      clearFailTimer();
     };
   }
 
@@ -280,6 +360,8 @@ export class Inbox {
       throw new Error("Inbox not found");
     }
 
+    await processed.core.waitForFullStreaming();
+
     return new Inbox(account, root, messages, processed, failed);
   }
 }

package/src/tools/implementation/createContext.ts

@@ -96,6 +96,7 @@ export async function createJazzContextFromExistingCredentials<
   AccountSchema: PropsAccountSchema,
   sessionProvider,
   onLogOut,
+  asActiveAccount,
 }: {
   credentials: Credentials;
   peersToLoadFrom: Peer[];
@@ -104,6 +105,7 @@ export async function createJazzContextFromExistingCredentials<
   sessionProvider: SessionProvider;
   onLogOut?: () => void;
   storage?: StorageAPI;
+  asActiveAccount: boolean;
 }): Promise<JazzContextWithAccount<InstanceOfSchema<S>>> {
   const { sessionID, sessionDone } = await sessionProvider(
     credentials.accountID,
@@ -125,14 +127,18 @@ export async function createJazzContextFromExistingCredentials<
     storage,
     migration: async (rawAccount, _node, creationProps) => {
       const account = AccountClass.fromRaw(rawAccount) as InstanceOfSchema<S>;
-      activeAccountContext.set(account);
+      if (asActiveAccount) {
+        activeAccountContext.set(account);
+      }
 
       await account.applyMigration(creationProps);
     },
   });
 
   const account = AccountClass.fromNode(node);
-  activeAccountContext.set(account);
+  if (asActiveAccount) {
+    activeAccountContext.set(account);
+  }
 
   return {
     node,
@@ -245,6 +251,7 @@ export async function createJazzContext<
         authSecretStorage.clearWithoutNotify();
       },
       storage: options.storage,
+      asActiveAccount: true,
     });
   } else {
     const secretSeed = options.crypto.newRandomSecretSeed();

package/src/tools/testing.ts

@@ -56,7 +56,7 @@ export function getPeerConnectedToTestSyncServer() {
     Math.random().toString(),
     Math.random().toString(),
     {
-      peer1role: "server",
+      peer1role: "client",
       peer2role: "server",
     },
   );

package/src/tools/tests/coFeed.test.ts

@@ -17,8 +17,13 @@ import {
   z,
 } from "../index.js";
 import { createJazzTestAccount, setupJazzTestSync } from "../testing.js";
-import { setupTwoNodes } from "./utils.js";
-import { CoFeed, ControlledAccount, TypeSym } from "../internal.js";
+import { setupTwoNodes, waitFor } from "./utils.js";
+import {
+  CoFeed,
+  CoFeedInstanceCoValuesNullable,
+  ControlledAccount,
+  TypeSym,
+} from "../internal.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -203,7 +208,7 @@ describe("CoFeed resolution", async () => {
 
     assert(myStream);
 
-    expect(myStream.value).toBeTruthy();
+    await waitFor(() => expect(myStream.value).toBeTruthy());
 
     assert(myStream.value);
 
@@ -211,7 +216,7 @@ describe("CoFeed resolution", async () => {
 
     assert(loadedNestedStreamByMe);
 
-    expect(loadedNestedStreamByMe.value).toBeTruthy();
+    await waitFor(() => expect(loadedNestedStreamByMe.value).toBeTruthy());
 
     assert(loadedNestedStreamByMe.value);
 
@@ -220,7 +225,7 @@ describe("CoFeed resolution", async () => {
 
     assert(loadedTwiceNestedStreamByMe);
 
-    expect(loadedTwiceNestedStreamByMe.value).toBe("milk");
+    await waitFor(() => expect(loadedTwiceNestedStreamByMe.value).toBe("milk"));
 
     assert(loadedTwiceNestedStreamByMe.value);
   });
@@ -231,21 +236,24 @@ describe("CoFeed resolution", async () => {
 
     const anotherAccount = await createJazzTestAccount();
 
-    const queue = new Channel();
+    let result: CoFeedInstanceCoValuesNullable<co.Feed<co.Feed<z.z.ZodString>>>;
 
     TestStream.subscribe(
       stream.$jazz.id,
       { loadAs: anotherAccount },
       (subscribedStream) => {
-        void queue.push(subscribedStream);
+        result = subscribedStream;
       },
     );
 
-    const update1 = (await queue.next()).value;
-    expect(
-      update1.perAccount[accountId]?.value?.perAccount[accountId]?.value
-        ?.perAccount[accountId]?.value,
-    ).toBe("milk");
+    await waitFor(() => expect(result).toBeDefined());
+
+    await waitFor(() => {
+      expect(
+        result.perAccount[accountId]?.value?.perAccount[accountId]?.value
+          ?.perAccount[accountId]?.value,
+      ).toBe("milk");
+    });
 
     // When assigning a new nested stream, we get an update
     const newTwiceNested = TwiceNestedStream.create(["butter"], {
@@ -258,19 +266,22 @@ describe("CoFeed resolution", async () => {
 
     stream.$jazz.push(newNested);
 
-    const update2 = (await queue.next()).value;
-    expect(
-      update2.perAccount[me.$jazz.id]?.value?.perAccount[me.$jazz.id]?.value
-        ?.perAccount[me.$jazz.id]?.value,
-    ).toBe("butter");
+    await waitFor(() => {
+      expect(
+        result.perAccount[accountId]?.value?.perAccount[accountId]?.value
+          ?.perAccount[accountId]?.value,
+      ).toBe("butter");
+    });
 
     // we get updates when the new nested stream changes
     newTwiceNested.$jazz.push("jam");
-    const update3 = (await queue.next()).value;
-    expect(
-      update3.perAccount[me.$jazz.id]?.value?.perAccount[me.$jazz.id]?.value
-        ?.perAccount[me.$jazz.id]?.value,
-    ).toBe("jam");
+
+    await waitFor(() => {
+      expect(
+        result.perAccount[accountId]?.value?.perAccount[accountId]?.value
+          ?.perAccount[accountId]?.value,
+      ).toBe("jam");
+    });
   });
 
   test("Subscription without options", async () => {

package/src/tools/tests/coList.test.ts

@@ -122,6 +122,25 @@ describe("Simple CoList operations", async () => {
     expect(list[0]).toEqual("milk");
   });
 
+  test("CoList keys can be iterated over just like an array's", () => {
+    const TestList = co.list(z.string());
+    const list = ["a", "b", "c", "d", "e"];
+    const coList = TestList.create(list);
+    const keys = [];
+    for (const key in coList) {
+      keys.push(key);
+    }
+    expect(keys).toEqual(Object.keys(list));
+    expect(Object.keys(coList)).toEqual(Object.keys(list));
+  });
+
+  test("a CoList is structurally equal to an array", () => {
+    const TestList = co.list(z.string());
+    const list = ["a", "b", "c", "d", "e"];
+    const coList = TestList.create(list);
+    expect(coList).toEqual(list);
+  });
+
   describe("Mutation", () => {
     test("assignment", () => {
       const list = TestList.create(["bread", "butter", "onion"], {
@@ -963,17 +982,19 @@ describe("CoList subscription", async () => {
 
     expect(spy).toHaveBeenCalledTimes(1);
 
-    expect(updates[0]?.[0]?.name).toEqual("Item 1");
-    expect(updates[0]?.[1]?.name).toEqual("Item 2");
+    await waitFor(() => {
+      expect(updates[0]?.[0]?.name).toEqual("Item 1");
+      expect(updates[0]?.[1]?.name).toEqual("Item 2");
+    });
 
     list[0]!.$jazz.set("name", "Updated Item 1");
 
-    await waitFor(() => expect(spy).toHaveBeenCalledTimes(2));
+    await waitFor(() => expect(spy).toHaveBeenCalledTimes(4));
 
     expect(updates[1]?.[0]?.name).toEqual("Updated Item 1");
     expect(updates[1]?.[1]?.name).toEqual("Item 2");
 
-    expect(spy).toHaveBeenCalledTimes(2);
+    expect(spy).toHaveBeenCalledTimes(4);
   });
 
   test("replacing list items triggers updates", async () => {
@@ -1312,3 +1333,25 @@ describe("co.list schema", () => {
     expect(keywords[1]?.toString()).toEqual("world");
   });
 });
+
+describe("lastUpdatedAt", () => {
+  test("empty list last updated time", () => {
+    const emptyList = co.list(z.number()).create([]);
+
+    expect(emptyList.$jazz.lastUpdatedAt).not.toEqual(0);
+    expect(emptyList.$jazz.lastUpdatedAt).toEqual(emptyList.$jazz.createdAt);
+  });
+
+  test("last update should change on push", async () => {
+    const list = co.list(z.string()).create(["John"]);
+
+    expect(list.$jazz.lastUpdatedAt).not.toEqual(0);
+
+    const updatedAt = list.$jazz.lastUpdatedAt;
+
+    await new Promise((r) => setTimeout(r, 10));
+    list.$jazz.push("Jane");
+
+    expect(list.$jazz.lastUpdatedAt).not.toEqual(updatedAt);
+  });
+});

package/src/tools/tests/coMap.test.ts

@@ -1009,7 +1009,10 @@ describe("CoMap resolution", async () => {
     });
 
     assert(loadedPerson);
-    expect(loadedPerson.dog?.name).toEqual("Rex");
+
+    await waitFor(() => {
+      expect(loadedPerson.dog?.name).toEqual("Rex");
+    });
   });
 
   test("loading a remotely available map with skipRetry set to true", async () => {
@@ -1118,7 +1121,10 @@ describe("CoMap resolution", async () => {
 
     expect(resolved).toBe(true);
     assert(loadedPerson);
-    expect(loadedPerson.dog?.name).toEqual("Rex");
+
+    await waitFor(() => {
+      expect(loadedPerson.dog?.name).toEqual("Rex");
+    });
   });
 
   test("accessing the value refs", async () => {
@@ -1396,15 +1402,17 @@ describe("CoMap resolution", async () => {
 
     expect(spy).toHaveBeenCalledTimes(1);
 
-    expect(updates[0]?.dog?.name).toEqual("Rex");
+    await waitFor(() => {
+      expect(updates[0]?.dog?.name).toEqual("Rex");
+    });
 
     person.dog!.$jazz.set("name", "Fido");
 
-    await waitFor(() => expect(spy).toHaveBeenCalledTimes(2));
+    await waitFor(() => expect(spy).toHaveBeenCalledTimes(3));
 
     expect(updates[1]?.dog?.name).toEqual("Fido");
 
-    expect(spy).toHaveBeenCalledTimes(2);
+    expect(spy).toHaveBeenCalledTimes(3);
   });
 
   test("replacing nested object triggers updates", async () => {

package/src/tools/tests/coPlainText.test.ts

@@ -183,6 +183,7 @@ describe("CoPlainText", () => {
           sessionProvider: randomSessionProvider,
           peersToLoadFrom: [initialAsPeer],
           crypto: Crypto,
+          asActiveAccount: true,
         });
 
       // Load the text on the second peer
@@ -214,6 +215,7 @@ describe("CoPlainText", () => {
         sessionProvider: randomSessionProvider,
         peersToLoadFrom: [initialAsPeer],
         crypto: Crypto,
+        asActiveAccount: true,
       });
 
     const queue = new Channel();
@@ -242,3 +244,25 @@ describe("CoPlainText", () => {
     expect(update3.toString()).toBe("hello world");
   });
 });
+
+describe("lastUpdatedAt", () => {
+  test("empty text last updated time", () => {
+    const text = co.plainText().create("");
+
+    expect(text.$jazz.lastUpdatedAt).toEqual(text.$jazz.createdAt);
+    expect(text.$jazz.lastUpdatedAt).not.toEqual(0);
+  });
+
+  test("last update should change on push", async () => {
+    const text = co.plainText().create("John");
+
+    expect(text.$jazz.lastUpdatedAt).not.toEqual(0);
+
+    const updatedAt = text.$jazz.lastUpdatedAt;
+
+    await new Promise((r) => setTimeout(r, 10));
+    text.$jazz.applyDiff("Jane");
+
+    expect(text.$jazz.lastUpdatedAt).not.toEqual(updatedAt);
+  });
+});