cojson 0.8.36 → 0.8.38

package/src/coValues/coMap.ts

@@ -1,5 +1,5 @@
 import { CoID, RawCoValue } from "../coValue.js";
-import { CoValueCore } from "../coValueCore.js";
+import { CoValueCore, DecryptedTransaction } from "../coValueCore.js";
 import { AgentID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
@@ -49,6 +49,9 @@ export class RawCoMapView<
   cachedOps?: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
   };
+  /** @internal */
+  validSortedTransactions?: DecryptedTransaction[];
+
   /** @internal */
   options?: { ignorePrivateTransactions: boolean; atTime?: number };
   /** @internal */
@@ -59,27 +62,58 @@ export class RawCoMapView<
   /** @internal */
   constructor(
     core: CoValueCore,
-    options?: { ignorePrivateTransactions: boolean; atTime?: number },
+    options?: {
+      ignorePrivateTransactions: boolean;
+      atTime?: number;
+      validSortedTransactions?: DecryptedTransaction[];
+      cachedOps?: {
+        [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
+      };
+    },
   ) {
     this.id = core.id as CoID<this>;
     this.core = core;
     this.latest = {};
     this.latestTxMadeAt = 0;
     this.options = options;
+    this.cachedOps = options?.cachedOps;
+    this.validSortedTransactions = options?.validSortedTransactions;
 
     this.processLatestTransactions();
   }
 
-  processLatestTransactions() {
+  /** @internal */
+  private getValidSortedTransactions() {
+    if (this.validSortedTransactions) {
+      return this.validSortedTransactions;
+    }
+
+    const validSortedTransactions = this.core.getValidSortedTransactions({
+      ignorePrivateTransactions:
+        this.options?.ignorePrivateTransactions ?? false,
+    });
+
+    this.validSortedTransactions = validSortedTransactions;
+
+    return validSortedTransactions;
+  }
+
+  private resetCachedValues() {
+    this.validSortedTransactions = undefined;
+    this.cachedOps = undefined;
+  }
+
+  private processLatestTransactions() {
+    // Reset all internal state and cached values
     this.latest = {};
     this.latestTxMadeAt = 0;
 
-    const { core, options, latest } = this;
+    const { latest } = this;
+
+    const atTimeFilter = this.options?.atTime;
 
-    for (const { txID, changes, madeAt } of core.getValidSortedTransactions({
-      ignorePrivateTransactions: options?.ignorePrivateTransactions ?? false,
-    })) {
-      if (options?.atTime && madeAt > options.atTime) {
+    for (const { txID, changes, madeAt } of this.getValidSortedTransactions()) {
+      if (atTimeFilter && madeAt > atTimeFilter) {
         continue;
       }
 
@@ -92,15 +126,14 @@ export class RawCoMapView<
           keyof Shape & string,
           Shape[keyof Shape & string]
         >;
-        let entry = latest[change.key];
+        const entry = latest[change.key];
         if (!entry) {
-          entry = {
+          latest[change.key] = {
             txID,
             madeAt,
             changeIdx,
             change,
           };
-          latest[change.key] = entry;
         } else if (madeAt >= entry.madeAt) {
           entry.txID = txID;
           entry.madeAt = madeAt;
@@ -111,6 +144,11 @@ export class RawCoMapView<
     }
   }
 
+  revalidateTransactions() {
+    this.resetCachedValues();
+    this.processLatestTransactions();
+  }
+
   private getOps() {
     if (this.cachedOps) {
       return this.cachedOps;
@@ -120,11 +158,7 @@ export class RawCoMapView<
       [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
     } = {};
 
-    for (const {
-      txID,
-      changes,
-      madeAt,
-    } of this.core.getValidSortedTransactions(this.options)) {
+    for (const { txID, changes, madeAt } of this.getValidSortedTransactions()) {
       for (let changeIdx = 0; changeIdx < changes.length; changeIdx++) {
         const change = changes[changeIdx] as MapOpPayload<
           keyof Shape & string,
@@ -168,6 +202,8 @@ export class RawCoMapView<
         ignorePrivateTransactions:
           this.options?.ignorePrivateTransactions ?? false,
         atTime: time,
+        cachedOps: this.cachedOps,
+        validSortedTransactions: this.validSortedTransactions,
       });
       Object.setPrototypeOf(clone, this);
       return clone as this;
@@ -182,10 +218,10 @@ export class RawCoMapView<
       return undefined;
     }
 
-    if (this.atTimeFilter) {
-      return this.getOps()[key]?.filter(
-        (op) => op.madeAt <= this.atTimeFilter!,
-      );
+    const atTimeFilter = this.options?.atTime;
+
+    if (atTimeFilter) {
+      return this.getOps()[key]?.filter((op) => op.madeAt <= atTimeFilter);
     } else {
       return this.getOps()[key];
     }
@@ -197,17 +233,13 @@ export class RawCoMapView<
    * @category 1. Reading */
   keys<K extends keyof Shape & string = keyof Shape & string>(): K[] {
     return (Object.keys(this.latest) as K[]).filter((key) => {
-      const latest = this.latest[key];
-      if (!latest) {
-        return undefined;
-      }
+      const latestChange = this.latest[key];
 
-      const includeUntil = this.atTimeFilter;
-      const lastEntry = includeUntil
-        ? this.getOps()[key]?.findLast((entry) => entry.madeAt <= includeUntil)
-        : latest!;
+      if (!latestChange) {
+        return false;
+      }
 
-      if (lastEntry?.change.op === "del") {
+      if (latestChange.change.op === "del") {
         return false;
       } else {
         return true;
@@ -221,22 +253,15 @@ export class RawCoMapView<
    * @category 1. Reading
    **/
   get<K extends keyof Shape & string>(key: K): Shape[K] | undefined {
-    const latest = this.latest[key];
-    if (!latest) {
+    const latestChange = this.latest[key];
+    if (!latestChange) {
       return undefined;
     }
 
-    const includeUntil = this.atTimeFilter;
-    const lastEntry = includeUntil
-      ? this.timeFilteredOps(key)?.findLast(
-          (entry) => entry.madeAt <= includeUntil,
-        )
-      : latest;
-
-    if (lastEntry?.change.op === "del") {
+    if (latestChange.change.op === "del") {
       return undefined;
     } else {
-      return lastEntry?.change.value;
+      return latestChange.change.value as Shape[K];
     }
   }
 
@@ -248,7 +273,7 @@ export class RawCoMapView<
       [K in keyof Shape & string]: Shape[K];
     }> = {};
 
-    for (const key of this.keys()) {
+    for (const key of Object.keys(this.latest) as (keyof Shape & string)[]) {
       const value = this.get(key);
       if (value !== undefined) {
         object[key] = value;
@@ -268,34 +293,21 @@ export class RawCoMapView<
   }
 
   /** @category 5. Edit history */
-  nthEditAt<K extends keyof Shape & string>(
-    key: K,
-    n: number,
-  ):
-    | {
-        by: RawAccountID | AgentID;
-        tx: TransactionID;
-        at: Date;
-        value?: Shape[K];
-      }
-    | undefined {
-    const ops = this.timeFilteredOps(key);
-    if (!ops || ops.length <= n) {
+  nthEditAt<K extends keyof Shape & string>(key: K, n: number) {
+    const ops = this.getOps()[key];
+
+    const atTimeFilter = this.options?.atTime;
+    const entry = ops?.[n];
+
+    if (!entry) {
       return undefined;
     }
 
-    const entry = ops[n]!;
-
-    if (this.atTimeFilter && entry.madeAt > this.atTimeFilter) {
+    if (atTimeFilter && entry.madeAt > atTimeFilter) {
       return undefined;
     }
 
-    return {
-      by: accountOrAgentIDfromSessionID(entry.txID.sessionID),
-      tx: entry.txID,
-      at: new Date(entry.madeAt),
-      value: entry.change.op === "del" ? undefined : entry.change.value,
-    };
+    return operationToEditEntry(entry);
   }
 
   /** @category 5. Edit history */
@@ -310,10 +322,13 @@ export class RawCoMapView<
       }
     | undefined {
     const ops = this.timeFilteredOps(key);
-    if (!ops || ops.length === 0) {
+    const lastEntry = ops?.[ops.length - 1];
+
+    if (!lastEntry) {
       return undefined;
     }
-    return this.nthEditAt(key, ops.length - 1);
+
+    return operationToEditEntry(lastEntry);
   }
 
   /** @category 5. Edit history */
@@ -323,8 +338,8 @@ export class RawCoMapView<
       return;
     }
 
-    for (let i = 0; i < ops.length; i++) {
-      yield this.nthEditAt(key, i)!;
+    for (const entry of ops) {
+      yield operationToEditEntry(entry);
     }
   }
 
@@ -370,8 +385,7 @@ export class RawCoMap<
       privacy,
     );
 
-    this.processLatestTransactions();
-    this.cachedOps = undefined;
+    this.revalidateTransactions();
   }
 
   /** Delete the given key (setting it to undefined).
@@ -396,7 +410,18 @@ export class RawCoMap<
       privacy,
     );
 
-    this.processLatestTransactions();
-    this.cachedOps = undefined;
+    this.revalidateTransactions();
   }
 }
+
+export function operationToEditEntry<
+  K extends string,
+  V extends JsonValue | undefined,
+>(op: MapOp<K, V>) {
+  return {
+    by: accountOrAgentIDfromSessionID(op.txID.sessionID),
+    tx: op.txID,
+    at: new Date(op.madeAt),
+    value: op.change.op === "del" ? undefined : op.change.value,
+  };
+}

package/src/coValues/group.ts

@@ -81,6 +81,7 @@ export class RawGroup<
     accountID: RawAccountID | AgentID | typeof EVERYONE,
   ): { role: Role; via: CoID<RawGroup> | undefined } | undefined {
     const roleHere = this.get(accountID);
+
     if (roleHere === "revoked") {
       return undefined;
     }
@@ -92,7 +93,7 @@ export class RawGroup<
         }
       | undefined = roleHere && { role: roleHere, via: undefined };
 
-    const parentGroups = this.getParentGroups();
+    const parentGroups = this.getParentGroups(this.options?.atTime);
 
     for (const parentGroup of parentGroups) {
       const roleInParent = parentGroup.roleOfInternal(accountID);
@@ -109,7 +110,7 @@ export class RawGroup<
     return roleInfo;
   }
 
-  getParentGroups() {
+  getParentGroups(atTime?: number) {
     const groups: RawGroup[] = [];
 
     for (const key of this.keys()) {
@@ -118,7 +119,14 @@ export class RawGroup<
           getParentGroupId(key),
           "Expected parent group to be loaded",
         );
-        groups.push(expectGroup(parent.getCurrentContent()));
+
+        const parentGroup = expectGroup(parent.getCurrentContent());
+
+        if (atTime) {
+          groups.push(parentGroup.atTime(atTime));
+        } else {
+          groups.push(parentGroup);
+        }
       }
     }
 

package/src/tests/PriorityBasedMessageQueue.test.ts

@@ -1,14 +1,90 @@
-import { describe, expect, test } from "vitest";
+import { metrics } from "@opentelemetry/api";
+import {
+  AggregationTemporality,
+  InMemoryMetricExporter,
+  MeterProvider,
+  MetricReader,
+  type MetricReaderOptions,
+  type PushMetricExporter,
+} from "@opentelemetry/sdk-metrics";
+import { afterEach, describe, expect, test } from "vitest";
 import { PriorityBasedMessageQueue } from "../PriorityBasedMessageQueue.js";
 import { CO_VALUE_PRIORITY } from "../priority.js";
-import { SyncMessage } from "../sync.js";
+import type { SyncMessage } from "../sync.js";
+
+interface A extends MetricReaderOptions {
+  exporter: PushMetricExporter;
+}
+
+/**
+ * This is a test metric reader that uses an in-memory metric exporter and exposes a method to get the value of a metric given its name and attributes.
+ *
+ * This is useful for testing the values of metrics that are collected by the SDK.
+ *
+ * TODO: We could move this to a separate file and make it a utility class that can be used in other tests.
+ * TODO: We may want to rethink how we access metrics (see `getMetricValue` method) to make it more flexible.
+ */
+class TestMetricReader extends MetricReader {
+  private _exporter = new InMemoryMetricExporter(
+    AggregationTemporality.CUMULATIVE,
+  );
+
+  protected onShutdown(): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+  protected onForceFlush(): Promise<void> {
+    throw new Error("Method not implemented.");
+  }
+
+  async getMetricValue(
+    name: string,
+    attributes: { [key: string]: string | number } = {},
+  ) {
+    await this.collectAndExport();
+    const metric = this._exporter
+      .getMetrics()[0]
+      ?.scopeMetrics[0]?.metrics.find((m) => m.descriptor.name === name);
+
+    const dp = metric?.dataPoints.find(
+      (dp) => JSON.stringify(dp.attributes) === JSON.stringify(attributes),
+    );
+
+    this._exporter.reset();
+
+    return dp?.value;
+  }
+
+  async collectAndExport(): Promise<void> {
+    const result = await this.collect();
+    await new Promise<void>((resolve, reject) => {
+      this._exporter.export(result.resourceMetrics, (result) => {
+        if (result.error != null) {
+          reject(result.error);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+}
 
 function setup() {
+  const metricReader = new TestMetricReader();
+  metrics.setGlobalMeterProvider(
+    new MeterProvider({
+      readers: [metricReader],
+    }),
+  );
+
   const queue = new PriorityBasedMessageQueue(CO_VALUE_PRIORITY.MEDIUM);
-  return { queue };
+  return { queue, metricReader };
 }
 
 describe("PriorityBasedMessageQueue", () => {
+  afterEach(() => {
+    metrics.disable();
+  });
+
   test("should initialize with correct properties", () => {
     const { queue } = setup();
     expect(queue["defaultPriority"]).toBe(CO_VALUE_PRIORITY.MEDIUM);
@@ -43,7 +119,7 @@ describe("PriorityBasedMessageQueue", () => {
   });
 
   test("should pull messages in priority order", async () => {
-    const { queue } = setup();
+    const { queue, metricReader } = setup();
     const lowPriorityMsg: SyncMessage = {
       action: "content",
       id: "co_zlow",
@@ -64,12 +140,42 @@ describe("PriorityBasedMessageQueue", () => {
     };
 
     void queue.push(lowPriorityMsg);
+    expect(
+      await metricReader.getMetricValue("jazz.messagequeue.size", {
+        priority: lowPriorityMsg.priority,
+      }),
+    ).toBe(1);
     void queue.push(mediumPriorityMsg);
+    expect(
+      await metricReader.getMetricValue("jazz.messagequeue.size", {
+        priority: mediumPriorityMsg.priority,
+      }),
+    ).toBe(1);
     void queue.push(highPriorityMsg);
+    expect(
+      await metricReader.getMetricValue("jazz.messagequeue.size", {
+        priority: highPriorityMsg.priority,
+      }),
+    ).toBe(1);
 
     expect(queue.pull()?.msg).toEqual(highPriorityMsg);
+    expect(
+      await metricReader.getMetricValue("jazz.messagequeue.size", {
+        priority: highPriorityMsg.priority,
+      }),
+    ).toBe(0);
     expect(queue.pull()?.msg).toEqual(mediumPriorityMsg);
+    expect(
+      await metricReader.getMetricValue("jazz.messagequeue.size", {
+        priority: mediumPriorityMsg.priority,
+      }),
+    ).toBe(0);
     expect(queue.pull()?.msg).toEqual(lowPriorityMsg);
+    expect(
+      await metricReader.getMetricValue("jazz.messagequeue.size", {
+        priority: lowPriorityMsg.priority,
+      }),
+    ).toBe(0);
   });
 
   test("should return undefined when pulling from empty queue", () => {

package/src/tests/coMap.test.ts

@@ -1,9 +1,10 @@
 import { expect, test } from "vitest";
 import { expectMap } from "../coValue.js";
+import { operationToEditEntry } from "../coValues/coMap.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { LocalNode } from "../localNode.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
-import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
+import { hotSleep, randomAnonymousAccountAndSessionID } from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -62,26 +63,40 @@ test("Can get CoMap entry values at different points in time", () => {
 
   expect(content.type).toEqual("comap");
 
-  const beforeA = Date.now();
-  while (Date.now() < beforeA + 10) {
-    /* hot sleep */
-  }
+  const beforeA = hotSleep(10);
   content.set("hello", "A", "trusting");
-  const beforeB = Date.now();
-  while (Date.now() < beforeB + 10) {
-    /* hot sleep */
-  }
+  const beforeB = hotSleep(10);
   content.set("hello", "B", "trusting");
-  const beforeC = Date.now();
-  while (Date.now() < beforeC + 10) {
-    /* hot sleep */
-  }
+  const beforeC = hotSleep(10);
   content.set("hello", "C", "trusting");
   expect(content.get("hello")).toEqual("C");
   expect(content.atTime(Date.now()).get("hello")).toEqual("C");
   expect(content.atTime(beforeA).get("hello")).toEqual(undefined);
   expect(content.atTime(beforeB).get("hello")).toEqual("A");
   expect(content.atTime(beforeC).get("hello")).toEqual("B");
+
+  const ops = content.timeFilteredOps("hello");
+
+  expect(content.atTime(beforeC).lastEditAt("hello")).toEqual(
+    operationToEditEntry(ops![1]!),
+  );
+  expect(content.atTime(beforeC).nthEditAt("hello", 0)).toEqual(
+    operationToEditEntry(ops![0]!),
+  );
+  expect(content.atTime(beforeC).nthEditAt("hello", 2)).toEqual(undefined);
+
+  expect([...content.atTime(beforeC).editsAt("hello")]).toEqual([
+    operationToEditEntry(ops![0]!),
+    operationToEditEntry(ops![1]!),
+  ]);
+
+  expect(content.atTime(beforeB).asObject()).toEqual({
+    hello: "A",
+  });
+
+  expect(content.atTime(beforeC).asObject()).toEqual({
+    hello: "B",
+  });
 });
 
 test("Can get all historic values of key in CoMap", () => {

package/src/tests/group.test.ts

@@ -7,10 +7,8 @@ import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { LocalNode } from "../localNode.js";
 import {
   createThreeConnectedNodes,
-  createTwoConnectedNodes,
   loadCoValueOrFail,
   randomAnonymousAccountAndSessionID,
-  waitFor,
 } from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();

package/src/tests/permissions.test.ts

@@ -7,6 +7,8 @@ import {
   createTwoConnectedNodes,
   groupWithTwoAdmins,
   groupWithTwoAdminsHighLevel,
+  hotSleep,
+  loadCoValueOrFail,
   newGroup,
   newGroupHighLevel,
 } from "./testUtils.js";
@@ -1802,29 +1804,25 @@ test("Admins can set child extensions", () => {
 });
 
 test("Admins can set child extensions when the admin role is inherited", async () => {
-  const { node1, node2 } = createTwoConnectedNodes("server", "server");
+  const { node1, node2 } = await createTwoConnectedNodes("server", "server");
 
-  const node2Account = node2.account;
-  const group = node1.createGroup();
+  const node2AccountOnNode1 = await loadCoValueOrFail(
+    node1.node,
+    node2.accountID,
+  );
 
-  group.addMember(node2Account, "admin");
+  const group = node1.node.createGroup();
 
-  const groupOnNode2 = await node2.load(group.id);
+  group.addMember(node2AccountOnNode1, "admin");
 
-  if (groupOnNode2 === "unavailable") {
-    throw new Error("Group not found on node2");
-  }
+  const groupOnNode2 = await loadCoValueOrFail(node2.node, group.id);
 
-  const childGroup = node2.createGroup();
+  const childGroup = node2.node.createGroup();
   childGroup.extend(groupOnNode2);
 
-  const childGroupOnNode1 = await node1.load(childGroup.id);
+  const childGroupOnNode1 = await loadCoValueOrFail(node1.node, childGroup.id);
 
-  if (childGroupOnNode1 === "unavailable") {
-    throw new Error("Child group not found on node1");
-  }
-
-  const grandChildGroup = node2.createGroup();
+  const grandChildGroup = node2.node.createGroup();
   grandChildGroup.extend(childGroupOnNode1);
 
   expect(childGroupOnNode1.get(`child_${grandChildGroup.id}`)).toEqual(
@@ -2315,6 +2313,38 @@ test("When rotating the key of a parent group, the keys of all child groups are
   expect(newChildReadKeyID).not.toEqual(currentChildReadKeyID);
 });
 
+test("When rotating the key of a parent group, the old transactions should still be valid", async () => {
+  const { node1, node2 } = await createTwoConnectedNodes("server", "server");
+
+  const group = node1.node.createGroup();
+  const parentGroup = node1.node.createGroup();
+
+  group.extend(parentGroup);
+
+  const node2AccountOnNode1 = await loadCoValueOrFail(
+    node1.node,
+    node2.accountID,
+  );
+
+  parentGroup.addMember(node2AccountOnNode1, "writer");
+
+  const map = group.createMap();
+  map.set("from", "node1", "private");
+
+  const mapOnNode2 = await loadCoValueOrFail(node2.node, map.id);
+  mapOnNode2.set("from", "node2", "private");
+
+  await new Promise((resolve) => setTimeout(resolve, 10));
+
+  parentGroup.removeMember(node2AccountOnNode1);
+
+  await new Promise((resolve) => setTimeout(resolve, 10));
+
+  const mapOnNode1 = await loadCoValueOrFail(node1.node, map.id);
+
+  expect(mapOnNode1.get("from")).toEqual("node2");
+});
+
 test("When rotating the key of a grand-parent group, the keys of all child and grand-child groups are also rotated", () => {
   const { group, node } = newGroupHighLevel();
   const grandParentGroup = node.createGroup();