cojson 0.20.17 → 0.20.19

package/src/tests/clockDrift.integration.test.ts

@@ -0,0 +1,261 @@
+import { afterEach, describe, expect, test, vi } from "vitest";
+import { logger } from "../logger.js";
+import {
+  loadCoValueOrFail,
+  setupTestAccount,
+  setupTestNode,
+  waitFor,
+} from "./testUtils.js";
+
+const CLIENT_SKEW_MS = 20_000;
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe("clock drift across peers with clock sync enabled", () => {
+  test("worker can publish and self-remove after a skewed client with clock sync creates group, grant and chat", async () => {
+    const errorSpy = vi.spyOn(logger, "error");
+
+    const realNow = Math.floor(performance.timeOrigin + performance.now());
+
+    vi.spyOn(Date, "now").mockImplementation(() => {
+      const real = performance.timeOrigin + performance.now();
+      return Math.floor(real) + CLIENT_SKEW_MS;
+    });
+
+    const worker = await setupTestAccount({
+      isSyncServer: true,
+    });
+
+    const client = await setupTestAccount({
+      connected: true,
+      experimental_clockSyncFromServerPings: true,
+    });
+
+    const skewedNow = Date.now();
+    client.node.clockOffset.addSample({
+      serverTime: skewedNow - CLIENT_SKEW_MS,
+      localReceiveTime: skewedNow,
+    });
+
+    const group = client.node.createGroup();
+
+    const workerAccountOnClient = await loadCoValueOrFail(
+      client.node,
+      worker.accountID,
+    );
+    group.addMember(workerAccountOnClient, "admin");
+
+    const oneToOneChat = group.createMap();
+    oneToOneChat.set("kind", "OneToOneChat", "trusting");
+    oneToOneChat.set("published", false, "trusting");
+
+    await oneToOneChat.core.waitForSync();
+    await group.core.waitForSync();
+
+    const clientGroupTxs = group.core.getValidSortedTransactions();
+    const maxClientGroupMadeAt = Math.max(
+      ...clientGroupTxs.map((tx) => tx.madeAt),
+    );
+    expect(maxClientGroupMadeAt).toBeLessThan(realNow + CLIENT_SKEW_MS - 5_000);
+
+    vi.restoreAllMocks();
+    const errorSpyAfter = vi.spyOn(logger, "error");
+
+    const chatOnWorker = await loadCoValueOrFail(worker.node, oneToOneChat.id);
+    const groupOnWorker = await loadCoValueOrFail(worker.node, group.id);
+    const workerAccountOnWorker = await loadCoValueOrFail(
+      worker.node,
+      worker.accountID,
+    );
+
+    await waitFor(() => {
+      expect(chatOnWorker.get("kind")).toBe("OneToOneChat");
+      expect(groupOnWorker.roleOf(worker.accountID)).toBe("admin");
+    });
+
+    chatOnWorker.set("published", true, "trusting");
+    groupOnWorker.removeMember(workerAccountOnWorker);
+
+    await chatOnWorker.core.waitForSync();
+    await groupOnWorker.core.waitForSync();
+
+    await waitFor(() => {
+      expect(oneToOneChat.get("published")).toBe(true);
+      expect(group.roleOf(worker.accountID)).not.toBe("admin");
+      expect(chatOnWorker.get("published")).toBe(true);
+      expect(groupOnWorker.roleOf(worker.accountID)).not.toBe("admin");
+    });
+
+    expect(oneToOneChat.get("published")).toBe(true);
+    expect(chatOnWorker.get("published")).toBe(true);
+    expect(group.roleOf(worker.accountID)).not.toBe("admin");
+    expect(groupOnWorker.roleOf(worker.accountID)).not.toBe("admin");
+
+    const forbiddenFragments = [
+      "invalid transaction",
+      "permission",
+      "rejected",
+      "not authorized",
+      "not authorised",
+    ];
+    const offendingCalls = [...errorSpy.mock.calls, ...errorSpyAfter.mock.calls]
+      .map((call) => call.map((arg) => String(arg)).join(" "))
+      .filter((line) =>
+        forbiddenFragments.some((f) => line.toLowerCase().includes(f)),
+      );
+    expect(offendingCalls).toEqual([]);
+  });
+});
+
+describe("experimental_clockSyncFromServerPings flag wiring", () => {
+  function seedOffset(
+    clockOffset: {
+      addSample: (s: { serverTime: number; localReceiveTime: number }) => void;
+    },
+    offsetMs: number,
+  ) {
+    const localReceiveTime = Date.now();
+    clockOffset.addSample({
+      serverTime: localReceiveTime + offsetMs,
+      localReceiveTime,
+    });
+  }
+
+  test("with the flag on, a seeded +10_000 ms offset pulls locally-stamped madeAt forward", () => {
+    const { node } = setupTestNode({
+      experimental_clockSyncFromServerPings: true,
+    });
+
+    seedOffset(node.clockOffset, 10_000);
+
+    const group = node.createGroup();
+    const map = group.createMap();
+
+    const before = Date.now();
+    map.set("k", "v", "trusting");
+    const after = Date.now();
+
+    const txs = map.core.getValidSortedTransactions();
+    const lastTx = txs.at(-1);
+    expect(lastTx).toBeDefined();
+    expect(lastTx!.madeAt).toBeGreaterThanOrEqual(before + 9_900);
+    expect(lastTx!.madeAt).toBeLessThanOrEqual(after + 10_100);
+  });
+
+  test("without the flag, a seeded +10_000 ms offset does NOT shift locally-stamped madeAt", () => {
+    const { node } = setupTestNode({
+      experimental_clockSyncFromServerPings: false,
+    });
+
+    seedOffset(node.clockOffset, 10_000);
+
+    const group = node.createGroup();
+    const map = group.createMap();
+
+    const before = Date.now();
+    map.set("k", "v", "trusting");
+    const after = Date.now();
+
+    const txs = map.core.getValidSortedTransactions();
+    const lastTx = txs.at(-1);
+    expect(lastTx).toBeDefined();
+    expect(lastTx!.madeAt).toBeGreaterThanOrEqual(before);
+    expect(lastTx!.madeAt).toBeLessThanOrEqual(after + 100);
+  });
+});
+
+describe("clock sync pulls skewed client stamps toward server time", () => {
+  test("with both nodes flag-on, a client whose wall clock is 20s ahead authors transactions that land near real time on the worker", async () => {
+    const errorSpy = vi.spyOn(logger, "error");
+
+    const worker = await setupTestAccount({
+      isSyncServer: true,
+      experimental_clockSyncFromServerPings: true,
+    });
+
+    const realNowBeforeClient = Date.now();
+
+    const SKEW_MS = 20_000;
+    vi.spyOn(Date, "now").mockImplementation(() => {
+      const real = Math.floor(performance.timeOrigin + performance.now());
+      return real + SKEW_MS;
+    });
+
+    const client = await setupTestAccount({
+      connected: true,
+      experimental_clockSyncFromServerPings: true,
+    });
+
+    const localReceiveTime = Date.now();
+    client.node.clockOffset.addSample({
+      serverTime: localReceiveTime - SKEW_MS,
+      localReceiveTime,
+    });
+
+    expect(client.node.clockOffset.currentOffset()).toBeLessThanOrEqual(
+      -SKEW_MS + 100,
+    );
+    expect(client.node.clockOffset.currentOffset()).toBeGreaterThanOrEqual(
+      -SKEW_MS - 100,
+    );
+
+    const group = client.node.createGroup();
+    group.addMember("everyone", "writer");
+
+    const map = group.createMap();
+    map.set("from", "skewed-client", "trusting");
+
+    await map.core.waitForSync();
+    await group.core.waitForSync();
+
+    vi.restoreAllMocks();
+    const errorSpyAfter = vi.spyOn(logger, "error");
+
+    const realNowAfter = Date.now();
+
+    const mapOnWorker = await loadCoValueOrFail(worker.node, map.id);
+
+    await waitFor(() => {
+      expect(mapOnWorker.get("from")).toBe("skewed-client");
+    });
+
+    const clientTxsOnWorker = mapOnWorker.core.getValidSortedTransactions();
+    const setTx = clientTxsOnWorker.find((tx) => {
+      const changes = tx.changes as ReadonlyArray<{
+        op?: string;
+        key?: string;
+        value?: unknown;
+      }>;
+      return (
+        changes?.[0]?.key === "from" && changes[0]?.value === "skewed-client"
+      );
+    });
+
+    expect(setTx).toBeDefined();
+
+    const TOLERANCE_MS = 2_000;
+    expect(setTx!.madeAt).toBeGreaterThanOrEqual(
+      realNowBeforeClient - TOLERANCE_MS,
+    );
+    expect(setTx!.madeAt).toBeLessThanOrEqual(realNowAfter + TOLERANCE_MS);
+
+    expect(setTx!.madeAt).toBeLessThan(realNowAfter + SKEW_MS - 5_000);
+
+    const forbiddenFragments = [
+      "invalid transaction",
+      "permission",
+      "rejected",
+      "not authorized",
+      "not authorised",
+      "out of order",
+    ];
+    const offendingCalls = [...errorSpy.mock.calls, ...errorSpyAfter.mock.calls]
+      .map((call) => call.map((arg) => String(arg)).join(" "))
+      .filter((line) =>
+        forbiddenFragments.some((f) => line.toLowerCase().includes(f)),
+      );
+    expect(offendingCalls).toEqual([]);
+  });
+});

package/src/tests/localNode.clockOffset.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, test } from "vitest";
+import { ClockOffset } from "../ClockOffset.js";
+import { LocalNode } from "../localNode.js";
+import { randomAgentAndSessionID } from "./testUtils.js";
+import { WasmCrypto } from "../crypto/WasmCrypto.js";
+
+const Crypto = await WasmCrypto.create();
+
+function makeNode(opts: { experimental_clockSyncFromServerPings?: boolean }) {
+  const [admin, session] = randomAgentAndSessionID();
+  return new LocalNode(
+    admin.agentSecret,
+    session,
+    Crypto,
+    undefined,
+    undefined,
+    opts,
+  );
+}
+
+describe("LocalNode clock offset wiring", () => {
+  test("clockOffset is always present on the node, regardless of flag state", () => {
+    const withFlag = makeNode({ experimental_clockSyncFromServerPings: true });
+    const withoutFlag = makeNode({
+      experimental_clockSyncFromServerPings: false,
+    });
+
+    expect(withFlag.clockOffset).toBeInstanceOf(ClockOffset);
+    expect(withoutFlag.clockOffset).toBeInstanceOf(ClockOffset);
+  });
+
+  test("with the flag enabled, seeding clockOffset with a +10_000 ms sample pulls stampNow() forward by ~10_000 ms", () => {
+    const flagged = makeNode({ experimental_clockSyncFromServerPings: true });
+    const control = makeNode({
+      experimental_clockSyncFromServerPings: false,
+    });
+
+    const localReceiveTime = Date.now();
+    flagged.clockOffset.addSample({
+      serverTime: localReceiveTime + 10_000,
+      localReceiveTime,
+    });
+
+    const controlStamp = control.stampNow();
+    const flaggedStamp = flagged.stampNow();
+
+    const delta = flaggedStamp - controlStamp;
+    expect(delta).toBeGreaterThanOrEqual(9_900);
+    expect(delta).toBeLessThanOrEqual(10_100);
+  });
+
+  test("getClockOffsetDiagnostics() reflects current offset and sample count", () => {
+    const node = makeNode({ experimental_clockSyncFromServerPings: true });
+
+    expect(node.getClockOffsetDiagnostics()).toEqual({
+      currentOffset: 0,
+      sampleCount: 0,
+    });
+
+    const base = Date.now();
+    node.clockOffset.addSample({
+      serverTime: base + 400,
+      localReceiveTime: base,
+    });
+    node.clockOffset.addSample({
+      serverTime: base + 600,
+      localReceiveTime: base,
+    });
+
+    expect(node.getClockOffsetDiagnostics()).toEqual({
+      currentOffset: 500,
+      sampleCount: 2,
+    });
+  });
+
+  test("without the flag, seeding clockOffset with a +10_000 ms sample does NOT affect stampNow()", () => {
+    const node = makeNode({ experimental_clockSyncFromServerPings: false });
+
+    const localReceiveTime = Date.now();
+    node.clockOffset.addSample({
+      serverTime: localReceiveTime + 10_000,
+      localReceiveTime,
+    });
+
+    const before = Date.now();
+    const stamp = node.stampNow();
+    const after = Date.now();
+
+    expect(stamp).toBeGreaterThanOrEqual(before);
+    expect(stamp).toBeLessThanOrEqual(after + 100);
+  });
+});

package/src/tests/sync.localStoreDurability.test.ts

@@ -0,0 +1,357 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+import { WasmCrypto } from "../crypto/WasmCrypto.js";
+import { LocalNode } from "../localNode";
+import {
+  SyncMessagesLog,
+  getSyncServerConnectedPeer,
+  loadCoValueOrFail,
+  setupTestNode,
+  waitFor,
+} from "./testUtils";
+import { getDbPath, registerStorageCleanupRunner } from "./testStorage";
+
+const Crypto = await WasmCrypto.create();
+
+let jazzCloud: ReturnType<typeof setupTestNode>;
+
+beforeEach(async () => {
+  registerStorageCleanupRunner();
+  SyncMessagesLog.clear();
+  jazzCloud = setupTestNode({ isSyncServer: true });
+});
+
+describe("StorageAPI.store done callback", () => {
+  test("async storage invokes done after the write completes", async () => {
+    const client = setupTestNode();
+    const group = client.node.createGroup();
+    // Attach storage after creating the group so our manual store is the only write
+    const { storage } = await client.addAsyncStorage();
+
+    const content = group.core.newContentSince(undefined)!;
+    const done = vi.fn();
+
+    storage.store(content[0]!, () => undefined, done);
+
+    expect(done).not.toHaveBeenCalled();
+    await waitFor(() => expect(done).toHaveBeenCalledTimes(1));
+  });
+
+  test("sync storage invokes done synchronously", async () => {
+    const client = setupTestNode();
+    const group = client.node.createGroup();
+    const { storage } = client.addStorage();
+
+    const content = group.core.newContentSince(undefined)!;
+    const done = vi.fn();
+
+    storage.store(content[0]!, () => undefined, done);
+
+    expect(done).toHaveBeenCalledTimes(1);
+  });
+
+  test("async storage invokes done after a successful correction round-trip", async () => {
+    const client = setupTestNode();
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("a", 1, "trusting");
+    const { storage } = await client.addAsyncStorage();
+
+    const fullContent = map.core.newContentSince(undefined)!;
+    // Content without the header, stored into empty storage → triggers a correction
+    const withoutHeader = map.core.newContentSince({
+      id: map.id,
+      header: true,
+      sessions: {},
+    })!;
+    const done = vi.fn();
+
+    storage.store(withoutHeader[0]!, () => fullContent, done);
+
+    await waitFor(() => expect(done).toHaveBeenCalledTimes(1));
+  });
+});
+
+describe("local store durability window", () => {
+  test("opens before the first peer send and closes when async storage drains", async () => {
+    const client = setupTestNode();
+    await client.addAsyncStorage();
+    const { peerState } = client.connectToSyncServer();
+
+    // Instrument the actual outgoing push: the window must open before any
+    // content message physically leaves the node
+    let contentSent = false;
+    const originalPush = peerState.pushOutgoingMessage.bind(peerState);
+    vi.spyOn(peerState, "pushOutgoingMessage").mockImplementation((msg) => {
+      if (msg.action === "content") {
+        contentSent = true;
+      }
+      return originalPush(msg);
+    });
+
+    const events: Array<{ hasPending: boolean; contentSentAlready: boolean }> =
+      [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) => {
+      events.push({ hasPending, contentSentAlready: contentSent });
+    };
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await waitFor(() => expect(events.some((e) => !e.hasPending)).toBe(true));
+
+    // The window opened BEFORE any content was sent to a peer
+    expect(events[0]).toEqual({ hasPending: true, contentSentAlready: false });
+    // ...and eventually closed once storage drained
+    expect(events.at(-1)!.hasPending).toBe(false);
+  });
+
+  test("reopens for a new local edit after the previous window closed", async () => {
+    const client = setupTestNode();
+    await client.addAsyncStorage();
+    client.connectToSyncServer();
+
+    const events: boolean[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) => {
+      events.push(hasPending);
+    };
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // First window: opened and closed once storage drained
+    await waitFor(() => expect(events).toEqual([true, false]));
+
+    // A second local edit reopens the window and it closes again on drain
+    map.set("second", "batch", "trusting");
+
+    await waitFor(() => expect(events).toEqual([true, false, true, false]));
+  });
+
+  test("reports the node's current session ID", async () => {
+    const client = setupTestNode();
+    await client.addAsyncStorage();
+    client.connectToSyncServer();
+
+    const sessions: string[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (
+      _hasPending,
+      sessionID,
+    ) => {
+      sessions.push(sessionID);
+    };
+
+    const group = client.node.createGroup();
+    group.createMap().set("hello", "world", "trusting");
+
+    await waitFor(() => expect(sessions.length).toBeGreaterThan(0));
+    expect(sessions.every((s) => s === client.node.currentSessionID)).toBe(
+      true,
+    );
+  });
+
+  test("does not fire with synchronous storage", async () => {
+    const client = setupTestNode();
+    client.addStorage();
+    client.connectToSyncServer();
+
+    const listener = vi.fn();
+    client.node.syncManager.onLocalStoreDurabilityChange = listener;
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+    expect(listener).not.toHaveBeenCalled();
+  });
+
+  test("does not fire when there are no peers to send to", async () => {
+    const client = setupTestNode(); // not connected
+    await client.addAsyncStorage();
+
+    const listener = vi.fn();
+    client.node.syncManager.onLocalStoreDurabilityChange = listener;
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await client.node.syncManager.waitForStorageSync(map.id);
+    expect(listener).not.toHaveBeenCalled();
+  });
+
+  test("remote content does not open the window on the receiving node", async () => {
+    const server = setupTestNode({ isSyncServer: true });
+    await server.addAsyncStorage({ ourName: "server" });
+
+    const listener = vi.fn();
+    server.node.syncManager.onLocalStoreDurabilityChange = listener;
+
+    const client = setupTestNode({ connected: true });
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+    expect(listener).not.toHaveBeenCalled();
+  });
+
+  test("window stays open when storage writes never complete (crash window)", async () => {
+    const client = setupTestNode();
+    const { storage } = await client.addAsyncStorage();
+    const { peerState } = client.connectToSyncServer();
+
+    // Simulate the crash window: storage accepts writes but never completes them
+    storage.store = async () => {};
+
+    const events: boolean[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) =>
+      events.push(hasPending);
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Wait for the content to reach the server (peer sync only) — storage sync
+    // never resolves in this scenario since store() is stubbed to a no-op, so
+    // we can't use map.core.waitForSync() here, which also waits on storage.
+    await client.node.syncManager.waitForSyncWithPeer(
+      peerState.id,
+      map.id,
+      5000,
+    );
+    expect(events).toEqual([true]); // opened, never closed
+  });
+
+  test("opens when pending content is sent through reconnection reconciliation", async () => {
+    const client = setupTestNode();
+    const { storage } = await client.addAsyncStorage();
+
+    // Writes never complete: the local edit below stays pending forever
+    storage.store = async () => {};
+
+    const events: boolean[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) =>
+      events.push(hasPending);
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Not connected yet: nothing was sent, so the window must not be open
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    expect(events).toEqual([]);
+
+    // Connecting sends the pending content through reconciliation
+    // (sendNewContent), not through syncContent — the window must still open
+    const { peerState } = client.connectToSyncServer();
+    await client.node.syncManager.waitForSyncWithPeer(
+      peerState.id,
+      map.id,
+      5000,
+    );
+
+    expect(events).toEqual([true]);
+  });
+});
+
+describe("LocalNode creation options", () => {
+  test("withNewlyCreatedAccount wires onLocalStoreDurabilityChange into the node's syncManager", async () => {
+    const listener = vi.fn();
+
+    const { node } = await LocalNode.withNewlyCreatedAccount({
+      creationProps: { name: "test" },
+      crypto: Crypto,
+      peers: [],
+      onLocalStoreDurabilityChange: listener,
+    });
+
+    expect(node.syncManager.onLocalStoreDurabilityChange).toBe(listener);
+    await node.gracefulShutdown();
+  });
+
+  test("withLoadedAccount wires onLocalStoreDurabilityChange", async () => {
+    const listener = vi.fn();
+
+    const created = await LocalNode.withNewlyCreatedAccount({
+      creationProps: { name: "test" },
+      crypto: Crypto,
+      peers: [],
+    });
+
+    const { peer } = getSyncServerConnectedPeer({
+      peerId: created.accountID,
+    });
+
+    // Sync the account to the server so it can be loaded
+    created.node.syncManager.addPeer(peer);
+    await created.node.syncManager.waitForAllCoValuesSync();
+
+    const { peer: peer2 } = getSyncServerConnectedPeer({
+      peerId: "loadingNode",
+    });
+
+    const node = await LocalNode.withLoadedAccount({
+      accountID: created.accountID,
+      accountSecret: created.accountSecret,
+      sessionID: undefined,
+      peers: [peer2],
+      crypto: Crypto,
+      onLocalStoreDurabilityChange: listener,
+    });
+
+    expect(node.syncManager.onLocalStoreDurabilityChange).toBe(listener);
+    await node.gracefulShutdown();
+    await created.node.gracefulShutdown();
+  });
+});
+
+describe("crash recovery with a fresh session", () => {
+  test("restarting with a new session over the same storage recovers and syncs cleanly", async () => {
+    const client = setupTestNode();
+    const dbPath = getDbPath();
+    const { storage } = await client.addAsyncStorage({ filename: dbPath });
+    const { peerState } = client.connectToSyncServer();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+    await map.core.waitForSync();
+
+    // Enter the crash window: further writes reach the server but not storage
+    storage.store = async () => {};
+    map.set("crashed", "yes", "trusting");
+    await client.node.syncManager.waitForSyncWithPeer(
+      peerState.id,
+      map.id,
+      5000,
+    );
+
+    // "Crash": abandon the node without flushing (no graceful shutdown), then
+    // restart with the SAME agent, SAME storage file, but a NEW session — which
+    // is exactly what the session providers do when the durability marker is set.
+    client.disconnect();
+    const restarted = setupTestNode({ secret: client.node.agentSecret });
+    await restarted.addAsyncStorage({ filename: dbPath });
+    restarted.connectToSyncServer();
+
+    // The fresh session is load-bearing: reusing the crashed session could fork its hash chain
+    expect(restarted.node.currentSessionID).not.toBe(
+      client.node.currentSessionID,
+    );
+
+    // The crashed transaction comes back down from the server
+    const mapOnRestart = await loadCoValueOrFail(restarted.node, map.id);
+    await waitFor(() => expect(mapOnRestart.get("crashed")).toEqual("yes"));
+
+    // New writes from the fresh session sync cleanly — no signature rejection
+    mapOnRestart.set("recovered", "yes", "trusting");
+    await mapOnRestart.core.waitForSync();
+
+    const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
+    expect(mapOnServer.get("recovered")).toEqual("yes");
+    expect(mapOnServer.get("crashed")).toEqual("yes");
+  });
+});