assistant-cloud 0.2.0 → 0.2.1

package/src/tests/CloudEngagementReporter.test.ts

@@ -0,0 +1,184 @@
+import { describe, expect, it, vi } from "vitest";
+import type { AssistantCloud } from "../AssistantCloud";
+import {
+  CloudEngagementReporter,
+  type EngagementIdResolver,
+} from "../CloudEngagementReporter";
+
+const createCloud = () => {
+  const track = vi.fn();
+  const cloud = { events: { track } } as unknown as AssistantCloud;
+  return { cloud, track };
+};
+
+const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
+
+describe("CloudEngagementReporter", () => {
+  it("resolves ids through the resolver and awaits the thread only for a send", async () => {
+    const { cloud, track } = createCloud();
+    const resolveIds = vi.fn<EngagementIdResolver>(
+      async (threadId, messageId) => ({
+        thread_id: `remote-${threadId}`,
+        ...(messageId ? { message_id: `remote-${messageId}` } : undefined),
+      }),
+    );
+    const reporter = new CloudEngagementReporter(cloud, resolveIds);
+
+    reporter.messageSent("t1", { messageId: "m1", chars: 12, attachments: 1 });
+    reporter.messageCopied("t1", "m2");
+    await flush();
+
+    expect(resolveIds).toHaveBeenNthCalledWith(1, "t1", "m1", {
+      awaitThread: true,
+    });
+    expect(resolveIds).toHaveBeenNthCalledWith(2, "t1", "m2", {
+      awaitThread: false,
+    });
+    expect(track).toHaveBeenCalledWith({
+      kind: "message_sent",
+      thread_id: "remote-t1",
+      message_id: "remote-m1",
+      props: { chars: 12, attachments: 1 },
+    });
+    expect(track).toHaveBeenCalledWith({
+      kind: "message_copied",
+      thread_id: "remote-t1",
+      message_id: "remote-m2",
+    });
+  });
+
+  it("reports tool approval decisions with their resolved message IDs", async () => {
+    const { cloud, track } = createCloud();
+    const reporter = new CloudEngagementReporter(
+      cloud,
+      (threadId, messageId) => ({
+        thread_id: `remote-${threadId}`,
+        ...(messageId !== undefined
+          ? { message_id: `remote-${messageId}` }
+          : {}),
+      }),
+    );
+
+    reporter.toolApproved("t1", "m1", "tool-1", "send_email");
+    reporter.toolRejected("t1", "m2", "tool-2", "delete_account");
+    await flush();
+
+    expect(track).toHaveBeenNthCalledWith(1, {
+      kind: "tool_approved",
+      thread_id: "remote-t1",
+      message_id: "remote-m1",
+      props: { toolCallId: "tool-1", toolName: "send_email" },
+    });
+    expect(track).toHaveBeenNthCalledWith(2, {
+      kind: "tool_rejected",
+      thread_id: "remote-t1",
+      message_id: "remote-m2",
+      props: { toolCallId: "tool-2", toolName: "delete_account" },
+    });
+  });
+
+  it("measures a stop against the run start and reports it once per run", async () => {
+    const { cloud, track } = createCloud();
+    const reporter = new CloudEngagementReporter(cloud);
+
+    vi.setSystemTime(new Date("2023-01-01T00:00:00.000Z"));
+    reporter.runStopped("t1");
+    reporter.runStarted("t1");
+    vi.setSystemTime(new Date("2023-01-01T00:00:01.500Z"));
+    reporter.runStopped("t1");
+    reporter.runStopped("t1");
+    await flush();
+
+    expect(track).toHaveBeenCalledOnce();
+    expect(track).toHaveBeenCalledWith({
+      kind: "run_stopped",
+      thread_id: "t1",
+      value: 1500,
+    });
+  });
+
+  it("carries the time since the previous run ended on the next send", async () => {
+    const { cloud, track } = createCloud();
+    const reporter = new CloudEngagementReporter(cloud);
+
+    vi.setSystemTime(new Date("2023-01-01T00:00:00.000Z"));
+    reporter.runStarted("t1");
+    reporter.runEnded("t1");
+    vi.setSystemTime(new Date("2023-01-01T00:00:04.000Z"));
+    reporter.messageSent("t1", { chars: 3, attachments: 0 });
+    await flush();
+
+    expect(track).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: "message_sent", value: 4000 }),
+    );
+  });
+
+  it("shows one error per run and one suggestion list per thread", async () => {
+    const { cloud, track } = createCloud();
+    const reporter = new CloudEngagementReporter(cloud);
+
+    reporter.runStarted("t1");
+    reporter.errorShown("t1", { messageId: "m1", reason: "error" });
+    reporter.errorShown("t1", { messageId: "m1", reason: "error" });
+    reporter.runStarted("t1");
+    reporter.errorShown("t1", { reason: "timeout" });
+    reporter.suggestionsShown("t1", 3);
+    reporter.suggestionsShown("t1", 3);
+    await flush();
+
+    const kinds = track.mock.calls.map(([event]) => event.kind);
+    expect(kinds).toEqual(["error_shown", "error_shown", "suggestions_shown"]);
+    expect(track).toHaveBeenCalledWith({
+      kind: "suggestions_shown",
+      thread_id: "t1",
+      value: 3,
+    });
+  });
+
+  it("drops a thread switch the cloud cannot attribute and survives a failing resolver", async () => {
+    const { cloud, track } = createCloud();
+    const reporter = new CloudEngagementReporter(cloud, (threadId) =>
+      threadId === "known" ? { thread_id: "remote" } : {},
+    );
+    reporter.threadSwitched("unknown");
+    reporter.threadSwitched("known");
+    const failing = new CloudEngagementReporter(cloud, () => {
+      throw new Error("no ids");
+    });
+    failing.messageCopied("t1");
+    await flush();
+
+    expect(track).toHaveBeenCalledOnce();
+    expect(track).toHaveBeenCalledWith({
+      kind: "thread_switched",
+      thread_id: "remote",
+    });
+  });
+
+  it("keeps the state of the most recently touched threads only", async () => {
+    const { cloud, track } = createCloud();
+    const reporter = new CloudEngagementReporter(cloud);
+    for (let index = 0; index < 300; index++) {
+      reporter.suggestionsShown(`t${index}`, 1);
+    }
+    reporter.suggestionsShown("t0", 1);
+    reporter.suggestionsShown("t299", 1);
+    await flush();
+
+    expect(track).toHaveBeenCalledTimes(301);
+  });
+
+  it("keeps a started run until it stops, however many threads ran since", async () => {
+    const { cloud, track } = createCloud();
+    const reporter = new CloudEngagementReporter(cloud);
+    for (let index = 0; index < 300; index++) {
+      reporter.runStarted(`t${index}`);
+    }
+    reporter.runStopped("t0");
+    await flush();
+
+    expect(track).toHaveBeenCalledWith(
+      expect.objectContaining({ kind: "run_stopped", thread_id: "t0" }),
+    );
+  });
+});

package/src/tests/CloudMessagePersistence.test.ts

@@ -252,6 +252,217 @@ describe("CloudMessagePersistence", () => {
     await persistence.load("thread-1");
 
     expect(persistence.isPersisted("msg-1")).toBe(true);
+    expect(await persistence.getRemoteId("msg-1")).toBe("msg-1");
+  });
+
+  it("preserves a pending append mapping when load returns the same ID", async () => {
+    const messages = createCloudMessages(1);
+    let resolveLoad!: (value: { messages: typeof messages }) => void;
+    let resolveAppend!: (value: { message_id: string }) => void;
+    vi.mocked(cloud.threads.messages.list).mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          resolveLoad = resolve;
+        }),
+    );
+    vi.mocked(cloud.threads.messages.create).mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          resolveAppend = resolve;
+        }),
+    );
+
+    const load = persistence.load("thread-1");
+    const append = persistence.append(
+      "thread-1",
+      "message-1",
+      null,
+      "aui/v0",
+      {},
+    );
+    resolveLoad({ messages });
+    await load;
+    resolveAppend({ message_id: "remote-2" });
+    await append;
+
+    expect(await persistence.getRemoteId("message-1")).toBe("remote-2");
+    await persistence.update("thread-1", "message-1", "aui/v0", {
+      text: "updated",
+    });
+    expect(cloud.threads.messages.update).toHaveBeenCalledWith(
+      "thread-1",
+      "remote-2",
+      { content: { text: "updated" } },
+    );
+  });
+
+  it("preserves a resolved append mapping when load returns the same ID", async () => {
+    const messages = createCloudMessages(1);
+    vi.mocked(cloud.threads.messages.create).mockResolvedValue({
+      message_id: "remote-1",
+    });
+    vi.mocked(cloud.threads.messages.list).mockResolvedValue({
+      messages,
+    });
+
+    await persistence.append("thread-1", "message-1", null, "aui/v0", {});
+    await persistence.load("thread-1");
+
+    expect(await persistence.getRemoteId("message-1")).toBe("remote-1");
+  });
+
+  it("keeps the loaded ID when a pending append for it fails", async () => {
+    const messages = createCloudMessages(1);
+    const failure = new Error("create failed");
+    let resolveLoad!: (value: { messages: typeof messages }) => void;
+    let rejectAppend!: (error: Error) => void;
+    vi.mocked(cloud.threads.messages.list).mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          resolveLoad = resolve;
+        }),
+    );
+    vi.mocked(cloud.threads.messages.create).mockImplementationOnce(
+      () =>
+        new Promise((_resolve, reject) => {
+          rejectAppend = reject;
+        }),
+    );
+
+    const load = persistence.load("thread-1");
+    const append = persistence.append(
+      "thread-1",
+      "message-1",
+      null,
+      "aui/v0",
+      {},
+    );
+    resolveLoad({ messages });
+    await load;
+    rejectAppend(failure);
+
+    await expect(append).rejects.toBe(failure);
+    expect(persistence.isPersisted("message-1")).toBe(true);
+    expect(await persistence.getRemoteId("message-1")).toBe("message-1");
+  });
+
+  it("does not restore a loaded ID after reset when its pending append fails", async () => {
+    const messages = createCloudMessages(1);
+    const failure = new Error("create failed");
+    let rejectAppend!: (error: Error) => void;
+    vi.mocked(cloud.threads.messages.list).mockResolvedValue({ messages });
+    vi.mocked(cloud.threads.messages.create).mockImplementationOnce(
+      () =>
+        new Promise((_resolve, reject) => {
+          rejectAppend = reject;
+        }),
+    );
+
+    const append = persistence.append(
+      "thread-1",
+      "message-1",
+      null,
+      "aui/v0",
+      {},
+    );
+    await persistence.load("thread-1");
+    persistence.reset();
+    rejectAppend(failure);
+
+    await expect(append).rejects.toBe(failure);
+    expect(persistence.isPersisted("message-1")).toBe(false);
+  });
+
+  it("does not restore IDs from a load that finishes after reset", async () => {
+    const oldMessages = createCloudMessages(1);
+    const newMessages = [{ ...oldMessages[0]!, id: "new-message" }];
+    let resolveOld!: (value: { messages: typeof oldMessages }) => void;
+    vi.mocked(cloud.threads.messages.list)
+      .mockImplementationOnce(
+        () =>
+          new Promise((resolve) => {
+            resolveOld = resolve;
+          }),
+      )
+      .mockResolvedValueOnce({ messages: newMessages });
+
+    const oldLoad = persistence.load("old-thread");
+    persistence.reset();
+    await persistence.load("new-thread");
+    resolveOld({ messages: oldMessages });
+
+    expect(await oldLoad).toEqual(oldMessages);
+    expect(persistence.isPersisted("message-1")).toBe(false);
+    expect(persistence.isPersisted("new-message")).toBe(true);
+  });
+
+  it("keeps a new append mapping when an old load returns the same ID", async () => {
+    const messages = createCloudMessages(1);
+    let resolveOld!: (value: { messages: typeof messages }) => void;
+    vi.mocked(cloud.threads.messages.list).mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          resolveOld = resolve;
+        }),
+    );
+    vi.mocked(cloud.threads.messages.create).mockResolvedValue({
+      message_id: "new-remote-id",
+    });
+
+    const oldLoad = persistence.load("old-thread");
+    persistence.reset();
+    await persistence.append("new-thread", "message-1", null, "aui/v0", {});
+    resolveOld({ messages });
+    await oldLoad;
+
+    expect(await persistence.getRemoteId("message-1")).toBe("new-remote-id");
+  });
+
+  it("does not populate any old IDs when reset happens between history pages", async () => {
+    const messages = createCloudMessages(201);
+    let resolveLast!: (value: { messages: typeof messages }) => void;
+    vi.mocked(cloud.threads.messages.list)
+      .mockResolvedValueOnce({ messages: messages.slice(0, 200) })
+      .mockImplementationOnce(
+        () =>
+          new Promise((resolve) => {
+            resolveLast = resolve;
+          }),
+      );
+
+    const load = persistence.load("old-thread");
+    await vi.waitFor(() =>
+      expect(cloud.threads.messages.list).toHaveBeenCalledTimes(2),
+    );
+    persistence.reset();
+    resolveLast({ messages: messages.slice(200) });
+
+    expect(await load).toEqual(messages);
+    expect(
+      messages.some((message) => persistence.isPersisted(message.id)),
+    ).toBe(false);
+  });
+
+  it("propagates an old load failure without disturbing post-reset state", async () => {
+    let rejectOld!: (error: Error) => void;
+    vi.mocked(cloud.threads.messages.list).mockImplementationOnce(
+      () =>
+        new Promise((_resolve, reject) => {
+          rejectOld = reject;
+        }),
+    );
+    vi.mocked(cloud.threads.messages.create).mockResolvedValue({
+      message_id: "new-remote-id",
+    });
+    const failure = new Error("history failed");
+    const oldLoad = persistence.load("old-thread");
+    const rejected = expect(oldLoad).rejects.toBe(failure);
+    persistence.reset();
+    await persistence.append("new-thread", "new-message", null, "aui/v0", {});
+    rejectOld(failure);
+
+    await rejected;
+    expect(await persistence.getRemoteId("new-message")).toBe("new-remote-id");
   });
 
   it("maps prototype-named loaded messages", async () => {
@@ -383,6 +594,30 @@ describe("CloudMessagePersistence", () => {
     expect(persistence.isPersisted("local-1")).toBe(false);
   });
 
+  it("does not restore an append mapping after reset", async () => {
+    let resolveAppend!: (value: { message_id: string }) => void;
+    vi.mocked(cloud.threads.messages.create).mockImplementationOnce(
+      () =>
+        new Promise((resolve) => {
+          resolveAppend = resolve;
+        }),
+    );
+
+    const append = persistence.append(
+      "thread-1",
+      "local-1",
+      null,
+      "aui/v0",
+      {},
+    );
+    persistence.reset();
+    resolveAppend({ message_id: "remote-1" });
+
+    await expect(append).resolves.toBeUndefined();
+    expect(persistence.isPersisted("local-1")).toBe(false);
+    expect(await persistence.getRemoteId("local-1")).toBeUndefined();
+  });
+
   it("reset clears all ID mappings", async () => {
     vi.mocked(cloud.threads.messages.create).mockResolvedValue({
       message_id: "remote-1",

package/src/tests/CloudRunReporter.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, it, vi } from "vitest";
+import type { AssistantCloud } from "../AssistantCloud";
+import { CloudRunReporter } from "../CloudRunReporter";
+
+const createCloud = (telemetry: AssistantCloud["telemetry"]) => {
+  const report = vi.fn().mockResolvedValue({ run_id: "run_1" });
+  const cloud = { telemetry, runs: { report } } as unknown as AssistantCloud;
+  return { cloud, report };
+};
+
+describe("CloudRunReporter", () => {
+  it("sends nothing while telemetry is disabled", async () => {
+    const { cloud, report } = createCloud({ enabled: false });
+    await new CloudRunReporter(cloud).report({
+      threadId: "thread_1",
+      status: "completed",
+    });
+    expect(report).not.toHaveBeenCalled();
+  });
+
+  it("stamps the cloud's environment, release and tags and applies beforeReport last", async () => {
+    const beforeReport = vi.fn((report) => ({ ...report, model_id: "gpt" }));
+    const { cloud, report } = createCloud({
+      enabled: true,
+      environment: "production",
+      release: "1.2.3",
+      tags: ["web"],
+      beforeReport,
+    });
+    await new CloudRunReporter(cloud).report({
+      threadId: "thread_1",
+      status: "completed",
+      outputText: "hi",
+    });
+    expect(beforeReport).toHaveBeenCalledWith(
+      expect.objectContaining({
+        thread_id: "thread_1",
+        environment: "production",
+        release: "1.2.3",
+        tags: ["web"],
+      }),
+    );
+    expect(report).toHaveBeenCalledWith(
+      expect.objectContaining({ thread_id: "thread_1", model_id: "gpt" }),
+    );
+  });
+
+  it("skips a report the hook vetoes without consuming its key", async () => {
+    const { cloud, report } = createCloud({
+      enabled: true,
+      beforeReport: vi
+        .fn()
+        .mockReturnValueOnce(null)
+        .mockImplementation((r) => r),
+    });
+    const reporter = new CloudRunReporter(cloud);
+    await reporter.report({ threadId: "t", status: "completed" }, "t:m");
+    await reporter.report({ threadId: "t", status: "completed" }, "t:m");
+    expect(report).toHaveBeenCalledOnce();
+  });
+
+  it("reports a keyed run once and an unkeyed run every time", async () => {
+    const { cloud, report } = createCloud({ enabled: true });
+    const reporter = new CloudRunReporter(cloud);
+    await reporter.report({ threadId: "t", status: "completed" }, "t:m");
+    await reporter.report({ threadId: "t", status: "completed" }, "t:m");
+    await reporter.report({ threadId: "t", status: "completed" });
+    await reporter.report({ threadId: "t", status: "completed" });
+    expect(report).toHaveBeenCalledTimes(3);
+  });
+
+  it("swallows a failed send and reads the cloud through a getter", async () => {
+    const { cloud } = createCloud({ enabled: true });
+    (cloud.runs.report as ReturnType<typeof vi.fn>).mockRejectedValue(
+      new Error("offline"),
+    );
+    await expect(
+      new CloudRunReporter(() => cloud).report({
+        threadId: "t",
+        status: "error",
+      }),
+    ).resolves.toBeUndefined();
+  });
+
+  it("resolves when the hook throws and reports nothing", async () => {
+    const { cloud, report } = createCloud({
+      enabled: true,
+      beforeReport: () => {
+        throw new Error("hook");
+      },
+    });
+    await expect(
+      new CloudRunReporter(cloud).report({
+        threadId: "t",
+        status: "completed",
+      }),
+    ).resolves.toBeUndefined();
+    expect(report).not.toHaveBeenCalled();
+  });
+});

package/src/version.ts

@@ -0,0 +1,4 @@
+export const ASSISTANT_CLOUD_VERSION: string =
+  typeof __AUI_PACKAGE_VERSION__ === "string"
+    ? __AUI_PACKAGE_VERSION__
+    : "0.0.0";