@cursor/july 0.1.19 → 0.1.21

package/src/internal/ab-snapshot.test.ts

@@ -0,0 +1,325 @@
+import { describe, expect, it } from "vitest";
+import { defineAB, emptyABMetrics, splitAlways, splitNone } from "../ab.js";
+import type { SessionEvent, SessionRecord, SessionSummary } from "../types.js";
+import { foldABStream } from "./ab-fold.js";
+import { buildABSnapshot } from "./ab-snapshot.js";
+
+function event(
+  type: SessionEvent["type"],
+  index: number,
+  at: string,
+  data: Record<string, unknown> = {},
+  sessionId = "ses_1"
+): SessionEvent {
+  return {
+    type,
+    index,
+    at,
+    sessionId,
+    data,
+  } as SessionEvent;
+}
+
+describe("foldABStream", () => {
+  it("assigns arms and accumulates built-in metrics", () => {
+    const folded = foldABStream([
+      event("session.started", 0, "2026-01-01T00:00:00.000Z", {
+        channelId: "http",
+      }),
+      event("ab.assigned", 1, "2026-01-01T00:00:00.100Z", {
+        experiment: "exp",
+        variant: "treatment",
+      }),
+      event("turn.started", 2, "2026-01-01T00:00:01.000Z"),
+      event("action.result", 3, "2026-01-01T00:00:01.500Z", {
+        callId: "c1",
+        toolName: "get_weather",
+        isError: false,
+      }),
+      event("turn.completed", 4, "2026-01-01T00:00:02.000Z", {
+        usage: {
+          inputTokens: 10,
+          outputTokens: 5,
+          cacheReadTokens: 0,
+          cacheWriteTokens: 0,
+        },
+      }),
+    ]);
+    expect(folded.arms.get("exp")).toBe("treatment");
+    expect(folded.metrics.get("exp")?.metrics).toMatchObject({
+      turns: 1,
+      toolCalls: 1,
+      inputTokens: 10,
+      outputTokens: 5,
+      wallTimeMs: 1000,
+    });
+  });
+
+  it("sticky skip leaves no metrics bucket", () => {
+    const folded = foldABStream([
+      event("ab.assigned", 0, "2026-01-01T00:00:00.000Z", {
+        experiment: "exp",
+        variant: null,
+      }),
+      event("turn.completed", 1, "2026-01-01T00:00:01.000Z", {}),
+    ]);
+    expect(folded.arms.get("exp")).toBeNull();
+    expect(folded.metrics.has("exp")).toBe(false);
+  });
+});
+
+describe("buildABSnapshot", () => {
+  it("omits purpose:eval sessions even if the log has ab.assigned", async () => {
+    const def = defineAB({
+      name: "exp",
+      variants: { control: {}, treatment: {} },
+      split: splitAlways("treatment"),
+      onSample: () => {},
+    });
+    const snap = await buildABSnapshot({
+      experiments: [{ name: "exp", definition: def }],
+      agentName: "weather",
+      sessions: [
+        {
+          sessionId: "ses_eval",
+          channelId: "http",
+          mode: "chat",
+          turnCount: 1,
+          running: false,
+          createdAt: "2026-01-01T00:00:00.000Z",
+          updatedAt: "2026-01-01T00:00:01.000Z",
+          continuationToken: null,
+        },
+      ],
+      loadRecord: async () =>
+        ({
+          sessionId: "ses_eval",
+          channelId: "http",
+          mode: "chat",
+          purpose: "eval",
+          continuationKey: "k",
+          auth: null,
+          turnCount: 1,
+          createdAt: "2026-01-01T00:00:00.000Z",
+          updatedAt: "2026-01-01T00:00:01.000Z",
+        }) satisfies SessionRecord,
+      loadEvents: async () => [
+        event(
+          "ab.assigned",
+          0,
+          "2026-01-01T00:00:00.100Z",
+          { experiment: "exp", variant: "treatment" },
+          "ses_eval"
+        ),
+        event(
+          "turn.completed",
+          1,
+          "2026-01-01T00:00:01.000Z",
+          {
+            usage: {
+              inputTokens: 1,
+              outputTokens: 1,
+              cacheReadTokens: 0,
+              cacheWriteTokens: 0,
+            },
+          },
+          "ses_eval"
+        ),
+      ],
+      sessionInfo: (record) => ({
+        id: record.sessionId,
+        channelId: record.channelId,
+        mode: record.mode,
+        purpose: record.purpose,
+        auth: record.auth,
+        continuationKey: record.continuationKey,
+      }),
+      logger: () => {},
+    });
+    expect(snap.sessions).toEqual([]);
+    expect(snap.experiments[0]!.arms.every((a) => a.sessions === 0)).toBe(true);
+  });
+
+  it("aggregates per arm and lists per-session metrics", async () => {
+    const def = defineAB({
+      name: "exp",
+      variants: {
+        control: { label: "Baseline" },
+        treatment: { label: "Shorter" },
+      },
+      split: splitAlways("treatment"),
+      derive: {
+        weather: (e) =>
+          e.type === "action.result" && e.data.toolName === "get_weather"
+            ? 1
+            : null,
+      },
+      onSample: () => {},
+    });
+    const skipDef = defineAB({
+      name: "paused",
+      variants: { a: {}, b: {} },
+      split: splitNone(),
+      onSample: () => {},
+    });
+
+    const eventsBySession: Record<string, SessionEvent[]> = {
+      ses_a: [
+        event(
+          "ab.assigned",
+          0,
+          "2026-01-01T00:00:00.000Z",
+          { experiment: "exp", variant: "treatment" },
+          "ses_a"
+        ),
+        event(
+          "ab.assigned",
+          1,
+          "2026-01-01T00:00:00.050Z",
+          { experiment: "paused", variant: null },
+          "ses_a"
+        ),
+        event("turn.started", 2, "2026-01-01T00:00:01.000Z", {}, "ses_a"),
+        event(
+          "action.result",
+          3,
+          "2026-01-01T00:00:01.200Z",
+          { callId: "c1", toolName: "get_weather", isError: false },
+          "ses_a"
+        ),
+        event(
+          "turn.completed",
+          4,
+          "2026-01-01T00:00:02.000Z",
+          {
+            usage: {
+              inputTokens: 4,
+              outputTokens: 2,
+              cacheReadTokens: 0,
+              cacheWriteTokens: 0,
+            },
+          },
+          "ses_a"
+        ),
+      ],
+      ses_b: [
+        event(
+          "ab.assigned",
+          0,
+          "2026-01-01T00:00:00.000Z",
+          { experiment: "exp", variant: "control" },
+          "ses_b"
+        ),
+        event("turn.started", 1, "2026-01-01T00:00:01.000Z", {}, "ses_b"),
+        event(
+          "turn.completed",
+          2,
+          "2026-01-01T00:00:01.500Z",
+          {
+            usage: {
+              inputTokens: 1,
+              outputTokens: 1,
+              cacheReadTokens: 0,
+              cacheWriteTokens: 0,
+            },
+          },
+          "ses_b"
+        ),
+      ],
+    };
+
+    const summaries: SessionSummary[] = [
+      {
+        sessionId: "ses_a",
+        channelId: "http",
+        mode: "chat",
+        turnCount: 1,
+        running: false,
+        createdAt: "2026-01-01T00:00:00.000Z",
+        updatedAt: "2026-01-01T00:00:02.000Z",
+        continuationToken: null,
+      },
+      {
+        sessionId: "ses_b",
+        channelId: "http",
+        mode: "chat",
+        turnCount: 1,
+        running: false,
+        createdAt: "2026-01-01T00:00:00.000Z",
+        updatedAt: "2026-01-01T00:00:01.500Z",
+        continuationToken: null,
+      },
+    ];
+
+    const snap = await buildABSnapshot({
+      experiments: [
+        { name: "exp", definition: def },
+        { name: "paused", definition: skipDef },
+      ],
+      agentName: "weather",
+      sessions: summaries,
+      loadRecord: async (sessionId) =>
+        ({
+          sessionId,
+          channelId: "http",
+          mode: "chat",
+          continuationKey: "k",
+          auth: null,
+          turnCount: 1,
+          createdAt: "2026-01-01T00:00:00.000Z",
+          updatedAt: "2026-01-01T00:00:02.000Z",
+        }) satisfies SessionRecord,
+      loadEvents: async (sessionId) => eventsBySession[sessionId] ?? [],
+      sessionInfo: (record) => ({
+        id: record.sessionId,
+        channelId: record.channelId,
+        mode: record.mode,
+        auth: record.auth,
+        continuationKey: record.continuationKey,
+      }),
+      logger: () => {},
+    });
+
+    expect(snap.sessions).toHaveLength(2);
+    const sesA = snap.sessions[0]!;
+    expect(sesA.assignments).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          experiment: "exp",
+          variant: "treatment",
+          variantLabel: "Shorter",
+          metrics: expect.objectContaining({
+            turns: 1,
+            toolCalls: 1,
+            custom: { weather: 1 },
+          }),
+        }),
+        expect.objectContaining({
+          experiment: "paused",
+          variant: null,
+        }),
+      ])
+    );
+
+    const exp = snap.experiments.find((e) => e.name === "exp")!;
+    expect(exp.skipped).toBe(0);
+    const treatment = exp.arms.find((a) => a.variant === "treatment")!;
+    const control = exp.arms.find((a) => a.variant === "control")!;
+    expect(treatment.sessions).toBe(1);
+    expect(treatment.metrics.toolCalls).toBe(1);
+    expect(treatment.metrics.custom.weather).toBe(1);
+    expect(control.sessions).toBe(1);
+    expect(control.metrics.turns).toBe(1);
+    expect(control.metrics).toMatchObject({
+      ...emptyABMetrics(),
+      turns: 1,
+      inputTokens: 1,
+      outputTokens: 1,
+      wallTimeMs: 500,
+    });
+
+    const paused = snap.experiments.find((e) => e.name === "paused")!;
+    expect(paused.skipped).toBe(1);
+    expect(paused.arms.every((a) => a.sessions === 0)).toBe(true);
+  });
+});

package/src/internal/approval-gate.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+import { ApprovalGate } from "./approval-gate.js";
+
+describe("ApprovalGate", () => {
+  it("parks until resolve(approve)", async () => {
+    const gate = new ApprovalGate();
+    const waiting = gate.wait({
+      sessionId: "s1",
+      callId: "c1",
+      turnId: "turn_1",
+      toolName: "post_weather_alert",
+      args: { city: "NYC" },
+      requestedAt: new Date().toISOString(),
+    });
+    expect(gate.has("s1", "c1")).toBe(true);
+    expect(gate.list("s1")).toHaveLength(1);
+    expect(gate.resolve("s1", "c1", "approve")).toBe(true);
+    await expect(waiting).resolves.toEqual({ decision: "approve" });
+    expect(gate.has("s1", "c1")).toBe(false);
+  });
+
+  it("returns false when no live waiter", () => {
+    const gate = new ApprovalGate();
+    expect(gate.resolve("missing", "c1", "deny")).toBe(false);
+  });
+
+  it("cancelSession denies all waiters for a session", async () => {
+    const gate = new ApprovalGate();
+    const first = gate.wait({
+      sessionId: "s1",
+      callId: "a",
+      turnId: "turn_1",
+      toolName: "t",
+      args: {},
+      requestedAt: "2026-01-01T00:00:00.000Z",
+    });
+    const second = gate.wait({
+      sessionId: "s1",
+      callId: "a2",
+      turnId: "turn_1",
+      toolName: "t2",
+      args: {},
+      requestedAt: "2026-01-01T00:00:00.000Z",
+    });
+    gate.cancelSession("s1", "deny");
+    await expect(first).resolves.toEqual({ decision: "deny" });
+    await expect(second).resolves.toEqual({ decision: "deny" });
+  });
+});