ai-spec-dev 0.31.0 → 0.35.0

package/tests/provider-utils.test.ts

@@ -0,0 +1,173 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { ProviderError, withReliability } from "../core/provider-utils";
+
+// ─── ProviderError ────────────────────────────────────────────────────────────
+
+describe("ProviderError", () => {
+  it("sets name, kind, and message", () => {
+    const err = new ProviderError("bad key", "auth");
+    expect(err.name).toBe("ProviderError");
+    expect(err.kind).toBe("auth");
+    expect(err.message).toBe("bad key");
+    expect(err instanceof Error).toBe(true);
+  });
+
+  it("stores the original error", () => {
+    const original = new Error("root cause");
+    const err = new ProviderError("wrapped", "network", original);
+    expect(err.originalError).toBe(original);
+  });
+
+  it("originalError is undefined when not provided", () => {
+    const err = new ProviderError("msg", "timeout");
+    expect(err.originalError).toBeUndefined();
+  });
+
+  it("supports all error kinds", () => {
+    const kinds = ["auth", "rate_limit", "timeout", "network", "provider"] as const;
+    for (const kind of kinds) {
+      expect(new ProviderError("msg", kind).kind).toBe(kind);
+    }
+  });
+});
+
+// ─── withReliability — success ────────────────────────────────────────────────
+
+describe("withReliability — success path", () => {
+  it("returns result immediately on first success", async () => {
+    const fn = vi.fn().mockResolvedValue("hello");
+    const result = await withReliability(fn, { timeoutMs: 5_000, retries: 0 });
+    expect(result).toBe("hello");
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it("passes through arbitrary return types", async () => {
+    const payload = { id: 1, data: [1, 2, 3] };
+    const fn = vi.fn().mockResolvedValue(payload);
+    const result = await withReliability(fn, { timeoutMs: 5_000, retries: 0 });
+    expect(result).toEqual(payload);
+  });
+});
+
+// ─── withReliability — error classification ───────────────────────────────────
+
+describe("withReliability — error classification (no retries)", () => {
+  it("classifies 401 as auth", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("unauthorized"), { status: 401 }));
+    await expect(withReliability(fn, { retries: 0, timeoutMs: 5_000 })).rejects.toMatchObject({
+      kind: "auth",
+    });
+  });
+
+  it("classifies 403 as auth", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("forbidden"), { status: 403 }));
+    await expect(withReliability(fn, { retries: 0, timeoutMs: 5_000 })).rejects.toMatchObject({
+      kind: "auth",
+    });
+  });
+
+  it("classifies 429 as rate_limit", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("too many"), { status: 429 }));
+    await expect(withReliability(fn, { retries: 0, timeoutMs: 5_000 })).rejects.toMatchObject({
+      kind: "rate_limit",
+    });
+  });
+
+  it("classifies ECONNRESET as network", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("reset"), { code: "ECONNRESET" }));
+    await expect(withReliability(fn, { retries: 0, timeoutMs: 5_000 })).rejects.toMatchObject({
+      kind: "network",
+    });
+  });
+
+  it("classifies ENOTFOUND as network", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("not found"), { code: "ENOTFOUND" }));
+    await expect(withReliability(fn, { retries: 0, timeoutMs: 5_000 })).rejects.toMatchObject({
+      kind: "network",
+    });
+  });
+
+  it("classifies 500 as provider", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("server error"), { status: 500 }));
+    await expect(withReliability(fn, { retries: 0, timeoutMs: 5_000 })).rejects.toMatchObject({
+      kind: "provider",
+    });
+  });
+
+  it("throws ProviderError (not raw error)", async () => {
+    const fn = vi.fn().mockRejectedValue(new Error("raw"));
+    const thrown = await withReliability(fn, { retries: 0, timeoutMs: 5_000 }).catch((e) => e);
+    expect(thrown).toBeInstanceOf(ProviderError);
+  });
+});
+
+// ─── withReliability — timeout ────────────────────────────────────────────────
+
+describe("withReliability — timeout", () => {
+  it("rejects with ProviderError when fn exceeds timeoutMs", async () => {
+    const fn = vi.fn().mockImplementation(() => new Promise((r) => setTimeout(r, 10_000)));
+    const err = await withReliability(fn, { retries: 0, timeoutMs: 30 }).catch((e) => e);
+    expect(err).toBeInstanceOf(ProviderError);
+    // timeout errors show up as "provider" or "timeout" kind
+    expect(["timeout", "provider"]).toContain(err.kind);
+  }, 3_000);
+});
+
+// ─── withReliability — retry behaviour ───────────────────────────────────────
+
+describe("withReliability — retry behaviour", () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it("does NOT retry auth errors (401)", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("auth"), { status: 401 }));
+    const promise = withReliability(fn, { retries: 3, timeoutMs: 999_999 });
+    // Attach rejection handler BEFORE running timers to avoid unhandled rejection warning
+    const settled = promise.catch((e) => e);
+    await vi.runAllTimersAsync();
+    const err = await settled;
+    expect(err).toBeInstanceOf(ProviderError);
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it("retries on 500 and succeeds on second attempt", async () => {
+    const fn = vi.fn()
+      .mockRejectedValueOnce(Object.assign(new Error("server error"), { status: 500 }))
+      .mockResolvedValueOnce("recovered");
+    const promise = withReliability(fn, { retries: 1, timeoutMs: 999_999 });
+    await vi.runAllTimersAsync();
+    await expect(promise).resolves.toBe("recovered");
+    expect(fn).toHaveBeenCalledTimes(2);
+  });
+
+  it("retries on ECONNRESET and succeeds", async () => {
+    const fn = vi.fn()
+      .mockRejectedValueOnce(Object.assign(new Error("reset"), { code: "ECONNRESET" }))
+      .mockResolvedValueOnce("ok");
+    const promise = withReliability(fn, { retries: 1, timeoutMs: 999_999 });
+    await vi.runAllTimersAsync();
+    await expect(promise).resolves.toBe("ok");
+    expect(fn).toHaveBeenCalledTimes(2);
+  });
+
+  it("calls onRetry callback with attempt number", async () => {
+    const onRetry = vi.fn();
+    const fn = vi.fn()
+      .mockRejectedValueOnce(Object.assign(new Error("fail"), { status: 500 }))
+      .mockResolvedValueOnce("ok");
+    const promise = withReliability(fn, { retries: 1, timeoutMs: 999_999, onRetry });
+    await vi.runAllTimersAsync();
+    await promise;
+    expect(onRetry).toHaveBeenCalledWith(1, expect.any(Error));
+  });
+
+  it("exhausts all retries and throws", async () => {
+    const fn = vi.fn().mockRejectedValue(Object.assign(new Error("always fails"), { status: 500 }));
+    const promise = withReliability(fn, { retries: 2, timeoutMs: 999_999 });
+    const settled = promise.catch((e) => e);
+    await vi.runAllTimersAsync();
+    const err = await settled;
+    expect(err).toBeInstanceOf(ProviderError);
+    expect(fn).toHaveBeenCalledTimes(3); // 1 initial + 2 retries
+  });
+});

package/tests/run-trend.test.ts

@@ -0,0 +1,186 @@
+import { describe, it, expect } from "vitest";
+import { buildTrendReport } from "../core/run-trend";
+import type { RunLog } from "../core/run-logger";
+
+// ─── Fixtures ─────────────────────────────────────────────────────────────────
+
+function makeLog(overrides: Partial<RunLog> = {}): RunLog {
+  return {
+    runId: `20260330-120000-aaaa`,
+    startedAt: "2026-03-30T12:00:00.000Z",
+    workingDir: "/project",
+    provider: "gemini",
+    model: "gemini-2.5-pro",
+    promptHash: "a3f2c1d8",
+    harnessScore: 7.5,
+    entries: [],
+    filesWritten: ["src/api/order.ts", "src/models/order.ts"],
+    errors: [],
+    endedAt: "2026-03-30T12:01:30.000Z",
+    totalDurationMs: 90000,
+    ...overrides,
+  };
+}
+
+// ─── buildTrendReport — basic shape ──────────────────────────────────────────
+
+describe("buildTrendReport — basic shape", () => {
+  it("returns empty entries and groups for empty log list", () => {
+    const report = buildTrendReport([]);
+    expect(report.entries).toHaveLength(0);
+    expect(report.promptGroups).toHaveLength(0);
+    expect(report.totalRuns).toBe(0);
+  });
+
+  it("filters out logs without harnessScore (non-create runs)", () => {
+    const noScore = makeLog({ harnessScore: undefined });
+    const report = buildTrendReport([noScore]);
+    expect(report.entries).toHaveLength(0);
+  });
+
+  it("includes logs that have a harnessScore", () => {
+    const log = makeLog({ harnessScore: 8.2 });
+    const report = buildTrendReport([log]);
+    expect(report.entries).toHaveLength(1);
+    expect(report.entries[0].harnessScore).toBe(8.2);
+  });
+
+  it("maps log fields to entry correctly", () => {
+    const log = makeLog({
+      runId: "20260330-120000-test",
+      harnessScore: 7.0,
+      promptHash: "cafebabe",
+      specPath: "specs/feature-v1.md",
+      provider: "claude",
+      model: "claude-sonnet-4-6",
+      filesWritten: ["a.ts", "b.ts", "c.ts"],
+      errors: ["err1"],
+      totalDurationMs: 60000,
+    });
+    const { entries } = buildTrendReport([log]);
+    expect(entries[0].runId).toBe("20260330-120000-test");
+    expect(entries[0].harnessScore).toBe(7.0);
+    expect(entries[0].promptHash).toBe("cafebabe");
+    expect(entries[0].provider).toBe("claude");
+    expect(entries[0].filesWritten).toBe(3);
+    expect(entries[0].errors).toBe(1);
+    expect(entries[0].totalDurationMs).toBe(60000);
+  });
+});
+
+// ─── buildTrendReport — filtering ────────────────────────────────────────────
+
+describe("buildTrendReport — last N filter", () => {
+  it("limits entries to last N", () => {
+    const logs = Array.from({ length: 10 }, (_, i) =>
+      makeLog({ runId: `run-${i}`, harnessScore: 5 + i * 0.3 })
+    );
+    const report = buildTrendReport(logs, { last: 3 });
+    expect(report.entries).toHaveLength(3);
+  });
+
+  it("returns all entries when last > total", () => {
+    const logs = [makeLog(), makeLog({ runId: "run-2" })];
+    const report = buildTrendReport(logs, { last: 100 });
+    expect(report.entries).toHaveLength(2);
+  });
+});
+
+describe("buildTrendReport — promptFilter", () => {
+  it("filters entries to matching prompt hash prefix", () => {
+    const logs = [
+      makeLog({ promptHash: "a3f2c1d8", harnessScore: 7 }),
+      makeLog({ runId: "run-2", promptHash: "b1e4a2f0", harnessScore: 8 }),
+      makeLog({ runId: "run-3", promptHash: "a3f2ffff", harnessScore: 6 }),
+    ];
+    const report = buildTrendReport(logs, { promptFilter: "a3f2" });
+    expect(report.entries).toHaveLength(2);
+    expect(report.entries.every((e) => e.promptHash?.startsWith("a3f2"))).toBe(true);
+  });
+
+  it("returns empty when prompt filter matches nothing", () => {
+    const logs = [makeLog({ promptHash: "a3f2c1d8" })];
+    const report = buildTrendReport(logs, { promptFilter: "zzzz" });
+    expect(report.entries).toHaveLength(0);
+  });
+});
+
+// ─── buildTrendReport — prompt groups ────────────────────────────────────────
+
+describe("buildTrendReport — promptGroups aggregation", () => {
+  it("groups runs by promptHash", () => {
+    const logs = [
+      makeLog({ promptHash: "aaa", harnessScore: 7 }),
+      makeLog({ runId: "run-2", promptHash: "aaa", harnessScore: 9 }),
+      makeLog({ runId: "run-3", promptHash: "bbb", harnessScore: 6 }),
+    ];
+    const { promptGroups } = buildTrendReport(logs);
+    expect(promptGroups).toHaveLength(2);
+    const aaa = promptGroups.find((g) => g.promptHash === "aaa");
+    expect(aaa?.runs).toBe(2);
+  });
+
+  it("computes avg, best, worst correctly", () => {
+    const logs = [
+      makeLog({ promptHash: "aaa", harnessScore: 6 }),
+      makeLog({ runId: "run-2", promptHash: "aaa", harnessScore: 8 }),
+      makeLog({ runId: "run-3", promptHash: "aaa", harnessScore: 7 }),
+    ];
+    const { promptGroups } = buildTrendReport(logs);
+    const aaa = promptGroups.find((g) => g.promptHash === "aaa")!;
+    expect(aaa.best).toBe(8);
+    expect(aaa.worst).toBe(6);
+    expect(aaa.avg).toBeCloseTo(7.0, 1);
+  });
+
+  it("marks the most recently used promptHash as isCurrent", () => {
+    // Logs are already sorted newest-first by loadRunLogs; we pass them in order
+    const logs = [
+      makeLog({ runId: "newer", promptHash: "new-hash", startedAt: "2026-03-30T14:00:00.000Z", harnessScore: 7 }),
+      makeLog({ runId: "older", promptHash: "old-hash", startedAt: "2026-03-29T10:00:00.000Z", harnessScore: 6 }),
+    ];
+    const { promptGroups } = buildTrendReport(logs);
+    const current = promptGroups.find((g) => g.isCurrent);
+    expect(current?.promptHash).toBe("new-hash");
+  });
+
+  it("does NOT mark non-current groups as isCurrent", () => {
+    const logs = [
+      makeLog({ promptHash: "new", harnessScore: 7 }),
+      makeLog({ runId: "r2", promptHash: "old", harnessScore: 6 }),
+    ];
+    const { promptGroups } = buildTrendReport(logs);
+    const nonCurrent = promptGroups.filter((g) => !g.isCurrent);
+    expect(nonCurrent.every((g) => g.isCurrent === false)).toBe(true);
+  });
+
+  it("handles (none) group for runs without a promptHash", () => {
+    const log = makeLog({ promptHash: undefined, harnessScore: 5 });
+    const { promptGroups } = buildTrendReport([log]);
+    expect(promptGroups[0].promptHash).toBe("(none)");
+  });
+
+  it("sorts groups by lastSeen descending (most recent first)", () => {
+    const logs = [
+      makeLog({ promptHash: "aaa", startedAt: "2026-03-30T10:00:00.000Z", harnessScore: 7 }),
+      makeLog({ runId: "r2", promptHash: "bbb", startedAt: "2026-03-28T10:00:00.000Z", harnessScore: 6 }),
+    ];
+    const { promptGroups } = buildTrendReport(logs);
+    expect(promptGroups[0].promptHash).toBe("aaa"); // most recent first
+    expect(promptGroups[1].promptHash).toBe("bbb");
+  });
+});
+
+// ─── buildTrendReport — totalRuns ────────────────────────────────────────────
+
+describe("buildTrendReport — totalRuns", () => {
+  it("totalRuns equals number of entries after filtering", () => {
+    const logs = [
+      makeLog({ harnessScore: 7 }),
+      makeLog({ runId: "r2", harnessScore: undefined }), // filtered out
+      makeLog({ runId: "r3", harnessScore: 6 }),
+    ];
+    const { totalRuns } = buildTrendReport(logs);
+    expect(totalRuns).toBe(2);
+  });
+});