muonroi-cli 1.4.0 → 1.5.0

package/src/council/__tests__/clarifier-ready-gate.test.ts

@@ -1,268 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-import type { StreamChunk } from "../../types/index.js";
-import { judgeReadiness, MAX_CLARIFY_ROUNDS, runClarification } from "../clarifier.js";
-import type { ClarifiedSpec, CouncilLLM, QuestionResponder } from "../types.js";
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-/** Drain the generator and return the final ClarifiedSpec. */
-async function _drain(gen: AsyncGenerator<StreamChunk, ClarifiedSpec, unknown>): Promise<ClarifiedSpec> {
-  let result: ClarifiedSpec | undefined;
-  let done = false;
-  while (!done) {
-    const next = await gen.next();
-    if (next.done) {
-      result = next.value;
-      done = true;
-    }
-  }
-  return result!;
-}
-
-/** Count "clarification_round active" phase events emitted by the generator. */
-async function countRounds(
-  gen: AsyncGenerator<StreamChunk, ClarifiedSpec, unknown>,
-): Promise<{ rounds: number; spec: ClarifiedSpec }> {
-  let rounds = 0;
-  let spec: ClarifiedSpec | undefined;
-  let done = false;
-  while (!done) {
-    const next = await gen.next();
-    if (next.done) {
-      spec = next.value;
-      done = true;
-    } else if (
-      next.value.type === "council_phase" &&
-      (next.value as any).councilPhase?.kind === "clarification_round" &&
-      (next.value as any).councilPhase?.state === "active"
-    ) {
-      rounds++;
-    }
-  }
-  return { rounds, spec: spec! };
-}
-
-// ---------------------------------------------------------------------------
-// Shared mock responder — always answers "some answer"
-// ---------------------------------------------------------------------------
-const alwaysAnswer: QuestionResponder = vi.fn().mockResolvedValue("some answer");
-
-// ---------------------------------------------------------------------------
-// Test A: judgeReadiness returns ready=true after round 1 → loop exits early
-// ---------------------------------------------------------------------------
-describe("P5 ready-gate: Test A — judge ready after round 1", () => {
-  it("exits loop after 1 round and populates spec.ready + spec.confidenceScore", async () => {
-    // First call: returns a clarification question
-    // Second call (judgeReadiness): returns ready=true
-    // Spec synthesis call: returns a valid spec JSON
-    let callCount = 0;
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockImplementation(async () => {
-        callCount++;
-        if (callCount === 1) {
-          // clarify_questions round 1
-          return JSON.stringify([{ question: "What is the target platform?", why: "Scope clarity", isRequired: true }]);
-        }
-        if (callCount === 2) {
-          // judgeReadiness verdict — ready after round 1
-          return JSON.stringify({ ready: true, confidence: 0.9, gaps: [] });
-        }
-        // spec_synthesis
-        return JSON.stringify({
-          problemStatement: "Build a todo app",
-          constraints: ["web only"],
-          successCriteria: ["User can add tasks", "User can mark done"],
-          scope: "Single user, local storage",
-        });
-      }),
-    } as any;
-
-    const gen = runClarification("Build a todo app", "leader-model", "", alwaysAnswer, mockLLM);
-    const { rounds, spec } = await countRounds(gen);
-
-    expect(rounds).toBe(1);
-    expect(spec.ready).toBe(true);
-    expect(spec.confidenceScore).toBeGreaterThan(0);
-    expect(spec.clarifyHistory).toHaveLength(1);
-    expect(spec.remainingGaps).toEqual([]);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Test B: judge returns ready=false for 3 rounds, then ready=true on round 4
-// ---------------------------------------------------------------------------
-describe("P5 ready-gate: Test B — judge ready at round 4", () => {
-  it("runs 4 rounds and history has 4 entries", async () => {
-    let callCount = 0;
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockImplementation(async () => {
-        callCount++;
-        // Pattern: clarify_q → judge → clarify_q → judge → ... (interleaved)
-        // Calls: 1=q1, 2=judge1(not ready), 3=q2, 4=judge2(not ready),
-        //        5=q3, 6=judge3(not ready), 7=q4, 8=judge4(ready), 9=synthesis
-        const judgeCallNums = [2, 4, 6, 8];
-        if (judgeCallNums.includes(callCount)) {
-          const isLastJudge = callCount === 8;
-          return JSON.stringify({
-            ready: isLastJudge,
-            confidence: isLastJudge ? 0.85 : 0.4,
-            gaps: isLastJudge ? [] : ["Target platform not specified.", "Authentication method unclear."],
-          });
-        }
-        if (callCount === 9) {
-          // spec synthesis
-          return JSON.stringify({
-            problemStatement: "Build a todo app",
-            constraints: [],
-            successCriteria: ["Tasks can be added"],
-            scope: "Web app",
-          });
-        }
-        // clarify_questions
-        return JSON.stringify([{ question: `Question ${callCount}`, why: "Gap", isRequired: true }]);
-      }),
-    } as any;
-
-    const gen = runClarification("Build a todo app", "leader-model", "", alwaysAnswer, mockLLM);
-    const { rounds, spec } = await countRounds(gen);
-
-    expect(rounds).toBe(4);
-    expect(spec.clarifyHistory).toHaveLength(4);
-    expect(spec.ready).toBe(true);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Test C: judge always returns ready=false → loop caps at MAX_CLARIFY_ROUNDS
-// ---------------------------------------------------------------------------
-describe("P5 ready-gate: Test C — hard cap at MAX_CLARIFY_ROUNDS", () => {
-  it(`caps at ${MAX_CLARIFY_ROUNDS} rounds, spec.ready=false, remainingGaps non-empty`, async () => {
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockImplementation(async (_modelId: string, system: string) => {
-        // Distinguish clarify prompt (system contains "clarification questions")
-        // from judge prompt (system contains "debate facilitator")
-        // from synthesis prompt (system contains "synthesizing")
-        if (system.includes("debate facilitator")) {
-          return JSON.stringify({ ready: false, confidence: 0.2, gaps: ["Gap A.", "Gap B."] });
-        }
-        if (system.includes("synthesizing") || system.includes("extracting")) {
-          return JSON.stringify({
-            problemStatement: "Build something",
-            constraints: [],
-            successCriteria: ["Something works"],
-            scope: "TBD",
-          });
-        }
-        // clarify_questions
-        return JSON.stringify([{ question: "Some question?", why: "Gap", isRequired: true }]);
-      }),
-    } as any;
-
-    const gen = runClarification("Build something vague", "leader-model", "", alwaysAnswer, mockLLM);
-    const { rounds, spec } = await countRounds(gen);
-
-    expect(rounds).toBe(MAX_CLARIFY_ROUNDS);
-    expect(spec.ready).toBe(false);
-    expect(spec.remainingGaps?.length ?? 0).toBeGreaterThan(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Test D: judgeReadiness unit tests
-// ---------------------------------------------------------------------------
-describe("P5 ready-gate: Test D — judgeReadiness unit", () => {
-  const baseSpec: ClarifiedSpec = {
-    problemStatement: "Build a REST API for a todo app",
-    constraints: ["TypeScript only"],
-    successCriteria: ["CRUD endpoints work", "Auth required"],
-    scope: "Backend only, no UI",
-    rawQA: [],
-  };
-
-  it("returns ready=true when LLM says so with detailed Q&A", async () => {
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockResolvedValue(JSON.stringify({ ready: true, confidence: 0.95, gaps: [] })),
-    } as any;
-
-    const qa = [
-      { question: "What platform?", answer: "Web — Node.js backend" },
-      { question: "Auth method?", answer: "JWT" },
-      { question: "Database?", answer: "PostgreSQL" },
-    ];
-
-    const result = await judgeReadiness(baseSpec, "Build a REST API", qa, mockLLM, "leader-model", false);
-    expect(result.ready).toBe(true);
-    expect(result.confidence).toBeGreaterThan(0.8);
-    expect(result.gaps).toHaveLength(0);
-  });
-
-  it("returns ready=false when topic only, no Q&A", async () => {
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockResolvedValue(
-        JSON.stringify({
-          ready: false,
-          confidence: 0.15,
-          gaps: ["Deployment target not specified.", "Auth approach unknown."],
-        }),
-      ),
-    } as any;
-
-    const partialSpec: ClarifiedSpec = {
-      problemStatement: "todo app",
-      constraints: [],
-      successCriteria: [],
-      scope: "",
-      rawQA: [],
-    };
-
-    const result = await judgeReadiness(partialSpec, "todo app", [], mockLLM, "leader-model", false);
-    expect(result.ready).toBe(false);
-    expect(result.gaps.length).toBeGreaterThan(0);
-  });
-
-  it("clamps confidence to [0, 1]", async () => {
-    const mockLLM: CouncilLLM = {
-      generate: vi
-        .fn()
-        .mockResolvedValue(JSON.stringify({ ready: false, confidence: 1.8, gaps: ["Something missing."] })),
-    } as any;
-
-    const result = await judgeReadiness(baseSpec, "topic", [], mockLLM, "model", false);
-    expect(result.confidence).toBeLessThanOrEqual(1);
-    expect(result.confidence).toBeGreaterThanOrEqual(0);
-  });
-
-  it("forces gaps=[] when ready=true regardless of LLM output", async () => {
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockResolvedValue(
-        // LLM contradicts itself: ready=true but also provides gaps
-        JSON.stringify({ ready: true, confidence: 0.9, gaps: ["Some gap that shouldn't be here."] }),
-      ),
-    } as any;
-
-    const result = await judgeReadiness(baseSpec, "topic", [], mockLLM, "model", false);
-    expect(result.ready).toBe(true);
-    expect(result.gaps).toHaveLength(0);
-  });
-
-  it("handles LLM failure gracefully — returns ready=false, confidence=0", async () => {
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockRejectedValue(new Error("network error")),
-    } as any;
-
-    const result = await judgeReadiness(baseSpec, "topic", [], mockLLM, "model", false);
-    expect(result.ready).toBe(false);
-    expect(result.confidence).toBe(0);
-  });
-
-  it("handles malformed JSON from LLM — returns ready=false", async () => {
-    const mockLLM: CouncilLLM = {
-      generate: vi.fn().mockResolvedValue("not json at all"),
-    } as any;
-
-    const result = await judgeReadiness(baseSpec, "topic", [], mockLLM, "model", false);
-    expect(result.ready).toBe(false);
-  });
-});

package/src/council/__tests__/cost-aware.test.ts

@@ -1,60 +0,0 @@
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import * as registry from "../../models/registry.js";
-import * as runtime from "../../providers/runtime.js";
-import type { ModelInfo } from "../../types/index.js";
-import { pickCouncilTaskModel } from "../leader.js";
-
-const catalog: ModelInfo[] = [
-  { id: "premium-x", provider: "anthropic", tier: "premium" } as ModelInfo,
-  { id: "balanced-x", provider: "anthropic", tier: "balanced" } as ModelInfo,
-  { id: "fast-x", provider: "anthropic", tier: "fast" } as ModelInfo,
-  { id: "fast-y", provider: "openai", tier: "fast" } as ModelInfo,
-];
-
-describe("pickCouncilTaskModel", () => {
-  beforeEach(() => {
-    vi.spyOn(registry, "getModelInfo").mockImplementation((id) => catalog.find((m) => m.id === id));
-    vi.spyOn(registry, "getModelByTier").mockImplementation((tier, prefer) => {
-      const onPrefer = catalog.find((m) => m.tier === tier && m.provider === prefer);
-      return onPrefer ?? catalog.find((m) => m.tier === tier);
-    });
-    vi.spyOn(runtime, "detectProviderForModel").mockImplementation((id) => {
-      const m = catalog.find((x) => x.id === id);
-      return (m?.provider ?? "anthropic") as ReturnType<typeof runtime.detectProviderForModel>;
-    });
-  });
-
-  afterEach(() => {
-    vi.restoreAllMocks();
-  });
-
-  it("returns leader unchanged when costAware=false", () => {
-    expect(pickCouncilTaskModel("research_need", "premium-x", false)).toBe("premium-x");
-  });
-
-  it("downshifts to fast tier for trivial classifier (research_need)", () => {
-    expect(pickCouncilTaskModel("research_need", "premium-x", true)).toBe("fast-x");
-  });
-
-  it("downshifts to balanced tier for evaluate_round", () => {
-    expect(pickCouncilTaskModel("evaluate_round", "premium-x", true)).toBe("balanced-x");
-  });
-
-  it("does not switch providers (leader anthropic, no anthropic-fast → falls back)", () => {
-    vi.spyOn(registry, "getModelByTier").mockImplementation((tier) =>
-      tier === "fast" ? catalog.find((m) => m.id === "fast-y") : undefined,
-    );
-    // fast-y is openai — must NOT be selected; fall back to leader.
-    expect(pickCouncilTaskModel("research_need", "premium-x", true)).toBe("premium-x");
-  });
-
-  it("does not downshift when leader is already at or below target tier", () => {
-    expect(pickCouncilTaskModel("evaluate_round", "balanced-x", true)).toBe("balanced-x");
-    expect(pickCouncilTaskModel("evaluate_round", "fast-x", true)).toBe("fast-x");
-  });
-
-  it("falls back to leader when target tier has no model anywhere", () => {
-    vi.spyOn(registry, "getModelByTier").mockReturnValue(undefined);
-    expect(pickCouncilTaskModel("research_need", "premium-x", true)).toBe("premium-x");
-  });
-});

package/src/council/__tests__/debate-planner-structured.test.ts

@@ -1,236 +0,0 @@
-/**
- * CQ-10: debate-planner uses generateObject with one-retry fallback
- *
- * Tests that planDebate:
- * 1. Calls generateObject on first attempt (structured output)
- * 2. Retries once via tracedGenerate when generateObject fails
- * 3. Returns FALLBACK_PLAN after both attempts fail
- */
-import { beforeEach, describe, expect, it, vi } from "vitest";
-
-const FALLBACK_STANCES = ["Primary Analyst", "Critical Reviewer"];
-
-describe("CQ-10: planDebate uses generateObject as first attempt", () => {
-  beforeEach(() => {
-    vi.resetModules();
-  });
-
-  it("calls generateObject (not generateText) on first attempt", async () => {
-    const mockGenerateObject = vi.fn().mockResolvedValue({
-      object: {
-        intentSummary: "Test intent",
-        stances: [
-          { name: "Analyst", lens: "Analyse the problem" },
-          { name: "Critic", lens: "Challenge assumptions" },
-        ],
-        outputShape: {
-          kind: "decision",
-          sections: [{ key: "result", heading: "Result", prompt: "findings", shape: "text" as const }],
-          guardrails: [],
-        },
-      },
-    });
-
-    vi.doMock("ai", () => ({
-      generateObject: mockGenerateObject,
-      generateText: vi.fn(),
-    }));
-    vi.doMock("../../providers/keychain.js", () => ({
-      loadKeyForProvider: vi.fn().mockResolvedValue("test-key"),
-    }));
-    vi.doMock("../../providers/runtime.js", () => ({
-      detectProviderForModel: vi.fn().mockReturnValue("openai"),
-      createProviderFactory: vi.fn().mockReturnValue({ factory: {} }),
-      resolveModelRuntime: vi.fn().mockReturnValue({ model: {}, providerOptions: undefined }),
-    }));
-    vi.doMock("../prompts.js", () => ({
-      buildDebatePlanPrompt: vi.fn().mockReturnValue({ system: "sys", prompt: "prompt" }),
-    }));
-
-    const { planDebate } = await import("../debate-planner.js");
-
-    const spec = {
-      problemStatement: "test",
-      constraints: [],
-      successCriteria: [],
-      scope: "test",
-      rawQA: [],
-    };
-
-    // Consume the generator to get the return value
-    const llm = {} as never;
-    const gen = planDebate(spec, "gpt-4o", llm);
-    let result = await gen.next();
-    while (!result.done) {
-      result = await gen.next();
-    }
-    const plan = result.value;
-
-    expect(mockGenerateObject).toHaveBeenCalledOnce();
-    expect(plan.stances.length).toBeGreaterThanOrEqual(2);
-    expect(plan.intentSummary).toBe("Test intent");
-  });
-
-  it("retries once with schema error feedback when generateObject fails", async () => {
-    const mockGenerateObject = vi.fn().mockRejectedValue(new Error("Schema validation failed: missing stances"));
-
-    const _mockTracedGenerateLlm = {
-      generate: vi.fn().mockResolvedValue(""),
-    };
-
-    // Capture what prompt the retry uses
-    let capturedRetryPrompt: string | undefined;
-
-    vi.doMock("ai", () => ({
-      generateObject: mockGenerateObject,
-      generateText: vi.fn(),
-    }));
-    vi.doMock("../../providers/keychain.js", () => ({
-      loadKeyForProvider: vi.fn().mockResolvedValue("test-key"),
-    }));
-    vi.doMock("../../providers/runtime.js", () => ({
-      detectProviderForModel: vi.fn().mockReturnValue("openai"),
-      createProviderFactory: vi.fn().mockReturnValue({ factory: {} }),
-      resolveModelRuntime: vi.fn().mockReturnValue({ model: {}, providerOptions: undefined }),
-    }));
-    vi.doMock("../prompts.js", () => ({
-      buildDebatePlanPrompt: vi.fn().mockReturnValue({ system: "sys", prompt: "original-prompt" }),
-    }));
-    // Mock tracedGenerate to return valid JSON on retry
-    vi.doMock("../llm.js", () => ({
-      tracedGenerate: vi.fn().mockImplementation(async function* (_llm: unknown, opts: { prompt: string }) {
-        capturedRetryPrompt = opts.prompt;
-        yield { type: "content", content: "" };
-        return JSON.stringify({
-          intentSummary: "Retry intent",
-          stances: [
-            { name: "Analyst", lens: "Analyse" },
-            { name: "Critic", lens: "Challenge" },
-          ],
-          outputShape: {
-            kind: "decision",
-            sections: [{ key: "r", heading: "Result", prompt: "r", shape: "text" }],
-            guardrails: [],
-          },
-        });
-      }),
-    }));
-
-    const { planDebate } = await import("../debate-planner.js");
-
-    const spec = {
-      problemStatement: "test",
-      constraints: [],
-      successCriteria: [],
-      scope: "test",
-      rawQA: [],
-    };
-
-    const llm = {} as never;
-    const gen = planDebate(spec, "gpt-4o", llm);
-    let result = await gen.next();
-    while (!result.done) {
-      result = await gen.next();
-    }
-    const plan = result.value;
-
-    // generateObject was tried
-    expect(mockGenerateObject).toHaveBeenCalledOnce();
-    // Retry prompt must contain schema error feedback
-    expect(capturedRetryPrompt).toContain("Schema validation failed");
-    // Should return the retry-parsed plan
-    expect(plan.stances.length).toBeGreaterThanOrEqual(2);
-  });
-
-  it("returns FALLBACK_PLAN after both generateObject and retry fail", async () => {
-    vi.doMock("ai", () => ({
-      generateObject: vi.fn().mockRejectedValue(new Error("API error")),
-      generateText: vi.fn(),
-    }));
-    vi.doMock("../../providers/keychain.js", () => ({
-      loadKeyForProvider: vi.fn().mockResolvedValue("test-key"),
-    }));
-    vi.doMock("../../providers/runtime.js", () => ({
-      detectProviderForModel: vi.fn().mockReturnValue("openai"),
-      createProviderFactory: vi.fn().mockReturnValue({ factory: {} }),
-      resolveModelRuntime: vi.fn().mockReturnValue({ model: {}, providerOptions: undefined }),
-    }));
-    vi.doMock("../prompts.js", () => ({
-      buildDebatePlanPrompt: vi.fn().mockReturnValue({ system: "sys", prompt: "prompt" }),
-    }));
-    // tracedGenerate also fails
-    vi.doMock("../llm.js", () => ({
-      tracedGenerate: vi.fn().mockImplementation(async function* () {
-        throw new Error("Retry also failed");
-      }),
-    }));
-
-    const { planDebate } = await import("../debate-planner.js");
-
-    const spec = {
-      problemStatement: "test",
-      constraints: [],
-      successCriteria: [],
-      scope: "test",
-      rawQA: [],
-    };
-
-    const llm = {} as never;
-    const gen = planDebate(spec, "gpt-4o", llm);
-    let result = await gen.next();
-    while (!result.done) {
-      result = await gen.next();
-    }
-    const plan = result.value;
-
-    // Must be FALLBACK_PLAN — check by known stance names
-    expect(plan.stances.map((s: { name: string }) => s.name)).toEqual(FALLBACK_STANCES);
-  });
-
-  it("schema error text is sliced to 200 chars in retry prompt (T-15-07 threat mitigation)", async () => {
-    const longError = "x".repeat(500);
-    vi.doMock("ai", () => ({
-      generateObject: vi.fn().mockRejectedValue(new Error(longError)),
-      generateText: vi.fn(),
-    }));
-    vi.doMock("../../providers/keychain.js", () => ({
-      loadKeyForProvider: vi.fn().mockResolvedValue("test-key"),
-    }));
-    vi.doMock("../../providers/runtime.js", () => ({
-      detectProviderForModel: vi.fn().mockReturnValue("openai"),
-      createProviderFactory: vi.fn().mockReturnValue({ factory: {} }),
-      resolveModelRuntime: vi.fn().mockReturnValue({ model: {}, providerOptions: undefined }),
-    }));
-    vi.doMock("../prompts.js", () => ({
-      buildDebatePlanPrompt: vi.fn().mockReturnValue({ system: "sys", prompt: "prompt" }),
-    }));
-
-    let capturedPrompt = "";
-    vi.doMock("../llm.js", () => ({
-      tracedGenerate: vi.fn().mockImplementation(async function* (_llm: unknown, opts: { prompt: string }) {
-        capturedPrompt = opts.prompt;
-        throw new Error("retry fail");
-      }),
-    }));
-
-    const { planDebate } = await import("../debate-planner.js");
-
-    const spec = { problemStatement: "test", constraints: [], successCriteria: [], scope: "test", rawQA: [] };
-    const gen = planDebate(spec, "gpt-4o", {} as never);
-    let result = await gen.next();
-    while (!result.done) {
-      result = await gen.next();
-    }
-
-    // The injected error feedback must not exceed 200 chars (T-15-07)
-    const _errorInPrompt = capturedPrompt.slice(
-      capturedPrompt.indexOf("Schema validation failed:") + "Schema validation failed: ".length,
-    );
-    // First token after "Schema validation failed: " must be at most 200 chars before next period/sentence
-    // Simple check: the injected error segment is <= 200 chars
-    const match = capturedPrompt.match(/Schema validation failed: (.{1,220})/);
-    expect(match).toBeTruthy();
-    const injected = match![1].split(".")[0]; // up to first period
-    expect(injected.length).toBeLessThanOrEqual(205); // small buffer for "." suffix
-  });
-});