@nathapp/nax 0.27.1 → 0.29.0

package/test/integration/tdd/tdd-orchestrator-failureCategory.test.ts

@@ -1,355 +0,0 @@
-import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
-import { existsSync } from "node:fs";
-import { mkdir, rm, writeFile } from "node:fs/promises";
-import path from "node:path";
-import type { AgentAdapter, AgentResult } from "../../../src/agents";
-import { DEFAULT_CONFIG } from "../../../src/config";
-import type { UserStory } from "../../../src/prd";
-import { runThreeSessionTdd } from "../../../src/tdd/orchestrator";
-import { VERDICT_FILE } from "../../../src/tdd/verdict";
-
-let originalSpawn: typeof Bun.spawn;
-
-beforeEach(() => {
-  originalSpawn = Bun.spawn;
-});
-
-afterEach(() => {
-  Bun.spawn = originalSpawn;
-});
-
-/** Create a mock agent that returns sequential results */
-function createMockAgent(results: Partial<AgentResult>[]): AgentAdapter {
-  let callCount = 0;
-  return {
-    name: "mock",
-    displayName: "Mock Agent",
-    binary: "mock",
-    isInstalled: async () => true,
-    buildCommand: () => ["mock"],
-    run: mock(async () => {
-      const r = results[callCount] || {};
-      callCount++;
-      return {
-        success: r.success ?? true,
-        exitCode: r.exitCode ?? 0,
-        output: r.output ?? "",
-        rateLimited: r.rateLimited ?? false,
-        durationMs: r.durationMs ?? 100,
-        estimatedCost: r.estimatedCost ?? 0.01,
-      };
-    }),
-  };
-}
-
-/** Mock Bun.spawn to intercept git commands */
-function mockGitSpawn(opts: {
-  /** Files returned by git diff for each session (indexed by git-diff call number) */
-  diffFiles: string[][];
-  /** Optional: mock test command success (default: true) */
-  testCommandSuccess?: boolean;
-}) {
-  let revParseCount = 0;
-  let diffCount = 0;
-  const testSuccess = opts.testCommandSuccess ?? true;
-
-  // @ts-ignore — mocking global
-  Bun.spawn = mock((cmd: string[], spawnOpts?: any) => {
-    // Intercept test commands (bun test, npm test, etc.)
-    if ((cmd[0] === "/bin/sh" || cmd[0] === "/bin/bash" || cmd[0] === "/bin/zsh") && cmd[1] === "-c") {
-      return {
-        pid: 9999,
-        exited: Promise.resolve(testSuccess ? 0 : 1),
-        stdout: new Response(testSuccess ? "tests pass\n" : "tests fail\n").body,
-        stderr: new Response("").body,
-      };
-    }
-    if (cmd[0] === "git" && cmd[1] === "rev-parse") {
-      revParseCount++;
-      return {
-        exited: Promise.resolve(0),
-        stdout: new Response(`ref-${revParseCount}\n`).body,
-        stderr: new Response("").body,
-      };
-    }
-    if (cmd[0] === "git" && cmd[1] === "checkout") {
-      // Intercept git checkout (used in zero-file fallback) — silently succeed
-      return {
-        exited: Promise.resolve(0),
-        stdout: new Response("").body,
-        stderr: new Response("").body,
-      };
-    }
-    if (cmd[0] === "git" && cmd[1] === "diff") {
-      const files = opts.diffFiles[diffCount] || [];
-      diffCount++;
-      return {
-        exited: Promise.resolve(0),
-        stdout: new Response(files.join("\n") + "\n").body,
-        stderr: new Response("").body,
-      };
-    }
-    return originalSpawn(cmd, spawnOpts);
-  });
-}
-
-const story: UserStory = {
-  id: "US-001",
-  title: "Add user validation",
-  description: "Add validation to user input",
-  acceptanceCriteria: ["Validation works", "Errors are clear"],
-  dependencies: [],
-  tags: [],
-  status: "pending",
-  passes: false,
-  escalations: [],
-  attempts: 0,
-};
-
-
-describe("runThreeSessionTdd — failureCategory", () => {
-  test("test-writer isolation failure sets failureCategory='isolation-violation'", async () => {
-    // Test-writer modifies source files → isolation violation
-    mockGitSpawn({
-      diffFiles: [
-        // Isolation check: test-writer touched source files!
-        ["src/user.ts", "test/user.test.ts"],
-        // getChangedFiles
-        ["src/user.ts", "test/user.test.ts"],
-      ],
-    });
-
-    const agent = createMockAgent([{ success: true, estimatedCost: 0.01 }]);
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: DEFAULT_CONFIG,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.success).toBe(false);
-    expect(result.failureCategory).toBe("isolation-violation");
-  });
-
-  test("test-writer zero files (non-auto strategy) sets failureCategory='isolation-violation'", async () => {
-    // In strict strategy, zero test files → greenfield-no-tests category (BUG-010 behavior)
-    mockGitSpawn({
-      diffFiles: [
-        ["requirements.md"], // s1 isolation — no source violations
-        ["requirements.md"], // s1 getChangedFiles — 0 test files
-      ],
-    });
-
-    const agent = createMockAgent([{ success: true, estimatedCost: 0.01 }]);
-
-    const configWithStrictStrategy = {
-      ...DEFAULT_CONFIG,
-      tdd: { ...DEFAULT_CONFIG.tdd, strategy: "strict" as const },
-    };
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: configWithStrictStrategy,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.success).toBe(false);
-    expect(result.failureCategory).toBe("greenfield-no-tests");
-  });
-
-  test("test-writer crash/timeout (non-isolation failure) sets failureCategory='session-failure'", async () => {
-    // Test-writer agent crashes/times out but isolation is clean
-    mockGitSpawn({
-      diffFiles: [
-        // Isolation check: only test files (passes)
-        ["test/user.test.ts"],
-        // getChangedFiles
-        ["test/user.test.ts"],
-      ],
-    });
-
-    const agent = createMockAgent([
-      { success: false, exitCode: 1, estimatedCost: 0.01 }, // Agent crash
-    ]);
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: DEFAULT_CONFIG,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.success).toBe(false);
-    // isolation.passed=true but agent failed → session-failure
-    expect(result.failureCategory).toBe("session-failure");
-  });
-
-  test("implementer failure sets failureCategory='session-failure'", async () => {
-    mockGitSpawn({
-      diffFiles: [
-        // Session 1 isolation: OK
-        ["test/user.test.ts"],
-        // Session 1 getChangedFiles
-        ["test/user.test.ts"],
-        // Session 2 isolation: OK
-        ["src/user.ts"],
-        // Session 2 getChangedFiles
-        ["src/user.ts"],
-      ],
-    });
-
-    const agent = createMockAgent([
-      { success: true, estimatedCost: 0.01 }, // test-writer OK
-      { success: false, exitCode: 1, estimatedCost: 0.02 }, // implementer fails
-    ]);
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: DEFAULT_CONFIG,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.success).toBe(false);
-    expect(result.failureCategory).toBe("session-failure");
-  });
-
-  test("post-TDD test failure sets failureCategory='tests-failing'", async () => {
-    // Verifier session fails AND independent test run also fails
-    let revParseCount = 0;
-    let diffCount = 0;
-
-    const diffFiles = [["test/user.test.ts"], ["test/user.test.ts"], ["src/user.ts"], ["src/user.ts"], ["src/user.ts"]];
-
-    // @ts-ignore — mocking global
-    Bun.spawn = mock((cmd: string[], spawnOpts?: any) => {
-      if (cmd[0] === "/bin/sh" && cmd[2]?.includes("bun test")) {
-        return {
-          pid: 9999,
-          exited: Promise.resolve(1), // Tests FAIL
-          stdout: new Response("3 pass, 2 fail\n").body,
-          stderr: new Response("Test errors...\n").body,
-        };
-      }
-      if (cmd[0] === "git" && cmd[1] === "rev-parse") {
-        revParseCount++;
-        return {
-          exited: Promise.resolve(0),
-          stdout: new Response(`ref-${revParseCount}\n`).body,
-          stderr: new Response("").body,
-        };
-      }
-      if (cmd[0] === "git" && cmd[1] === "diff") {
-        const files = diffFiles[diffCount] || [];
-        diffCount++;
-        return {
-          exited: Promise.resolve(0),
-          stdout: new Response(files.join("\n") + "\n").body,
-          stderr: new Response("").body,
-        };
-      }
-      return originalSpawn(cmd, spawnOpts);
-    });
-
-    const agent = createMockAgent([
-      { success: true, estimatedCost: 0.01 },
-      { success: true, estimatedCost: 0.02 },
-      { success: false, exitCode: 1, estimatedCost: 0.01 }, // verifier fails
-    ]);
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: DEFAULT_CONFIG,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.success).toBe(false);
-    expect(result.failureCategory).toBe("tests-failing");
-  });
-
-  test("success path has no failureCategory", async () => {
-    mockGitSpawn({
-      diffFiles: [["test/user.test.ts"], ["test/user.test.ts"], ["src/user.ts"], ["src/user.ts"], ["src/user.ts"]],
-    });
-
-    const agent = createMockAgent([
-      { success: true, estimatedCost: 0.01 },
-      { success: true, estimatedCost: 0.02 },
-      { success: true, estimatedCost: 0.01 },
-    ]);
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: DEFAULT_CONFIG,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.success).toBe(true);
-    expect(result.failureCategory).toBeUndefined();
-  });
-
-  test("zero-file scenario (auto strategy) returns greenfield-no-tests (BUG-010 removed auto-fallback)", async () => {
-    // BUG-010: In auto strategy, zero test files → return greenfield-no-tests (no more fallback)
-    let diffCount = 0;
-
-    const diffFiles = [
-      ["requirements.md"], // s1 isolation (strict) — no source violations
-      ["requirements.md"], // s1 getChangedFiles (strict) — 0 test files → return greenfield-no-tests
-    ];
-
-    // @ts-ignore — mocking global
-    Bun.spawn = mock((cmd: string[], spawnOpts?: any) => {
-      if (cmd[0] === "git" && cmd[1] === "rev-parse") {
-        return {
-          exited: Promise.resolve(0),
-          stdout: new Response("ref-1\n").body,
-          stderr: new Response("").body,
-        };
-      }
-      if (cmd[0] === "git" && cmd[1] === "diff") {
-        const files = diffFiles[diffCount] || [];
-        diffCount++;
-        return {
-          exited: Promise.resolve(0),
-          stdout: new Response(files.join("\n") + "\n").body,
-          stderr: new Response("").body,
-        };
-      }
-      return originalSpawn(cmd, spawnOpts);
-    });
-
-    const agent = createMockAgent([
-      { success: true, estimatedCost: 0.01 }, // s1 strict test-writer
-    ]);
-
-    const configWithAutoStrategy = {
-      ...DEFAULT_CONFIG,
-      tdd: { ...DEFAULT_CONFIG.tdd, strategy: "auto" as const },
-    };
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: configWithAutoStrategy,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.success).toBe(false);
-    expect(result.lite).toBe(false);
-    expect(result.failureCategory).toBe("greenfield-no-tests");
-  });
-});
-
-// ─── T9: Verdict integration tests ───────────────────────────────────────────
-

package/test/integration/tdd/tdd-orchestrator-fallback.test.ts

@@ -1,311 +0,0 @@
-import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
-import { existsSync } from "node:fs";
-import { mkdir, rm, writeFile } from "node:fs/promises";
-import path from "node:path";
-import type { AgentAdapter, AgentResult } from "../../../src/agents";
-import { DEFAULT_CONFIG } from "../../../src/config";
-import type { UserStory } from "../../../src/prd";
-import { runThreeSessionTdd } from "../../../src/tdd/orchestrator";
-import { VERDICT_FILE } from "../../../src/tdd/verdict";
-
-let originalSpawn: typeof Bun.spawn;
-
-beforeEach(() => {
-  originalSpawn = Bun.spawn;
-});
-
-afterEach(() => {
-  Bun.spawn = originalSpawn;
-});
-
-/** Create a mock agent that returns sequential results */
-function createMockAgent(results: Partial<AgentResult>[]): AgentAdapter {
-  let callCount = 0;
-  return {
-    name: "mock",
-    displayName: "Mock Agent",
-    binary: "mock",
-    isInstalled: async () => true,
-    buildCommand: () => ["mock"],
-    run: mock(async () => {
-      const r = results[callCount] || {};
-      callCount++;
-      return {
-        success: r.success ?? true,
-        exitCode: r.exitCode ?? 0,
-        output: r.output ?? "",
-        rateLimited: r.rateLimited ?? false,
-        durationMs: r.durationMs ?? 100,
-        estimatedCost: r.estimatedCost ?? 0.01,
-      };
-    }),
-  };
-}
-
-/** Mock Bun.spawn to intercept git commands */
-function mockGitSpawn(opts: {
-  /** Files returned by git diff for each session (indexed by git-diff call number) */
-  diffFiles: string[][];
-  /** Optional: mock test command success (default: true) */
-  testCommandSuccess?: boolean;
-}) {
-  let revParseCount = 0;
-  let diffCount = 0;
-  const testSuccess = opts.testCommandSuccess ?? true;
-
-  // @ts-ignore — mocking global
-  Bun.spawn = mock((cmd: string[], spawnOpts?: any) => {
-    // Intercept test commands (bun test, npm test, etc.)
-    if ((cmd[0] === "/bin/sh" || cmd[0] === "/bin/bash" || cmd[0] === "/bin/zsh") && cmd[1] === "-c") {
-      return {
-        pid: 9999,
-        exited: Promise.resolve(testSuccess ? 0 : 1),
-        stdout: new Response(testSuccess ? "tests pass\n" : "tests fail\n").body,
-        stderr: new Response("").body,
-      };
-    }
-    if (cmd[0] === "git" && cmd[1] === "rev-parse") {
-      revParseCount++;
-      return {
-        exited: Promise.resolve(0),
-        stdout: new Response(`ref-${revParseCount}\n`).body,
-        stderr: new Response("").body,
-      };
-    }
-    if (cmd[0] === "git" && cmd[1] === "checkout") {
-      // Intercept git checkout (used in zero-file fallback) — silently succeed
-      return {
-        exited: Promise.resolve(0),
-        stdout: new Response("").body,
-        stderr: new Response("").body,
-      };
-    }
-    if (cmd[0] === "git" && cmd[1] === "diff") {
-      const files = opts.diffFiles[diffCount] || [];
-      diffCount++;
-      return {
-        exited: Promise.resolve(0),
-        stdout: new Response(files.join("\n") + "\n").body,
-        stderr: new Response("").body,
-      };
-    }
-    return originalSpawn(cmd, spawnOpts);
-  });
-}
-
-const story: UserStory = {
-  id: "US-001",
-  title: "Add user validation",
-  description: "Add validation to user input",
-  acceptanceCriteria: ["Validation works", "Errors are clear"],
-  dependencies: [],
-  tags: [],
-  status: "pending",
-  passes: false,
-  escalations: [],
-  attempts: 0,
-};
-
-
-describe("runThreeSessionTdd — zero-file fallback", () => {
-  /** Extended git mock that also handles `git checkout .` */
-  function mockGitSpawnWithCheckout(opts: {
-    diffFiles: string[][];
-    onCheckout?: () => void;
-    testCommandSuccess?: boolean;
-  }) {
-    let revParseCount = 0;
-    let diffCount = 0;
-    const testSuccess = opts.testCommandSuccess ?? true;
-
-    // @ts-ignore — mocking global
-    Bun.spawn = mock((cmd: string[], spawnOpts?: any) => {
-      // Intercept test commands
-      if ((cmd[0] === "/bin/sh" || cmd[0] === "/bin/bash" || cmd[0] === "/bin/zsh") && cmd[1] === "-c") {
-        return {
-          pid: 9999,
-          exited: Promise.resolve(testSuccess ? 0 : 1),
-          stdout: new Response(testSuccess ? "tests pass\n" : "tests fail\n").body,
-          stderr: new Response("").body,
-        };
-      }
-      if (cmd[0] === "git" && cmd[1] === "rev-parse") {
-        revParseCount++;
-        return {
-          exited: Promise.resolve(0),
-          stdout: new Response(`ref-${revParseCount}\n`).body,
-          stderr: new Response("").body,
-        };
-      }
-      if (cmd[0] === "git" && cmd[1] === "checkout") {
-        opts.onCheckout?.();
-        return {
-          exited: Promise.resolve(0),
-          stdout: new Response("").body,
-          stderr: new Response("").body,
-        };
-      }
-      if (cmd[0] === "git" && cmd[1] === "diff") {
-        const files = opts.diffFiles[diffCount] || [];
-        diffCount++;
-        return {
-          exited: Promise.resolve(0),
-          stdout: new Response(files.join("\n") + "\n").body,
-          stderr: new Response("").body,
-        };
-      }
-      return originalSpawn(cmd, spawnOpts);
-    });
-  }
-
-  test("fallback NO LONGER triggers when strategy='auto' and 0 test files (BUG-010 removed auto-fallback)", async () => {
-    let checkoutCalled = false;
-
-    // BUG-010: Zero-file scenarios now return greenfield-no-tests immediately
-    // No fallback to lite mode occurs
-    mockGitSpawnWithCheckout({
-      diffFiles: [
-        ["requirements.md"], // s1 isolation (strict) — no source violations
-        ["requirements.md"], // s1 getChangedFiles (strict) — 0 test files → return greenfield-no-tests
-      ],
-      onCheckout: () => {
-        checkoutCalled = true;
-      },
-    });
-
-    const agent = createMockAgent([
-      { success: true, estimatedCost: 0.01 }, // s1 strict test-writer
-    ]);
-
-    const configWithAutoStrategy = {
-      ...DEFAULT_CONFIG,
-      tdd: { ...DEFAULT_CONFIG.tdd, strategy: "auto" as const },
-    };
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: configWithAutoStrategy,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(checkoutCalled).toBe(false); // git checkout NOT called (no fallback)
-    expect(result.lite).toBe(false); // not in lite mode
-    expect(result.success).toBe(false); // fails with greenfield-no-tests
-    expect(result.failureCategory).toBe("greenfield-no-tests");
-  });
-
-  test("zero-file scenario returns greenfield-no-tests (BUG-010 removed lite fallback)", async () => {
-    // BUG-010: No more auto-fallback to lite mode
-    mockGitSpawn({
-      diffFiles: [
-        ["docs/plan.md"], // s1 isolation (strict)
-        ["docs/plan.md"], // s1 getChangedFiles (strict) → 0 test files
-      ],
-    });
-
-    const agent = createMockAgent([{ success: true, estimatedCost: 0.01 }]);
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: DEFAULT_CONFIG,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    expect(result.lite).toBe(false);
-    expect(result.success).toBe(false);
-    expect(result.failureCategory).toBe("greenfield-no-tests");
-  });
-
-  test("fallback does NOT trigger when strategy='strict' (explicit strict mode)", async () => {
-    // In strategy='strict', no fallback — should return failure
-    mockGitSpawn({
-      diffFiles: [
-        ["requirements.md"], // s1 isolation — no source violations
-        ["requirements.md"], // s1 getChangedFiles — 0 test files
-      ],
-    });
-
-    const agent = createMockAgent([{ success: true, estimatedCost: 0.01 }]);
-
-    const configWithStrictStrategy = {
-      ...DEFAULT_CONFIG,
-      tdd: { ...DEFAULT_CONFIG.tdd, strategy: "strict" as const },
-    };
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: configWithStrictStrategy,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-    });
-
-    // Should fail (no fallback in strict mode)
-    expect(result.success).toBe(false);
-    expect(result.needsHumanReview).toBe(true);
-    expect(result.reviewReason).toBe("Test writer session created no test files (greenfield project)");
-    expect(result.lite).toBe(false); // Was called in strict mode, no fallback
-  });
-
-  test("fallback does NOT trigger when already in lite mode", async () => {
-    // Calling with lite=true — if 0 test files, should return failure (not recurse again)
-    mockGitSpawn({
-      diffFiles: [
-        ["requirements.md"], // s1 getChangedFiles (lite, no isolation) — 0 test files
-      ],
-    });
-
-    const agent = createMockAgent([{ success: true, estimatedCost: 0.01 }]);
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: DEFAULT_CONFIG,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-      lite: true,
-    });
-
-    // Should fail — no further fallback from lite mode
-    expect(result.success).toBe(false);
-    expect(result.needsHumanReview).toBe(true);
-    expect(result.reviewReason).toBe("Test writer session created no test files (greenfield project)");
-    expect(result.lite).toBe(true);
-  });
-
-  test("fallback does NOT trigger when strategy='lite' config", async () => {
-    // When strategy='lite', runThreeSessionTdd is called with lite=true (from execution stage)
-    // So !lite = false → no fallback
-    mockGitSpawn({
-      diffFiles: [
-        [], // s1 getChangedFiles (lite, no isolation) — 0 test files
-      ],
-    });
-
-    const agent = createMockAgent([{ success: true, estimatedCost: 0.01 }]);
-
-    const configWithLiteStrategy = {
-      ...DEFAULT_CONFIG,
-      tdd: { ...DEFAULT_CONFIG.tdd, strategy: "lite" as const },
-    };
-
-    const result = await runThreeSessionTdd({
-      agent,
-      story,
-      config: configWithLiteStrategy,
-      workdir: "/tmp/test",
-      modelTier: "balanced",
-      lite: true, // router sets this for lite strategy
-    });
-
-    expect(result.success).toBe(false);
-    expect(result.lite).toBe(true);
-  });
-});
-
-// ─── T4: failureCategory tests ────────────────────────────────────────────────
-