@nathapp/nax 0.18.1

package/test/integration/test-scanner.test.ts

@@ -0,0 +1,403 @@
+import { describe, expect, test } from "bun:test";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import {
+  type TestFileInfo,
+  deriveTestPatterns,
+  extractTestStructure,
+  formatTestSummary,
+  generateTestCoverageSummary,
+  scanTestFiles,
+  truncateToTokenBudget,
+} from "../../src/context/test-scanner";
+
+describe("extractTestStructure", () => {
+  test("extracts describe and test blocks", () => {
+    const source = `
+describe("Store", () => {
+  test("creates a record", () => {});
+  test("reads a record", () => {});
+});
+
+describe("Validation", () => {
+  it("rejects empty name", () => {});
+});
+`;
+    const result = extractTestStructure(source);
+    expect(result.testCount).toBe(3);
+    expect(result.describes).toHaveLength(2);
+    expect(result.describes[0].name).toBe("Store");
+    expect(result.describes[0].tests).toEqual(["creates a record", "reads a record"]);
+    expect(result.describes[1].name).toBe("Validation");
+    expect(result.describes[1].tests).toEqual(["rejects empty name"]);
+  });
+
+  test("handles single-quoted strings", () => {
+    const source = `describe('Auth', () => { it('logs in', () => {}); });`;
+    const result = extractTestStructure(source);
+    expect(result.describes[0].name).toBe("Auth");
+    expect(result.describes[0].tests).toEqual(["logs in"]);
+  });
+
+  test("handles backtick strings", () => {
+    const source = "describe(`Math utils`, () => { test(`adds numbers`, () => {}); });";
+    const result = extractTestStructure(source);
+    expect(result.describes[0].name).toBe("Math utils");
+  });
+
+  test("handles top-level tests without describe", () => {
+    const source = `
+test("standalone test 1", () => {});
+test("standalone test 2", () => {});
+`;
+    const result = extractTestStructure(source);
+    expect(result.testCount).toBe(2);
+    expect(result.describes).toHaveLength(1);
+    expect(result.describes[0].name).toBe("(top-level)");
+    expect(result.describes[0].tests).toHaveLength(2);
+  });
+
+  test("returns empty for file with no tests", () => {
+    const source = `export function helper() { return 42; }`;
+    const result = extractTestStructure(source);
+    expect(result.testCount).toBe(0);
+    expect(result.describes).toHaveLength(0);
+  });
+
+  test("handles mixed describe and top-level tests", () => {
+    const source = `
+describe("Suite", () => {
+  test("in suite", () => {});
+});
+`;
+    const result = extractTestStructure(source);
+    expect(result.testCount).toBe(1);
+    expect(result.describes).toHaveLength(1);
+  });
+});
+
+describe("formatTestSummary", () => {
+  const files: TestFileInfo[] = [
+    {
+      relativePath: "test/store.test.ts",
+      testCount: 5,
+      describes: [{ name: "CRUD", tests: ["create", "read", "update", "delete", "upsert"] }],
+    },
+    {
+      relativePath: "test/validation.test.ts",
+      testCount: 3,
+      describes: [{ name: "Input validation", tests: ["required name", "max length", "type check"] }],
+    },
+  ];
+
+  test("names-only shows file and count", () => {
+    const result = formatTestSummary(files, "names-only");
+    expect(result).toContain("test/store.test.ts");
+    expect(result).toContain("(5 tests)");
+    expect(result).toContain("test/validation.test.ts");
+    expect(result).toContain("(3 tests)");
+    expect(result).not.toContain("CRUD");
+  });
+
+  test("names-and-counts shows describe blocks", () => {
+    const result = formatTestSummary(files, "names-and-counts");
+    expect(result).toContain("CRUD (5 tests)");
+    expect(result).toContain("Input validation (3 tests)");
+    expect(result).not.toContain("create");
+  });
+
+  test("describe-blocks shows individual test names", () => {
+    const result = formatTestSummary(files, "describe-blocks");
+    expect(result).toContain("create");
+    expect(result).toContain("read");
+    expect(result).toContain("required name");
+  });
+
+  test("includes dedup instruction", () => {
+    const result = formatTestSummary(files, "names-only");
+    expect(result).toContain("DO NOT duplicate");
+  });
+
+  test("shows total count in header", () => {
+    const result = formatTestSummary(files, "names-only");
+    expect(result).toContain("8 tests across 2 files");
+  });
+
+  test("returns empty string for no files", () => {
+    expect(formatTestSummary([], "names-only")).toBe("");
+  });
+});
+
+describe("truncateToTokenBudget", () => {
+  const files: TestFileInfo[] = [
+    {
+      relativePath: "test/store.test.ts",
+      testCount: 10,
+      describes: [{ name: "CRUD", tests: Array.from({ length: 10 }, (_, i) => `test ${i}`) }],
+    },
+    {
+      relativePath: "test/auth.test.ts",
+      testCount: 8,
+      describes: [{ name: "Auth", tests: Array.from({ length: 8 }, (_, i) => `auth test ${i}`) }],
+    },
+  ];
+
+  test("uses preferred detail if within budget", () => {
+    const result = truncateToTokenBudget(files, 5000, "describe-blocks");
+    expect(result.detail).toBe("describe-blocks");
+    expect(result.truncated).toBe(false);
+  });
+
+  test("falls back to simpler detail if over budget", () => {
+    // Very tight budget — force fallback
+    const result = truncateToTokenBudget(files, 50, "describe-blocks");
+    expect(result.truncated).toBe(true);
+  });
+
+  test("handles tiny budget with fallback message", () => {
+    const result = truncateToTokenBudget(files, 10, "describe-blocks");
+    expect(result.truncated).toBe(true);
+    expect(result.summary).toContain("test files");
+  });
+});
+
+describe("deriveTestPatterns", () => {
+  test("derives test patterns from source file paths", () => {
+    const contextFiles = ["src/health.service.ts", "src/db/connection.ts"];
+    const patterns = deriveTestPatterns(contextFiles);
+
+    // Should generate patterns for health.service.ts
+    expect(patterns).toContain("health.service.test.ts");
+    expect(patterns).toContain("health.service.spec.ts");
+    expect(patterns).toContain("health.test.ts"); // Simple basename without .service
+
+    // Should generate patterns for connection.ts
+    expect(patterns).toContain("connection.test.ts");
+    expect(patterns).toContain("connection.spec.ts");
+  });
+
+  test("handles files without special suffixes", () => {
+    const contextFiles = ["src/utils.ts"];
+    const patterns = deriveTestPatterns(contextFiles);
+
+    expect(patterns).toContain("utils.test.ts");
+    expect(patterns).toContain("utils.spec.ts");
+    expect(patterns).toContain("utils.test.js");
+    expect(patterns).toContain("utils.spec.js");
+  });
+
+  test("handles various file extensions", () => {
+    const contextFiles = ["src/component.tsx", "src/script.jsx"];
+    const patterns = deriveTestPatterns(contextFiles);
+
+    expect(patterns).toContain("component.test.tsx");
+    expect(patterns).toContain("component.spec.tsx");
+    expect(patterns).toContain("script.test.jsx");
+    expect(patterns).toContain("script.spec.jsx");
+  });
+
+  test("strips common suffixes like .service, .controller, .module", () => {
+    const contextFiles = ["src/user.service.ts", "src/api.controller.ts", "src/app.module.ts"];
+    const patterns = deriveTestPatterns(contextFiles);
+
+    // Should include both full and simplified patterns
+    expect(patterns).toContain("user.service.test.ts");
+    expect(patterns).toContain("user.test.ts"); // Simplified
+    expect(patterns).toContain("api.controller.test.ts");
+    expect(patterns).toContain("api.test.ts"); // Simplified
+  });
+
+  test("returns empty array for empty input", () => {
+    const patterns = deriveTestPatterns([]);
+    expect(patterns).toEqual([]);
+  });
+
+  test("deduplicates patterns", () => {
+    const contextFiles = ["src/foo.ts", "src/foo.service.ts"];
+    const patterns = deriveTestPatterns(contextFiles);
+
+    // Both files generate "foo.test.ts", should only appear once
+    const fooTestCount = patterns.filter((p) => p === "foo.test.ts").length;
+    expect(fooTestCount).toBe(1);
+  });
+});
+
+describe("scanTestFiles with scoping", () => {
+  test("scopes test files to contextFiles when scopeToStory=true", async () => {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "nax-test-scanner-"));
+
+    try {
+      // Create test directory structure
+      const testDir = path.join(tempDir, "test");
+      await fs.mkdir(testDir);
+
+      // Create test files
+      await fs.writeFile(
+        path.join(testDir, "health.service.test.ts"),
+        'describe("Health Service", () => { test("works", () => {}); });',
+      );
+      await fs.writeFile(
+        path.join(testDir, "db.connection.test.ts"),
+        'describe("DB Connection", () => { test("connects", () => {}); });',
+      );
+      await fs.writeFile(
+        path.join(testDir, "auth.service.test.ts"),
+        'describe("Auth Service", () => { test("authenticates", () => {}); });',
+      );
+
+      // Scan with contextFiles (only health.service.ts)
+      const result = await scanTestFiles({
+        workdir: tempDir,
+        testDir: "test",
+        contextFiles: ["src/health.service.ts"],
+        scopeToStory: true,
+      });
+
+      // Should only include health.service.test.ts
+      expect(result.length).toBe(1);
+      expect(result[0].relativePath).toBe("test/health.service.test.ts");
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  test("scans all test files when scopeToStory=false", async () => {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "nax-test-scanner-"));
+
+    try {
+      const testDir = path.join(tempDir, "test");
+      await fs.mkdir(testDir);
+
+      await fs.writeFile(
+        path.join(testDir, "health.service.test.ts"),
+        'describe("Health", () => { test("works", () => {}); });',
+      );
+      await fs.writeFile(
+        path.join(testDir, "auth.service.test.ts"),
+        'describe("Auth", () => { test("works", () => {}); });',
+      );
+
+      // Scan with scopeToStory=false (should scan all)
+      const result = await scanTestFiles({
+        workdir: tempDir,
+        testDir: "test",
+        contextFiles: ["src/health.service.ts"],
+        scopeToStory: false,
+      });
+
+      expect(result.length).toBe(2);
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  test("falls back to full scan when no contextFiles provided", async () => {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "nax-test-scanner-"));
+
+    try {
+      const testDir = path.join(tempDir, "test");
+      await fs.mkdir(testDir);
+
+      await fs.writeFile(path.join(testDir, "test1.test.ts"), 'describe("Test1", () => { test("works", () => {}); });');
+      await fs.writeFile(path.join(testDir, "test2.test.ts"), 'describe("Test2", () => { test("works", () => {}); });');
+
+      // No contextFiles, should scan all
+      const result = await scanTestFiles({
+        workdir: tempDir,
+        testDir: "test",
+        scopeToStory: true,
+      });
+
+      expect(result.length).toBe(2);
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  });
+});
+
+describe("generateTestCoverageSummary with scoping", () => {
+  test("generates scoped summary when contextFiles provided", async () => {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "nax-test-scanner-"));
+
+    try {
+      const testDir = path.join(tempDir, "test");
+      await fs.mkdir(testDir);
+
+      await fs.writeFile(
+        path.join(testDir, "health.service.test.ts"),
+        'describe("Health", () => { test("check", () => {}); });',
+      );
+      await fs.writeFile(
+        path.join(testDir, "auth.service.test.ts"),
+        'describe("Auth", () => { test("login", () => {}); });',
+      );
+
+      const result = await generateTestCoverageSummary({
+        workdir: tempDir,
+        testDir: "test",
+        contextFiles: ["src/health.service.ts"],
+        scopeToStory: true,
+        maxTokens: 500,
+        detail: "names-and-counts",
+      });
+
+      // Should only include health.service.test.ts
+      expect(result.files.length).toBe(1);
+      expect(result.totalTests).toBe(1);
+      expect(result.summary).toContain("health.service.test.ts");
+      expect(result.summary).not.toContain("auth.service.test.ts");
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  test("falls back to full scan when scopeToStory=true but no contextFiles", async () => {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "nax-test-scanner-"));
+
+    try {
+      const testDir = path.join(tempDir, "test");
+      await fs.mkdir(testDir);
+
+      await fs.writeFile(path.join(testDir, "test.test.ts"), 'describe("Test", () => { test("works", () => {}); });');
+
+      // scopeToStory=true but no contextFiles → should fall back to full scan
+      // (warning logged via structured logger, not console.warn)
+      const result = await generateTestCoverageSummary({
+        workdir: tempDir,
+        testDir: "test",
+        scopeToStory: true, // true but no contextFiles
+        maxTokens: 500,
+      });
+
+      // Should still scan all files (fallback behavior)
+      expect(result.totalTests).toBeGreaterThan(0);
+      expect(result.files.length).toBe(1);
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  test("returns empty result when no test files found", async () => {
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "nax-test-scanner-"));
+
+    try {
+      const testDir = path.join(tempDir, "test");
+      await fs.mkdir(testDir);
+
+      const result = await generateTestCoverageSummary({
+        workdir: tempDir,
+        testDir: "test",
+        contextFiles: ["src/health.service.ts"],
+        scopeToStory: true,
+      });
+
+      expect(result.files).toEqual([]);
+      expect(result.totalTests).toBe(0);
+      expect(result.summary).toBe("");
+      expect(result.tokens).toBe(0);
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
+  });
+});

package/test/integration/verification-asset-check.test.ts

@@ -0,0 +1,142 @@
+/**
+ * Tests for asset verification with contextFiles/expectedFiles split
+ */
+
+import { describe, expect, test } from "bun:test";
+import { getExpectedFiles } from "../../src/prd";
+import type { UserStory } from "../../src/prd";
+
+const createStory = (partial: Partial<UserStory>): UserStory => ({
+  id: "US-001",
+  title: "Test Story",
+  description: "Test description",
+  acceptanceCriteria: ["AC1"],
+  tags: [],
+  dependencies: [],
+  status: "pending",
+  passes: false,
+  escalations: [],
+  attempts: 0,
+  ...partial,
+});
+
+describe("Asset verification behavior", () => {
+  test("story with relevantFiles but no expectedFiles should skip asset check", () => {
+    const story = createStory({
+      relevantFiles: ["src/foo.ts", "src/bar.ts"],
+    });
+
+    const filesToVerify = getExpectedFiles(story);
+
+    // Asset check is opt-in only - empty array means skip verification
+    expect(filesToVerify).toEqual([]);
+  });
+
+  test("story with expectedFiles should verify those files", () => {
+    const story = createStory({
+      expectedFiles: ["dist/output.js", "dist/types.d.ts"],
+    });
+
+    const filesToVerify = getExpectedFiles(story);
+
+    expect(filesToVerify).toEqual(["dist/output.js", "dist/types.d.ts"]);
+  });
+
+  test("story with both contextFiles and expectedFiles should only verify expectedFiles", () => {
+    const story = createStory({
+      contextFiles: ["src/input.ts", "src/helper.ts"],
+      expectedFiles: ["dist/output.js"],
+    });
+
+    const filesToVerify = getExpectedFiles(story);
+
+    // Only expectedFiles are verified, not contextFiles
+    expect(filesToVerify).toEqual(["dist/output.js"]);
+  });
+
+  test("story with no files specified should skip asset check", () => {
+    const story = createStory({});
+
+    const filesToVerify = getExpectedFiles(story);
+
+    expect(filesToVerify).toEqual([]);
+  });
+
+  test("story with empty expectedFiles array should skip asset check", () => {
+    const story = createStory({
+      expectedFiles: [],
+    });
+
+    const filesToVerify = getExpectedFiles(story);
+
+    expect(filesToVerify).toEqual([]);
+  });
+
+  test("legacy story with relevantFiles for both context and verification", () => {
+    // Old behavior: relevantFiles used for both context AND verification
+    // New behavior: relevantFiles used ONLY for context fallback, NOT verification
+    const legacyStory = createStory({
+      relevantFiles: ["src/module.ts"],
+    });
+
+    const filesToVerify = getExpectedFiles(legacyStory);
+
+    // CRITICAL: No automatic verification of relevantFiles
+    // This prevents false negatives from LLM-hallucinated filenames
+    expect(filesToVerify).toEqual([]);
+  });
+
+  test("migrated story with explicit expectedFiles for verification", () => {
+    const migratedStory = createStory({
+      contextFiles: ["src/module.ts"],
+      expectedFiles: ["src/module.ts"], // Explicitly opt-in to verification
+    });
+
+    const filesToVerify = getExpectedFiles(migratedStory);
+
+    expect(filesToVerify).toEqual(["src/module.ts"]);
+  });
+});
+
+describe("Verification scenarios from dogfood runs", () => {
+  test("Run F scenario: LLM predicts wrong filename but code is correct", () => {
+    // Context: LLM predicted "src/cli/status.ts" but actual file was "src/cli/queue-status.ts"
+    // Old behavior: Asset check fails even though code is correct
+    // New behavior: Asset check is opt-in only
+    const story = createStory({
+      relevantFiles: ["src/cli/status.ts"], // LLM prediction (wrong)
+      // No expectedFiles - asset check is opt-in
+    });
+
+    const filesToVerify = getExpectedFiles(story);
+
+    // Asset check skipped - no false negative
+    expect(filesToVerify).toEqual([]);
+  });
+
+  test("Run H scenario: explicit expectedFiles for critical output", () => {
+    // Context: When output files MUST exist, explicitly set expectedFiles
+    const story = createStory({
+      contextFiles: ["src/input.ts"], // Context for agent
+      expectedFiles: ["dist/bundle.js", "dist/bundle.css"], // Must exist
+    });
+
+    const filesToVerify = getExpectedFiles(story);
+
+    // Asset check runs on explicitly specified files only
+    expect(filesToVerify).toEqual(["dist/bundle.js", "dist/bundle.css"]);
+  });
+
+  test("common case: simple refactor with context but no asset requirement", () => {
+    // Most stories: agent needs context but doesn't create new files
+    const story = createStory({
+      contextFiles: ["src/utils/formatter.ts", "src/types.ts"],
+      // No expectedFiles - just modifying existing files
+    });
+
+    const filesToVerify = getExpectedFiles(story);
+
+    // No asset check needed
+    expect(filesToVerify).toEqual([]);
+  });
+});