@nathapp/nax 0.18.1 → 0.18.3

package/test/execution/structured-failure.test.ts

@@ -0,0 +1,414 @@
+/**
+ * Unit tests for StructuredFailure and priorFailures tracking
+ *
+ * Tests the structured failure context for escalated tiers to know exactly what failed.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { loadPRD } from "../../src/prd";
+import type { StructuredFailure, TestFailureContext, UserStory } from "../../src/prd";
+
+describe("StructuredFailure Type", () => {
+  test("should have all required fields", () => {
+    const failure: StructuredFailure = {
+      attempt: 1,
+      modelTier: "balanced",
+      stage: "verify",
+      summary: "Test failed",
+      timestamp: new Date().toISOString(),
+    };
+
+    expect(failure.attempt).toBe(1);
+    expect(failure.modelTier).toBe("balanced");
+    expect(failure.stage).toBe("verify");
+    expect(failure.summary).toBe("Test failed");
+    expect(failure.timestamp).toBeDefined();
+    expect(typeof failure.timestamp).toBe("string");
+  });
+
+  test("should have optional testFailures field", () => {
+    const testFailure: TestFailureContext = {
+      file: "test/foo.test.ts",
+      testName: "should pass",
+      error: "Expected 1 to equal 2",
+      stackTrace: ["at foo.ts:10:15", "at Object.test (foo.ts:8:3)"],
+    };
+
+    const failure: StructuredFailure = {
+      attempt: 1,
+      modelTier: "balanced",
+      stage: "verify",
+      summary: "Test failed",
+      testFailures: [testFailure],
+      timestamp: new Date().toISOString(),
+    };
+
+    expect(failure.testFailures).toBeDefined();
+    expect(failure.testFailures?.length).toBe(1);
+    expect(failure.testFailures?.[0].file).toBe("test/foo.test.ts");
+    expect(failure.testFailures?.[0].testName).toBe("should pass");
+    expect(failure.testFailures?.[0].error).toBe("Expected 1 to equal 2");
+    expect(failure.testFailures?.[0].stackTrace.length).toBe(2);
+  });
+
+  test("should support all verification stages", () => {
+    const stages: Array<StructuredFailure["stage"]> = [
+      "verify",
+      "review",
+      "regression",
+      "rectification",
+      "agent-session",
+      "escalation",
+    ];
+
+    for (const stage of stages) {
+      const failure: StructuredFailure = {
+        attempt: 1,
+        modelTier: "balanced",
+        stage,
+        summary: "Test failed",
+        timestamp: new Date().toISOString(),
+      };
+      expect(failure.stage).toBe(stage);
+    }
+  });
+
+  test("should allow different model tiers", () => {
+    const tiers = ["fast", "balanced", "powerful"];
+
+    for (const tier of tiers) {
+      const failure: StructuredFailure = {
+        attempt: 1,
+        modelTier: tier,
+        stage: "verify",
+        summary: "Test failed",
+        timestamp: new Date().toISOString(),
+      };
+      expect(failure.modelTier).toBe(tier);
+    }
+  });
+
+  test("should track multiple test failures", () => {
+    const testFailures: TestFailureContext[] = [
+      {
+        file: "test/foo.test.ts",
+        testName: "test 1",
+        error: "Error 1",
+        stackTrace: ["at foo.ts:10"],
+      },
+      {
+        file: "test/bar.test.ts",
+        testName: "test 2",
+        error: "Error 2",
+        stackTrace: ["at bar.ts:20"],
+      },
+      {
+        file: "test/baz.test.ts",
+        testName: "test 3",
+        error: "Error 3",
+        stackTrace: ["at baz.ts:30"],
+      },
+    ];
+
+    const failure: StructuredFailure = {
+      attempt: 2,
+      modelTier: "balanced",
+      stage: "regression",
+      summary: "Multiple test failures",
+      testFailures,
+      timestamp: new Date().toISOString(),
+    };
+
+    expect(failure.testFailures?.length).toBe(3);
+    expect(failure.testFailures?.[0].file).toBe("test/foo.test.ts");
+    expect(failure.testFailures?.[1].file).toBe("test/bar.test.ts");
+    expect(failure.testFailures?.[2].file).toBe("test/baz.test.ts");
+  });
+});
+
+describe("TestFailureContext Type", () => {
+  test("should have all required fields", () => {
+    const context: TestFailureContext = {
+      file: "test/example.test.ts",
+      testName: "should do something",
+      error: "AssertionError: expected true to be false",
+      stackTrace: ["at Object.test (example.test.ts:42:10)", "at async runTest (test.ts:100:5)"],
+    };
+
+    expect(context.file).toBe("test/example.test.ts");
+    expect(context.testName).toBe("should do something");
+    expect(context.error).toBe("AssertionError: expected true to be false");
+    expect(context.stackTrace.length).toBe(2);
+  });
+
+  test("should handle nested test names", () => {
+    const context: TestFailureContext = {
+      file: "test/example.test.ts",
+      testName: "describe block > nested block > test name",
+      error: "Error",
+      stackTrace: [],
+    };
+
+    expect(context.testName).toContain("describe block");
+    expect(context.testName).toContain("nested block");
+    expect(context.testName).toContain("test name");
+  });
+
+  test("should support empty stack traces", () => {
+    const context: TestFailureContext = {
+      file: "test/example.test.ts",
+      testName: "test",
+      error: "Error",
+      stackTrace: [],
+    };
+
+    expect(context.stackTrace.length).toBe(0);
+  });
+
+  test("should support multiple stack trace lines", () => {
+    const context: TestFailureContext = {
+      file: "test/example.test.ts",
+      testName: "test",
+      error: "Error",
+      stackTrace: [
+        "at foo.ts:10:15",
+        "at bar.ts:20:10",
+        "at baz.ts:30:5",
+        "at Object.test (example.ts:40:3)",
+        "at async runTest (test.ts:50:5)",
+      ],
+    };
+
+    expect(context.stackTrace.length).toBe(5);
+  });
+});
+
+describe("UserStory priorFailures Field", () => {
+  test("should have optional priorFailures field", () => {
+    const story: UserStory = {
+      id: "US-001",
+      title: "Test Story",
+      description: "A test story",
+      acceptanceCriteria: [],
+      tags: [],
+      dependencies: [],
+      status: "pending",
+      passes: false,
+      escalations: [],
+      attempts: 0,
+    };
+
+    expect(story.priorFailures).toBeUndefined();
+  });
+
+  test("should initialize priorFailures to empty array in loadPRD", async () => {
+    // Create a temporary PRD file without priorFailures
+    const prdContent = JSON.stringify({
+      project: "test",
+      feature: "test-feature",
+      branchName: "test-branch",
+      createdAt: new Date().toISOString(),
+      updatedAt: new Date().toISOString(),
+      userStories: [
+        {
+          id: "US-001",
+          title: "Test Story",
+          description: "Description",
+          acceptanceCriteria: [],
+          tags: [],
+          dependencies: [],
+          status: "pending",
+          passes: false,
+          escalations: [],
+          attempts: 0,
+        },
+      ],
+    });
+
+    // Write to temp file
+    const tmpFile = "/tmp/test-prd-priorFailures.json";
+    await Bun.write(tmpFile, prdContent);
+
+    // Load and verify
+    const prd = await loadPRD(tmpFile);
+    expect(prd.userStories[0].priorFailures).toBeDefined();
+    expect(Array.isArray(prd.userStories[0].priorFailures)).toBe(true);
+    expect(prd.userStories[0].priorFailures?.length).toBe(0);
+
+    // Cleanup
+    await Bun.file(tmpFile).delete();
+  });
+
+  test("should preserve existing priorFailures when loading PRD", async () => {
+    const existingFailure: StructuredFailure = {
+      attempt: 1,
+      modelTier: "balanced",
+      stage: "verify",
+      summary: "Test failed",
+      timestamp: new Date().toISOString(),
+    };
+
+    const prdContent = JSON.stringify({
+      project: "test",
+      feature: "test-feature",
+      branchName: "test-branch",
+      createdAt: new Date().toISOString(),
+      updatedAt: new Date().toISOString(),
+      userStories: [
+        {
+          id: "US-001",
+          title: "Test Story",
+          description: "Description",
+          acceptanceCriteria: [],
+          tags: [],
+          dependencies: [],
+          status: "pending",
+          passes: false,
+          escalations: [],
+          attempts: 1,
+          priorFailures: [existingFailure],
+        },
+      ],
+    });
+
+    const tmpFile = "/tmp/test-prd-existing-failures.json";
+    await Bun.write(tmpFile, prdContent);
+
+    const prd = await loadPRD(tmpFile);
+    expect(prd.userStories[0].priorFailures?.length).toBe(1);
+    expect(prd.userStories[0].priorFailures?.[0].attempt).toBe(1);
+    expect(prd.userStories[0].priorFailures?.[0].stage).toBe("verify");
+
+    await Bun.file(tmpFile).delete();
+  });
+
+  test("should allow adding multiple priorFailures to a story", () => {
+    const story: UserStory = {
+      id: "US-001",
+      title: "Test Story",
+      description: "Description",
+      acceptanceCriteria: [],
+      tags: [],
+      dependencies: [],
+      status: "failed",
+      passes: false,
+      escalations: [],
+      attempts: 3,
+      priorFailures: [
+        {
+          attempt: 1,
+          modelTier: "fast",
+          stage: "verify",
+          summary: "First failure",
+          timestamp: new Date().toISOString(),
+        },
+        {
+          attempt: 2,
+          modelTier: "balanced",
+          stage: "regression",
+          summary: "Second failure",
+          timestamp: new Date().toISOString(),
+        },
+      ],
+    };
+
+    expect(story.priorFailures?.length).toBe(2);
+    expect(story.priorFailures?.[0].modelTier).toBe("fast");
+    expect(story.priorFailures?.[1].modelTier).toBe("balanced");
+  });
+});
+
+describe("StructuredFailure Attempt Tracking", () => {
+  test("should increment attempt number correctly", () => {
+    const failures: StructuredFailure[] = [];
+
+    for (let i = 1; i <= 3; i++) {
+      failures.push({
+        attempt: i,
+        modelTier: "balanced",
+        stage: "verify",
+        summary: `Attempt ${i} failed`,
+        timestamp: new Date().toISOString(),
+      });
+    }
+
+    expect(failures[0].attempt).toBe(1);
+    expect(failures[1].attempt).toBe(2);
+    expect(failures[2].attempt).toBe(3);
+  });
+
+  test("should track tier escalation in priorFailures", () => {
+    const failures: StructuredFailure[] = [
+      {
+        attempt: 1,
+        modelTier: "fast",
+        stage: "verify",
+        summary: "Failed on fast tier",
+        timestamp: new Date().toISOString(),
+      },
+      {
+        attempt: 2,
+        modelTier: "balanced",
+        stage: "escalation",
+        summary: "Escalated to balanced tier",
+        timestamp: new Date().toISOString(),
+      },
+      {
+        attempt: 3,
+        modelTier: "powerful",
+        stage: "escalation",
+        summary: "Escalated to powerful tier",
+        timestamp: new Date().toISOString(),
+      },
+    ];
+
+    expect(failures.length).toBe(3);
+    expect(failures[0].modelTier).toBe("fast");
+    expect(failures[1].modelTier).toBe("balanced");
+    expect(failures[2].modelTier).toBe("powerful");
+
+    // Verify escalation stages
+    expect(failures[1].stage).toBe("escalation");
+    expect(failures[2].stage).toBe("escalation");
+  });
+});
+
+describe("StructuredFailure Timestamps", () => {
+  test("should use ISO format timestamps", () => {
+    const failure: StructuredFailure = {
+      attempt: 1,
+      modelTier: "balanced",
+      stage: "verify",
+      summary: "Test failed",
+      timestamp: new Date().toISOString(),
+    };
+
+    // ISO format: YYYY-MM-DDTHH:mm:ss.sssZ
+    expect(failure.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/);
+  });
+
+  test("should have different timestamps for different failures", async () => {
+    const failure1: StructuredFailure = {
+      attempt: 1,
+      modelTier: "balanced",
+      stage: "verify",
+      summary: "First failure",
+      timestamp: new Date().toISOString(),
+    };
+
+    // Small delay to ensure different timestamp
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    const failure2: StructuredFailure = {
+      attempt: 2,
+      modelTier: "balanced",
+      stage: "verify",
+      summary: "Second failure",
+      timestamp: new Date().toISOString(),
+    };
+
+    // While millisecond precision should make them different, we just verify they're valid
+    expect(failure1.timestamp).toBeDefined();
+    expect(failure2.timestamp).toBeDefined();
+  });
+});

package/test/integration/logger.test.ts

@@ -1,7 +1,7 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { existsSync, readFileSync, rmSync } from "node:fs";
 import path from "node:path";
-import { Logger, getLogger, initLogger, resetLogger } from "../../src/logger/logger.js";
+import { Logger, getLogger, initLogger, resetLogger } from "../../src/logger";
 
 const TEST_LOG_DIR = path.join(process.cwd(), "test-logs");
 const TEST_LOG_FILE = path.join(TEST_LOG_DIR, "test.jsonl");

package/test/integration/review-plugin-integration.test.ts

@@ -124,7 +124,8 @@ describe("Review Stage - Plugin Integration", () => {
       const result = await reviewStage.execute(ctx);
 
       expect(pluginCalled).toBe(false);
-      expect(result.action).toBe("fail");
+      // BUG-030: Review lint/typecheck failures now escalate instead of hard-failing
+      expect(result.action).toBe("escalate");
       expect(result.reason).toContain("Review failed");
     });
 

package/test/integration/story-id-in-events.test.ts

@@ -11,7 +11,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import type { NaxConfig } from "../../src/config/schema";
 import { initLogger, resetLogger } from "../../src/logger";
-import { getLogger } from "../../src/logger/logger";
+import { getLogger } from "../../src/logger";
 import type { LogEntry } from "../../src/logger/types";
 import { executionStage } from "../../src/pipeline/stages/execution";
 import { verifyStage } from "../../src/pipeline/stages/verify";

package/test/unit/config/smart-runner-flag.test.ts

@@ -0,0 +1,249 @@
+/**
+ * Smart Test Runner Config Flag Tests (STR-004)
+ *
+ * Verifies that execution.smartTestRunner is present in the ExecutionConfig
+ * interface, defaults to true in the Zod schema, and loads correctly.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { globalConfigPath, loadConfig } from "../../../src/config/loader";
+import { DEFAULT_CONFIG } from "../../../src/config/defaults";
+import { NaxConfigSchema } from "../../../src/config/schemas";
+
+describe("execution.smartTestRunner config flag", () => {
+  let tempDir: string;
+  let globalBackup: string | null = null;
+
+  beforeEach(() => {
+    tempDir = join(tmpdir(), `nax-str-004-${Date.now()}`);
+    mkdirSync(join(tempDir, "nax"), { recursive: true });
+
+    const globalPath = globalConfigPath();
+    if (existsSync(globalPath)) {
+      globalBackup = `${globalPath}.test-backup-${Date.now()}`;
+      renameSync(globalPath, globalBackup);
+    }
+  });
+
+  afterEach(() => {
+    if (tempDir) {
+      rmSync(tempDir, { recursive: true, force: true });
+    }
+    if (globalBackup && existsSync(globalBackup)) {
+      const globalPath = globalConfigPath();
+      if (existsSync(globalPath)) rmSync(globalPath);
+      renameSync(globalBackup, globalPath);
+      globalBackup = null;
+    }
+  });
+
+  test("DEFAULT_CONFIG has smartTestRunner: true", () => {
+    expect(DEFAULT_CONFIG.execution.smartTestRunner).toBe(true);
+  });
+
+  test("Zod schema defaults smartTestRunner to enabled object when field is absent", () => {
+    const minimal = {
+      version: 1,
+      models: {
+        fast: { provider: "anthropic", model: "haiku" },
+        balanced: { provider: "anthropic", model: "sonnet" },
+        powerful: { provider: "anthropic", model: "opus" },
+      },
+      autoMode: {
+        enabled: true,
+        defaultAgent: "claude",
+        fallbackOrder: [],
+        complexityRouting: { simple: "fast", medium: "balanced", complex: "powerful", expert: "powerful" },
+        escalation: { enabled: true, tierOrder: [{ tier: "fast", attempts: 1 }] },
+      },
+      routing: { strategy: "keyword" },
+      execution: {
+        maxIterations: 10,
+        iterationDelayMs: 0,
+        costLimit: 1,
+        sessionTimeoutSeconds: 60,
+        maxStoriesPerFeature: 10,
+        rectification: {
+          enabled: true,
+          maxRetries: 1,
+          fullSuiteTimeoutSeconds: 30,
+          maxFailureSummaryChars: 500,
+          abortOnIncreasingFailures: true,
+        },
+        regressionGate: { enabled: true, timeoutSeconds: 30 },
+        contextProviderTokenBudget: 100,
+        // smartTestRunner intentionally omitted
+      },
+      quality: {
+        requireTypecheck: true,
+        requireLint: true,
+        requireTests: true,
+        commands: {},
+        forceExit: false,
+        detectOpenHandles: true,
+        detectOpenHandlesRetries: 1,
+        gracePeriodMs: 500,
+        drainTimeoutMs: 0,
+        shell: "/bin/sh",
+        stripEnvVars: [],
+        environmentalEscalationDivisor: 2,
+      },
+      tdd: { maxRetries: 0, autoVerifyIsolation: false, autoApproveVerifier: false },
+      constitution: { enabled: false, path: "constitution.md", maxTokens: 100 },
+      analyze: { llmEnhanced: false, model: "balanced", fallbackToKeywords: true, maxCodebaseSummaryTokens: 100 },
+      review: { enabled: false, checks: [], commands: {} },
+      plan: { model: "balanced", outputPath: "spec.md" },
+      acceptance: { enabled: false, maxRetries: 0, generateTests: false, testPath: "acceptance.test.ts" },
+      context: {
+        testCoverage: { enabled: false, detail: "names-only", maxTokens: 50, testPattern: "**/*.test.ts", scopeToStory: false },
+        autoDetect: { enabled: false, maxFiles: 1, traceImports: false },
+      },
+    };
+
+    const result = NaxConfigSchema.safeParse(minimal);
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.execution.smartTestRunner).toEqual({
+        enabled: true,
+        testFilePatterns: ["test/**/*.test.ts"],
+        fallback: "import-grep",
+      });
+    }
+  });
+
+  test("Zod schema coerces smartTestRunner: false to disabled config object", () => {
+    const result = NaxConfigSchema.safeParse({
+      ...buildMinimalConfig(),
+      execution: { ...buildMinimalConfig().execution, smartTestRunner: false },
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.execution.smartTestRunner).toEqual({
+        enabled: false,
+        testFilePatterns: ["test/**/*.test.ts"],
+        fallback: "import-grep",
+      });
+    }
+  });
+
+  test("Zod schema coerces smartTestRunner: true to enabled config object", () => {
+    const result = NaxConfigSchema.safeParse({
+      ...buildMinimalConfig(),
+      execution: { ...buildMinimalConfig().execution, smartTestRunner: true },
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.execution.smartTestRunner).toEqual({
+        enabled: true,
+        testFilePatterns: ["test/**/*.test.ts"],
+        fallback: "import-grep",
+      });
+    }
+  });
+
+  test("loadConfig defaults smartTestRunner to enabled object when not in project config", async () => {
+    const configPath = join(tempDir, "nax", "config.json");
+    writeFileSync(configPath, JSON.stringify({ routing: { strategy: "keyword" } }, null, 2));
+
+    const config = await loadConfig(join(tempDir, "nax"));
+    expect(config.execution.smartTestRunner).toEqual({
+      enabled: true,
+      testFilePatterns: ["test/**/*.test.ts"],
+      fallback: "import-grep",
+    });
+  });
+
+  test("loadConfig coerces smartTestRunner: false to disabled config object", async () => {
+    const configPath = join(tempDir, "nax", "config.json");
+    writeFileSync(
+      configPath,
+      JSON.stringify({ execution: { smartTestRunner: false } }, null, 2),
+    );
+
+    const config = await loadConfig(join(tempDir, "nax"));
+    expect(config.execution.smartTestRunner).toEqual({
+      enabled: false,
+      testFilePatterns: ["test/**/*.test.ts"],
+      fallback: "import-grep",
+    });
+  });
+
+  test("loadConfig normalizes to enabled object when field is absent (backward compat)", async () => {
+    const configPath = join(tempDir, "nax", "config.json");
+    // Config without smartTestRunner field at all
+    writeFileSync(configPath, JSON.stringify({}, null, 2));
+
+    const config = await loadConfig(join(tempDir, "nax"));
+    expect(config.execution.smartTestRunner).toEqual({
+      enabled: true,
+      testFilePatterns: ["test/**/*.test.ts"],
+      fallback: "import-grep",
+    });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Minimal valid config helper
+// ---------------------------------------------------------------------------
+
+function buildMinimalConfig() {
+  return {
+    version: 1,
+    models: {
+      fast: { provider: "anthropic", model: "haiku" },
+      balanced: { provider: "anthropic", model: "sonnet" },
+      powerful: { provider: "anthropic", model: "opus" },
+    },
+    autoMode: {
+      enabled: true,
+      defaultAgent: "claude",
+      fallbackOrder: [],
+      complexityRouting: { simple: "fast", medium: "balanced", complex: "powerful", expert: "powerful" },
+      escalation: { enabled: true, tierOrder: [{ tier: "fast", attempts: 1 }] },
+    },
+    routing: { strategy: "keyword" as const },
+    execution: {
+      maxIterations: 10,
+      iterationDelayMs: 0,
+      costLimit: 1,
+      sessionTimeoutSeconds: 60,
+      maxStoriesPerFeature: 10,
+      rectification: {
+        enabled: true,
+        maxRetries: 1,
+        fullSuiteTimeoutSeconds: 30,
+        maxFailureSummaryChars: 500,
+        abortOnIncreasingFailures: true,
+      },
+      regressionGate: { enabled: true, timeoutSeconds: 30 },
+      contextProviderTokenBudget: 100,
+    },
+    quality: {
+      requireTypecheck: true,
+      requireLint: true,
+      requireTests: true,
+      commands: {},
+      forceExit: false,
+      detectOpenHandles: true,
+      detectOpenHandlesRetries: 1,
+      gracePeriodMs: 500,
+      drainTimeoutMs: 0,
+      shell: "/bin/sh",
+      stripEnvVars: [],
+      environmentalEscalationDivisor: 2,
+    },
+    tdd: { maxRetries: 0, autoVerifyIsolation: false, autoApproveVerifier: false },
+    constitution: { enabled: false, path: "constitution.md", maxTokens: 100 },
+    analyze: { llmEnhanced: false, model: "balanced", fallbackToKeywords: true, maxCodebaseSummaryTokens: 100 },
+    review: { enabled: false, checks: [] as Array<"typecheck" | "lint" | "test">, commands: {} },
+    plan: { model: "balanced", outputPath: "spec.md" },
+    acceptance: { enabled: false, maxRetries: 0, generateTests: false, testPath: "acceptance.test.ts" },
+    context: {
+      testCoverage: { enabled: false, detail: "names-only" as const, maxTokens: 50, testPattern: "**/*.test.ts", scopeToStory: false },
+      autoDetect: { enabled: false, maxFiles: 1, traceImports: false },
+    },
+  };
+}