@nathapp/nax 0.21.0 → 0.22.0

package/test/unit/pipeline/stages/regression-stage.test.ts

@@ -0,0 +1,69 @@
+// RE-ARCH: keep
+import { describe, expect, test } from "bun:test";
+import { regressionStage, _regressionStageDeps } from "../../../../src/pipeline/stages/regression";
+import { makePassResult, makeFailResult } from "../../../../src/verification/orchestrator-types";
+import type { PipelineContext } from "../../../../src/pipeline/types";
+import { DEFAULT_CONFIG } from "../../../../src/config";
+
+function makeCtx(mode: "deferred" | "per-story" | "disabled" = "per-story"): PipelineContext {
+  return {
+    config: {
+      ...DEFAULT_CONFIG,
+      execution: {
+        ...DEFAULT_CONFIG.execution,
+        regressionGate: { enabled: true, mode, timeoutSeconds: 60, acceptOnTimeout: true },
+      },
+      quality: { ...DEFAULT_CONFIG.quality, commands: { test: "bun test" } },
+    } as any,
+    prd: { stories: [] } as any,
+    story: { id: "US-001", title: "t", status: "in-progress", acceptanceCriteria: [] } as any,
+    stories: [],
+    routing: { complexity: "simple", modelTier: "fast", testStrategy: "test-after", reasoning: "" },
+    workdir: "/tmp",
+    hooks: {},
+  };
+}
+
+describe("regressionStage", () => {
+  test("disabled when mode is deferred", () => {
+    expect(regressionStage.enabled(makeCtx("deferred"))).toBe(false);
+  });
+
+  test("disabled when mode is disabled", () => {
+    expect(regressionStage.enabled(makeCtx("disabled"))).toBe(false);
+  });
+
+  test("enabled when mode is per-story", () => {
+    expect(regressionStage.enabled(makeCtx("per-story"))).toBe(true);
+  });
+
+  test("disabled when verifyResult is a failure", () => {
+    const ctx = makeCtx("per-story");
+    ctx.verifyResult = makeFailResult("US-001", "scoped", "TEST_FAILURE");
+    expect(regressionStage.enabled(ctx)).toBe(false);
+  });
+
+  test("returns continue when regression passes", async () => {
+    const saved = { ..._regressionStageDeps };
+    _regressionStageDeps.verifyRegression = async () => makePassResult("US-001", "regression");
+
+    const result = await regressionStage.execute(makeCtx("per-story"));
+
+    Object.assign(_regressionStageDeps, saved);
+
+    expect(result.action).toBe("continue");
+  });
+
+  test("returns escalate when regression fails", async () => {
+    const saved = { ..._regressionStageDeps };
+    _regressionStageDeps.verifyRegression = async () =>
+      makeFailResult("US-001", "regression", "TEST_FAILURE", { failCount: 3 });
+
+    const result = await regressionStage.execute(makeCtx("per-story"));
+
+    Object.assign(_regressionStageDeps, saved);
+
+    expect(result.action).toBe("escalate");
+    if (result.action === "escalate") expect(result.reason).toContain("3 test");
+  });
+});

package/test/unit/pipeline/stages/verify.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Verify Stage — per-story regression gate skipping (US-003)
  *

package/test/unit/pipeline/subscribers/hooks.test.ts

@@ -0,0 +1,45 @@
+// RE-ARCH: keep
+import { describe, expect, test, mock } from "bun:test";
+import { wireHooks } from "../../../../src/pipeline/subscribers/hooks";
+import { PipelineEventBus } from "../../../../src/pipeline/event-bus";
+import type { LoadedHooksConfig } from "../../../../src/hooks";
+
+const EMPTY_HOOKS: LoadedHooksConfig = {};
+
+describe("wireHooks", () => {
+  test("subscribes to all lifecycle events", () => {
+    const bus = new PipelineEventBus();
+    wireHooks(bus, EMPTY_HOOKS, "/tmp", "test-feature");
+
+    // Check subscriptions are registered
+    const events = ["run:started", "story:started", "story:completed", "story:failed", "story:paused", "run:paused", "run:completed"] as const;
+    for (const ev of events) {
+      expect(bus.subscriberCount(ev)).toBe(1);
+    }
+  });
+
+  test("returns unsubscribe function that removes all subscriptions", () => {
+    const bus = new PipelineEventBus();
+    const unsub = wireHooks(bus, EMPTY_HOOKS, "/tmp", "test-feature");
+
+    unsub();
+
+    const events = ["run:started", "story:started", "story:completed"] as const;
+    for (const ev of events) {
+      expect(bus.subscriberCount(ev)).toBe(0);
+    }
+  });
+
+  test("errors in hooks don't propagate to callers", async () => {
+    const bus = new PipelineEventBus();
+    const badHooks: LoadedHooksConfig = {
+      "on-story-complete": { command: "exit 1", timeout: 1 } as any,
+    };
+    wireHooks(bus, badHooks, "/tmp", "test-feature");
+
+    // Should not throw
+    expect(() =>
+      bus.emit({ type: "story:completed", storyId: "US-001", story: { id: "US-001" } as any, passed: true, durationMs: 100 }),
+    ).not.toThrow();
+  });
+});

package/test/unit/pipeline/subscribers/interaction.test.ts

@@ -0,0 +1,31 @@
+// RE-ARCH: keep
+import { describe, expect, test } from "bun:test";
+import { wireInteraction } from "../../../../src/pipeline/subscribers/interaction";
+import { PipelineEventBus } from "../../../../src/pipeline/event-bus";
+import { DEFAULT_CONFIG } from "../../../../src/config";
+
+describe("wireInteraction", () => {
+  test("no subscriptions when interactionChain is null", () => {
+    const bus = new PipelineEventBus();
+    wireInteraction(bus, null, DEFAULT_CONFIG);
+    expect(bus.subscriberCount("human-review:requested")).toBe(0);
+  });
+
+  test("no subscriptions when human-review trigger is disabled", () => {
+    const bus = new PipelineEventBus();
+    const config = {
+      ...DEFAULT_CONFIG,
+      interaction: { ...DEFAULT_CONFIG.interaction, triggers: { "human-review": { enabled: false } } },
+    } as any;
+    const chain = {} as any;
+    wireInteraction(bus, chain, config);
+    expect(bus.subscriberCount("human-review:requested")).toBe(0);
+  });
+
+  test("returns unsubscribe function", () => {
+    const bus = new PipelineEventBus();
+    const unsub = wireInteraction(bus, null, DEFAULT_CONFIG);
+    expect(typeof unsub).toBe("function");
+    unsub(); // should not throw
+  });
+});

package/test/unit/pipeline/subscribers/reporters.test.ts

@@ -0,0 +1,90 @@
+// RE-ARCH: keep
+import { describe, expect, test } from "bun:test";
+import { wireReporters } from "../../../../src/pipeline/subscribers/reporters";
+import { PipelineEventBus } from "../../../../src/pipeline/event-bus";
+import type { IReporter } from "../../../../src/plugins/types";
+
+function makeReporter(): IReporter & { calls: string[] } {
+  const calls: string[] = [];
+  return {
+    name: "test-reporter",
+    calls,
+    async onRunStart() { calls.push("onRunStart"); },
+    async onStoryComplete(ev) { calls.push(`onStoryComplete:${ev.status}`); },
+    async onRunEnd() { calls.push("onRunEnd"); },
+  };
+}
+
+function makeRegistry(reporter: IReporter) {
+  return { getReporters: () => [reporter] } as any;
+}
+
+describe("wireReporters", () => {
+  test("run:started fires onRunStart", async () => {
+    const bus = new PipelineEventBus();
+    const reporter = makeReporter();
+    wireReporters(bus, makeRegistry(reporter), "run-1", Date.now());
+
+    bus.emit({ type: "run:started", feature: "test", totalStories: 5, workdir: "/tmp" });
+
+    await Bun.sleep(10); // let fire-and-forget settle
+    expect(reporter.calls).toContain("onRunStart");
+  });
+
+  test("story:completed fires onStoryComplete(completed)", async () => {
+    const bus = new PipelineEventBus();
+    const reporter = makeReporter();
+    wireReporters(bus, makeRegistry(reporter), "run-1", Date.now());
+
+    bus.emit({ type: "story:completed", storyId: "US-001", story: { id: "US-001" } as any, passed: true, durationMs: 100 });
+
+    await Bun.sleep(10);
+    expect(reporter.calls).toContain("onStoryComplete:completed");
+  });
+
+  test("story:failed fires onStoryComplete(failed)", async () => {
+    const bus = new PipelineEventBus();
+    const reporter = makeReporter();
+    wireReporters(bus, makeRegistry(reporter), "run-1", Date.now());
+
+    bus.emit({ type: "story:failed", storyId: "US-001", story: { id: "US-001" } as any, reason: "tests failed", countsTowardEscalation: true });
+
+    await Bun.sleep(10);
+    expect(reporter.calls).toContain("onStoryComplete:failed");
+  });
+
+  test("story:paused fires onStoryComplete(paused)", async () => {
+    const bus = new PipelineEventBus();
+    const reporter = makeReporter();
+    wireReporters(bus, makeRegistry(reporter), "run-1", Date.now());
+
+    bus.emit({ type: "story:paused", storyId: "US-001", reason: "needs review", cost: 0.5 });
+
+    await Bun.sleep(10);
+    expect(reporter.calls).toContain("onStoryComplete:paused");
+  });
+
+  test("run:completed fires onRunEnd", async () => {
+    const bus = new PipelineEventBus();
+    const reporter = makeReporter();
+    wireReporters(bus, makeRegistry(reporter), "run-1", Date.now());
+
+    bus.emit({ type: "run:completed", totalStories: 5, passedStories: 4, failedStories: 1, durationMs: 60000 });
+
+    await Bun.sleep(10);
+    expect(reporter.calls).toContain("onRunEnd");
+  });
+
+  test("reporter errors don't propagate", async () => {
+    const bus = new PipelineEventBus();
+    const badReporter: IReporter = {
+      name: "bad",
+      async onStoryComplete() { throw new Error("reporter crash"); },
+    };
+    wireReporters(bus, makeRegistry(badReporter), "run-1", Date.now());
+
+    expect(() =>
+      bus.emit({ type: "story:completed", storyId: "US-001", story: { id: "US-001" } as any, passed: true, durationMs: 100 }),
+    ).not.toThrow();
+  });
+});

package/test/unit/pipeline/verify-smart-runner.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Verify Stage --- Smart Runner Integration Tests (STR-005)
  *

package/test/unit/prd-auto-default.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * PRD Auto-Default Tests (US-006 / BUG-004)
  *

package/test/unit/prd-failure-category.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * PRD failureCategory Tests
  *

package/test/unit/prd-get-next-story.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * getNextStory() Unit Tests (BUG-022)
  *

package/test/unit/precheck-checks.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Tests for src/precheck/checks.ts
  *

package/test/unit/precheck-story-size-gate.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Tests for src/precheck/story-size-gate.ts
  *

package/test/unit/precheck-types.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Tests for src/precheck/types.ts
  *

package/test/unit/prompts.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Prompt builder tests
  */

package/test/unit/rectification.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Unit tests for rectification core logic (v0.11)
  */
@@ -8,7 +9,7 @@ import {
   type RectificationState,
   createRectificationPrompt,
   shouldRetryRectification,
-} from "../../src/execution/rectification";
+} from "../../src/verification/rectification";
 import type { TestFailure } from "../../src/execution/test-output-parser";
 import type { UserStory } from "../../src/prd";
 

package/test/unit/registry.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * Tests for src/plugins/registry.ts
  *

package/test/unit/routing/routing-stability.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * v0.18.4 Routing Stability Tests
  *

package/test/unit/routing/strategies/llm.test.ts

@@ -1,3 +1,4 @@
+// RE-ARCH: keep
 /**
  * LLM Routing Strategy Tests
  *

package/test/unit/routing-advanced.test.ts

@@ -0,0 +1,313 @@
+// RE-ARCH: keep
+/**
+ * Routing Tests
+ *
+ * Consolidated test suite for routing system including:
+ * - Core routing logic (classifyComplexity, determineTestStrategy, routeTask)
+ * - Routing strategies (keyword, llm, manual, adaptive)
+ * - Strategy chain execution
+ * - Async support and chain delegation
+ */
+
+import { beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
+import { DEFAULT_CONFIG } from "../../src/config";
+import type { NaxConfig } from "../../src/config";
+import { escalateTier } from "../../src/execution/runner";
+import type { AggregateMetrics } from "../../src/metrics/types";
+import type { UserStory } from "../../src/prd/types";
+import { classifyComplexity, determineTestStrategy, routeTask } from "../../src/routing";
+import { buildStrategyChain } from "../../src/routing/builder";
+import { StrategyChain } from "../../src/routing/chain";
+import { keywordStrategy, llmStrategy, manualStrategy } from "../../src/routing/strategies";
+import { adaptiveStrategy } from "../../src/routing/strategies/adaptive";
+import {
+  buildBatchPrompt,
+  buildRoutingPrompt,
+  clearCache,
+  clearCacheForStory,
+  getCacheSize,
+  llmStrategy as llmStrategyFull,
+  parseRoutingResponse,
+  routeBatch,
+  stripCodeFences,
+  validateRoutingDecision,
+} from "../../src/routing/strategies/llm";
+import type { RoutingContext, RoutingDecision, RoutingStrategy } from "../../src/routing/strategy";
+
+
+function createStory(
+  id: string,
+  title: string,
+  description: string,
+  acceptanceCriteria: string[] = [],
+  tags: string[] = [],
+): UserStory {
+  return {
+    id,
+    title,
+    description,
+    acceptanceCriteria,
+    tags,
+    status: "pending",
+    dependencies: [],
+    passes: false,
+    escalations: [],
+    attempts: 0,
+  };
+}
+
+function createContext(metrics?: AggregateMetrics, config: NaxConfig = DEFAULT_CONFIG): RoutingContext {
+  return {
+    config,
+    metrics,
+  };
+}
+
+function createMockMetrics(
+  complexityData: Record<string, { predicted: number; actualTierUsed: string; mismatchRate: number }>,
+): AggregateMetrics {
+  return {
+    totalRuns: 10,
+    totalCost: 5.0,
+    totalStories: 100,
+    firstPassRate: 0.75,
+    escalationRate: 0.25,
+    avgCostPerStory: 0.05,
+    avgCostPerFeature: 0.5,
+    modelEfficiency: {
+      "claude-haiku-4-5": {
+        attempts: 60,
+        successes: 50,
+        passRate: 0.833,
+        avgCost: 0.005,
+        totalCost: 0.25,
+      },
+      "claude-sonnet-4.5": {
+        attempts: 30,
+        successes: 28,
+        passRate: 0.933,
+        avgCost: 0.02,
+        totalCost: 0.56,
+      },
+      "claude-opus-4-6": {
+        attempts: 10,
+        successes: 10,
+        passRate: 1.0,
+        avgCost: 0.08,
+        totalCost: 0.8,
+      },
+    },
+    complexityAccuracy: complexityData,
+  };
+}
+
+describe("Adaptive Routing Strategy", () => {
+  describe("No metrics available", () => {
+    test("should fallback to configured strategy when no metrics", async () => {
+      const story = createStory("US-001", "Add user login", "Implement user authentication", [
+        "User can log in with email and password",
+      ]);
+
+      const context = createContext(undefined);
+      const decision = await adaptiveStrategy.route(story, context);
+
+      expect(decision).not.toBeNull();
+      expect(decision?.reasoning).toContain("no metrics available");
+      expect(decision?.reasoning).toContain("fallback to");
+    });
+  });
+
+  describe("Insufficient data fallback", () => {
+    test("should fallback when samples below minSamples threshold", async () => {
+      const metrics = createMockMetrics({
+        simple: {
+          predicted: 5,
+          actualTierUsed: "fast",
+          mismatchRate: 0.2,
+        },
+      });
+
+      const story = createStory("US-002", "Fix typo", "Fix typo in README", ["Typo is fixed"]);
+
+      const context = createContext(metrics);
+      const decision = await adaptiveStrategy.route(story, context);
+
+      expect(decision).not.toBeNull();
+      expect(decision?.reasoning).toContain("insufficient data");
+      expect(decision?.reasoning).toContain("5/10 samples");
+      expect(decision?.reasoning).toContain("fallback to");
+    });
+  });
+
+  describe("Sufficient data - adaptive routing", () => {
+    test("should route to fast tier when low mismatch rate", async () => {
+      const metrics = createMockMetrics({
+        simple: {
+          predicted: 50,
+          actualTierUsed: "fast",
+          mismatchRate: 0.1,
+        },
+      });
+
+      const story = createStory("US-004", "Add button", "Add a submit button to the form", [
+        "Button is visible",
+        "Button triggers submit",
+      ]);
+
+      const context = createContext(metrics);
+      const decision = await adaptiveStrategy.route(story, context);
+
+      expect(decision).not.toBeNull();
+      expect(decision?.complexity).toBe("simple");
+      expect(decision?.modelTier).toBe("fast");
+      expect(decision?.reasoning).toContain("adaptive");
+      expect(decision?.reasoning).toContain("simple → fast");
+      expect(decision?.reasoning).toContain("samples: 50");
+      expect(decision?.reasoning).toContain("mismatch: 10.0%");
+    });
+
+    test("should include cost information in reasoning", async () => {
+      const metrics = createMockMetrics({
+        complex: {
+          predicted: 15,
+          actualTierUsed: "powerful",
+          mismatchRate: 0.2,
+        },
+      });
+
+      const story = createStory(
+        "US-006",
+        "Refactor authentication",
+        "Refactor the auth module to use JWT",
+        Array.from({ length: 10 }, (_, i) => `Criterion ${i + 1}`),
+        ["security", "breaking-change"],
+      );
+
+      const context = createContext(metrics);
+      const decision = await adaptiveStrategy.route(story, context);
+
+      expect(decision).not.toBeNull();
+      expect(decision?.reasoning).toContain("cost:");
+      expect(decision?.reasoning).toMatch(/\$\d+\.\d{4}/);
+    });
+  });
+
+  describe("Edge cases", () => {
+    test("should handle zero mismatch rate gracefully", async () => {
+      const metrics = createMockMetrics({
+        simple: {
+          predicted: 100,
+          actualTierUsed: "fast",
+          mismatchRate: 0.0,
+        },
+      });
+
+      const story = createStory("US-014", "Add text", "Add help text", ["Text added"]);
+      const context = createContext(metrics);
+      const decision = await adaptiveStrategy.route(story, context);
+
+      expect(decision).not.toBeNull();
+      expect(decision?.modelTier).toBe("fast");
+    });
+  });
+});
+
+// ============================================================================
+// LLM Cache Clearing Tests (BUG-028 fix)
+// ============================================================================
+
+describe("LLM Cache Clearing on Tier Escalation", () => {
+  beforeEach(() => {
+    // Clear cache before each test
+    clearCache();
+  });
+
+  test("cache hit returns cached decision", () => {
+    const story: UserStory = {
+      id: "US-cache-001",
+      title: "Test story",
+      description: "Test story for cache",
+      acceptanceCriteria: ["AC1"],
+      tags: [],
+      dependencies: [],
+      status: "pending",
+      passes: false,
+      escalations: [],
+      attempts: 0,
+    };
+
+    const originalDecision: RoutingDecision = {
+      complexity: "simple",
+      modelTier: "fast",
+      testStrategy: "test-after",
+      reasoning: "Original decision",
+    };
+
+    const configWithoutLlm = { ...DEFAULT_CONFIG, routing: { ...DEFAULT_CONFIG.routing, llm: undefined } };
+    const context: RoutingContext = { config: configWithoutLlm };
+
+    // Simulate cached decision
+    const cachedDecisions = new Map<string, RoutingDecision>();
+    cachedDecisions.set(story.id, originalDecision);
+
+    // Verify initial cache state
+    expect(getCacheSize()).toBe(0);
+
+    // Note: We're testing the behavior through the exported functions
+    // In a real scenario, the LLM strategy would populate the cache
+    // For this test, we verify the cache clearing mechanism works
+  });
+
+  test("clearCacheForStory removes cache entry", () => {
+    const storyId = "US-cache-002";
+
+    // Clear cache first
+    clearCache();
+    expect(getCacheSize()).toBe(0);
+
+    // Clear non-existent entry should not throw
+    clearCacheForStory(storyId);
+    expect(getCacheSize()).toBe(0);
+  });
+
+  test("clearCacheForStory after tier escalation forces re-routing", () => {
+    const storyId = "US-cache-003";
+
+    // Clear all caches
+    clearCache();
+    expect(getCacheSize()).toBe(0);
+
+    // Simulate clearing for escalation
+    clearCacheForStory(storyId);
+
+    // Cache should still be empty
+    expect(getCacheSize()).toBe(0);
+  });
+
+  test("clearing one story does not affect other cached stories", () => {
+    clearCache();
+
+    const story1Id = "US-escalate-1";
+    const story2Id = "US-escalate-2";
+
+    // Verify we can clear individual stories
+    clearCacheForStory(story1Id);
+    clearCacheForStory(story2Id);
+
+    expect(getCacheSize()).toBe(0);
+  });
+
+  test("clearCacheForStory is idempotent", () => {
+    const storyId = "US-idempotent";
+
+    clearCache();
+    expect(getCacheSize()).toBe(0);
+
+    // Clear multiple times should be safe
+    clearCacheForStory(storyId);
+    clearCacheForStory(storyId);
+    clearCacheForStory(storyId);
+
+    expect(getCacheSize()).toBe(0);
+  });
+});