@nathapp/nax 0.25.0 → 0.26.0

package/test/unit/metrics/aggregator.test.ts

@@ -0,0 +1,164 @@
+/**
+ * Metrics Aggregator — RRP-002: complexityAccuracy uses initialComplexity
+ *
+ * AC-6: calculateAggregateMetrics complexityAccuracy compares
+ *       initialComplexity (predicted) vs finalTier (actual), not
+ *       complexity (which may reflect post-escalation state).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { calculateAggregateMetrics } from "../../../src/metrics/aggregator";
+import type { RunMetrics, StoryMetrics } from "../../../src/metrics/types";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeStoryMetrics(overrides: Partial<StoryMetrics> & { storyId: string }): StoryMetrics {
+  return {
+    storyId: overrides.storyId,
+    complexity: "medium",
+    modelTier: "balanced",
+    modelUsed: "claude-sonnet-4-5",
+    attempts: 1,
+    finalTier: "balanced",
+    success: true,
+    cost: 0.01,
+    durationMs: 5000,
+    firstPassSuccess: true,
+    startedAt: "2026-01-01T00:00:00Z",
+    completedAt: "2026-01-01T00:00:05Z",
+    ...overrides,
+  };
+}
+
+function makeRun(stories: StoryMetrics[]): RunMetrics {
+  return {
+    runId: "run-001",
+    feature: "test-feature",
+    startedAt: "2026-01-01T00:00:00Z",
+    completedAt: "2026-01-01T00:01:00Z",
+    totalCost: stories.reduce((sum, s) => sum + s.cost, 0),
+    totalStories: stories.length,
+    storiesCompleted: stories.filter((s) => s.success).length,
+    storiesFailed: stories.filter((s) => !s.success).length,
+    totalDurationMs: 60000,
+    stories,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// AC-6: complexityAccuracy uses initialComplexity as predicted complexity
+// ---------------------------------------------------------------------------
+
+describe("calculateAggregateMetrics - complexityAccuracy uses initialComplexity", () => {
+  test("complexityAccuracy keyed by initialComplexity when present", () => {
+    // Story originally predicted as 'simple' but escalated (finalTier = 'powerful')
+    const story = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",          // post-escalation complexity
+      initialComplexity: "simple",   // original prediction
+      modelTier: "fast",
+      finalTier: "powerful",
+      attempts: 2,
+      firstPassSuccess: false,
+    });
+
+    const runs = [makeRun([story])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    // complexityAccuracy should be keyed by initialComplexity ("simple"), not complexity ("medium")
+    expect(aggregate.complexityAccuracy["simple"]).toBeDefined();
+    expect(aggregate.complexityAccuracy["medium"]).toBeUndefined();
+  });
+
+  test("mismatch detected when initialComplexity tier != finalTier", () => {
+    const escalatedStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",
+      initialComplexity: "simple",
+      modelTier: "fast",
+      finalTier: "powerful",
+      attempts: 2,
+      firstPassSuccess: false,
+    });
+
+    const runs = [makeRun([escalatedStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    // simple -> powerful: mismatch expected
+    expect(aggregate.complexityAccuracy["simple"].mismatchRate).toBeGreaterThan(0);
+  });
+
+  test("no mismatch when initialComplexity tier matches finalTier", () => {
+    const successStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",
+      initialComplexity: "medium",
+      modelTier: "balanced",
+      finalTier: "balanced",
+      attempts: 1,
+      firstPassSuccess: true,
+    });
+
+    const runs = [makeRun([successStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    expect(aggregate.complexityAccuracy["medium"].mismatchRate).toBe(0);
+  });
+
+  test("falls back to complexity when initialComplexity is absent (backward compat)", () => {
+    // Legacy story metrics without initialComplexity
+    const legacyStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "complex",
+      // no initialComplexity
+      modelTier: "powerful",
+      finalTier: "powerful",
+    });
+
+    const runs = [makeRun([legacyStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    // Falls back to complexity as key
+    expect(aggregate.complexityAccuracy["complex"]).toBeDefined();
+  });
+
+  test("mixes initialComplexity-keyed and legacy entries correctly", () => {
+    const modernStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",
+      initialComplexity: "simple",
+      modelTier: "balanced",
+      finalTier: "balanced",
+    });
+    const legacyStory = makeStoryMetrics({
+      storyId: "US-002",
+      complexity: "complex",
+      // no initialComplexity
+      modelTier: "powerful",
+      finalTier: "powerful",
+    });
+
+    const runs = [makeRun([modernStory, legacyStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    expect(aggregate.complexityAccuracy["simple"]).toBeDefined();   // from initialComplexity
+    expect(aggregate.complexityAccuracy["complex"]).toBeDefined();  // from complexity fallback
+    expect(aggregate.complexityAccuracy["medium"]).toBeUndefined(); // NOT used (initialComplexity takes over)
+  });
+
+  test("complexityAccuracy.predicted count matches number of stories with that initialComplexity", () => {
+    const stories = [
+      makeStoryMetrics({ storyId: "US-001", complexity: "medium", initialComplexity: "simple", finalTier: "balanced" }),
+      makeStoryMetrics({ storyId: "US-002", complexity: "medium", initialComplexity: "simple", finalTier: "balanced" }),
+      makeStoryMetrics({ storyId: "US-003", complexity: "complex", initialComplexity: "complex", finalTier: "powerful" }),
+    ];
+
+    const runs = [makeRun(stories)];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    expect(aggregate.complexityAccuracy["simple"].predicted).toBe(2);
+    expect(aggregate.complexityAccuracy["complex"].predicted).toBe(1);
+  });
+});

package/test/unit/metrics/tracker.test.ts

@@ -0,0 +1,186 @@
+/**
+ * Metrics Tracker — RRP-002: initialComplexity in StoryMetrics
+ *
+ * AC-4: StoryMetrics gains initialComplexity?: string field
+ * AC-5: collectStoryMetrics() reads story.routing.initialComplexity,
+ *       falls back to routing.complexity for backward compat
+ */
+
+import { describe, expect, test } from "bun:test";
+import { DEFAULT_CONFIG } from "../../../src/config/defaults";
+import type { NaxConfig } from "../../../src/config";
+import type { PipelineContext } from "../../../src/pipeline/types";
+import type { PRD, UserStory } from "../../../src/prd";
+import type { StoryRouting } from "../../../src/prd/types";
+import { collectStoryMetrics } from "../../../src/metrics/tracker";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeStory(overrides?: Partial<UserStory>): UserStory {
+  return {
+    id: "US-001",
+    title: "Test Story",
+    description: "Test description",
+    acceptanceCriteria: [],
+    tags: [],
+    dependencies: [],
+    status: "passed",
+    passes: true,
+    escalations: [],
+    attempts: 1,
+    ...overrides,
+  };
+}
+
+function makePRD(story: UserStory): PRD {
+  return {
+    project: "test-project",
+    feature: "test-feature",
+    branchName: "feat/test",
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    userStories: [story],
+  };
+}
+
+function makeConfig(): NaxConfig {
+  return { ...DEFAULT_CONFIG };
+}
+
+function makeCtx(story: UserStory, routingOverrides?: Partial<PipelineContext["routing"]>): PipelineContext {
+  return {
+    config: makeConfig(),
+    prd: makePRD(story),
+    story,
+    stories: [story],
+    routing: {
+      complexity: "medium",
+      modelTier: "balanced",
+      testStrategy: "test-after",
+      reasoning: "test",
+      ...routingOverrides,
+    },
+    workdir: "/tmp/nax-tracker-test",
+    hooks: { hooks: {} },
+    agentResult: {
+      success: true,
+      output: "",
+      estimatedCost: 0.01,
+      durationMs: 5000,
+    },
+  } as unknown as PipelineContext;
+}
+
+// ---------------------------------------------------------------------------
+// AC-5: collectStoryMetrics reads initialComplexity from story.routing
+// ---------------------------------------------------------------------------
+
+describe("collectStoryMetrics - initialComplexity field", () => {
+  test("includes initialComplexity from story.routing.initialComplexity", () => {
+    const routing: StoryRouting = {
+      complexity: "medium",
+      initialComplexity: "simple", // original prediction before potential escalation
+      testStrategy: "test-after",
+      reasoning: "test",
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "medium" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("simple");
+  });
+
+  test("initialComplexity differs from complexity when story was escalated", () => {
+    const routing: StoryRouting = {
+      complexity: "medium",         // complexity as classified
+      initialComplexity: "simple",  // original first-classify prediction
+      modelTier: "powerful",        // escalated tier
+      testStrategy: "three-session-tdd",
+      reasoning: "escalated",
+    };
+    const story = makeStory({
+      routing,
+      escalations: [
+        {
+          fromTier: "balanced",
+          toTier: "powerful",
+          reason: "test failure",
+          timestamp: new Date().toISOString(),
+        },
+      ],
+      attempts: 2,
+    });
+    const ctx = makeCtx(story, { complexity: "medium", modelTier: "balanced" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("simple");
+    // complexity field unchanged (backward compat)
+    expect(metrics.complexity).toBe("medium");
+  });
+
+  test("falls back to routing.complexity when story.routing.initialComplexity is absent", () => {
+    // Backward compat: story.routing exists but has no initialComplexity
+    const routing: StoryRouting = {
+      complexity: "complex",
+      testStrategy: "three-session-tdd",
+      reasoning: "legacy routing",
+      // no initialComplexity
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "complex" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("complex");
+  });
+
+  test("falls back to routing.complexity when story.routing is undefined", () => {
+    const story = makeStory({ routing: undefined });
+    const ctx = makeCtx(story, { complexity: "simple" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("simple");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// AC-4: StoryMetrics type has initialComplexity?: string
+// ---------------------------------------------------------------------------
+
+describe("StoryMetrics type - initialComplexity field", () => {
+  test("StoryMetrics includes initialComplexity field", () => {
+    const routing: StoryRouting = {
+      complexity: "medium",
+      initialComplexity: "simple",
+      testStrategy: "test-after",
+      reasoning: "test",
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "medium" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    // TypeScript will error at compile time if initialComplexity is not on StoryMetrics
+    expect("initialComplexity" in metrics).toBe(true);
+  });
+
+  test("initialComplexity is a string when present", () => {
+    const routing: StoryRouting = {
+      complexity: "expert",
+      initialComplexity: "expert",
+      testStrategy: "three-session-tdd",
+      reasoning: "test",
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "expert" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(typeof metrics.initialComplexity).toBe("string");
+  });
+});

package/test/unit/pipeline/stages/routing-idempotence.test.ts

@@ -0,0 +1,139 @@
+/**
+ * Routing Stage — RRP-001: Idempotence and Dependencies
+ *
+ * AC-4: Tests for idempotent persistence and dependency exposure.
+ */
+
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+import { DEFAULT_CONFIG } from "../../../../src/config/defaults";
+import type { NaxConfig } from "../../../../src/config";
+import type { PRD, UserStory } from "../../../../src/prd";
+import type { PipelineContext } from "../../../../src/pipeline/types";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeStory(overrides?: Partial<UserStory>): UserStory {
+  return {
+    id: "US-001",
+    title: "Test Story",
+    description: "Test description",
+    acceptanceCriteria: [],
+    tags: [],
+    dependencies: [],
+    status: "in-progress",
+    passes: false,
+    escalations: [],
+    attempts: 0,
+    ...overrides,
+  };
+}
+
+function makePRD(story: UserStory): PRD {
+  return {
+    project: "test-project",
+    feature: "test-feature",
+    branchName: "feat/test",
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    userStories: [story],
+  };
+}
+
+function makeConfig(): NaxConfig {
+  return {
+    ...DEFAULT_CONFIG,
+    tdd: {
+      ...DEFAULT_CONFIG.tdd,
+      greenfieldDetection: false,
+    },
+  };
+}
+
+function makeCtx(story: UserStory, overrides?: Partial<PipelineContext>): PipelineContext & { prdPath: string } {
+  const prd = makePRD(story);
+  return {
+    config: makeConfig(),
+    prd,
+    story,
+    stories: [story],
+    routing: {
+      complexity: "simple",
+      modelTier: "fast",
+      testStrategy: "test-after",
+      reasoning: "test",
+    },
+    workdir: "/tmp/nax-routing-test",
+    hooks: { hooks: {} },
+    prdPath: "/tmp/nax-routing-test/nax/prd.json",
+    ...overrides,
+  } as PipelineContext & { prdPath: string };
+}
+
+const FRESH_ROUTING_RESULT = {
+  complexity: "medium" as const,
+  modelTier: "balanced" as const,
+  testStrategy: "three-session-tdd" as const,
+  reasoning: "classified by routeStory",
+};
+
+// ---------------------------------------------------------------------------
+// AC-4: savePRD called once per story, not on every iteration
+// ---------------------------------------------------------------------------
+
+describe("routingStage - savePRD called exactly once per story (not per iteration)", () => {
+  let origRoutingDeps: typeof import("../../../../src/pipeline/stages/routing")["_routingDeps"];
+
+  afterEach(() => {
+    mock.restore();
+    if (origRoutingDeps) {
+      const { _routingDeps } = require("../../../../src/pipeline/stages/routing");
+      Object.assign(_routingDeps, origRoutingDeps);
+    }
+  });
+
+  test("calling routingStage twice with routing already set only triggers savePRD once (first call)", async () => {
+    const { routingStage, _routingDeps } = await import(
+      "../../../../src/pipeline/stages/routing"
+    );
+
+    origRoutingDeps = { ..._routingDeps };
+
+    let savePRDCallCount = 0;
+
+    _routingDeps.routeStory = mock(() =>
+      Promise.resolve({ ...FRESH_ROUTING_RESULT }),
+    );
+    _routingDeps.isGreenfieldStory = mock(() => Promise.resolve(false));
+    _routingDeps.savePRD = mock((_prd: PRD, _path: string) => {
+      savePRDCallCount++;
+      return Promise.resolve();
+    });
+
+    // First iteration: story.routing is undefined → should persist
+    const story = makeStory({ routing: undefined });
+    const ctx = makeCtx(story);
+
+    await routingStage.execute(ctx as Parameters<typeof routingStage.execute>[0]);
+
+    // After first execution, story.routing is populated (simulating resume after crash)
+    // Second iteration: story.routing is now set → should NOT persist again
+    await routingStage.execute(ctx as Parameters<typeof routingStage.execute>[0]);
+
+    expect(savePRDCallCount).toBe(1);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Sanity: _routingDeps exposes savePRD (fail if not added to deps object)
+// ---------------------------------------------------------------------------
+
+describe("routingStage - _routingDeps exposes savePRD", () => {
+  test("_routingDeps has a savePRD function", async () => {
+    const { _routingDeps } = await import(
+      "../../../../src/pipeline/stages/routing"
+    );
+    expect(typeof _routingDeps.savePRD).toBe("function");
+  });
+});