@nathapp/nax 0.28.0 → 0.30.0

package/test/unit/routing/routing-stability.test.ts

@@ -1,208 +0,0 @@
-// RE-ARCH: keep
-/**
- * v0.18.4 Routing Stability Tests
- *
- * BUG-031: Keyword classifier inconsistency across retries
- * BUG-033: LLM routing retry on timeout — config fields and defaults
- */
-
-import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
-import { keywordStrategy } from "../../../src/routing/strategies/keyword";
-import type { RoutingContext } from "../../../src/routing/strategy";
-import type { UserStory } from "../../../src/prd/types";
-import { DEFAULT_CONFIG } from "../../../src/config/defaults";
-import { initLogger, resetLogger } from "../../../src/logger";
-
-// ---------------------------------------------------------------------------
-// Fixtures
-// ---------------------------------------------------------------------------
-
-function makeStory(overrides: Partial<UserStory> = {}): UserStory {
-  return {
-    id: "TEST-001",
-    title: "Add login button",
-    description: "Simple button feature",
-    acceptanceCriteria: ["Button renders", "Click works"],
-    tags: [],
-    dependencies: [],
-    status: "pending",
-    passes: false,
-    escalations: [],
-    attempts: 0,
-    ...overrides,
-  };
-}
-
-const ctx: RoutingContext = {
-  config: { ...DEFAULT_CONFIG, routing: { ...DEFAULT_CONFIG.routing, llm: undefined } },
-};
-
-beforeEach(() => {
-  resetLogger();
-  initLogger({ level: "error", useChalk: false });
-});
-
-afterEach(() => {
-  mock.restore();
-  resetLogger();
-});
-
-// ---------------------------------------------------------------------------
-// BUG-031: Keyword classifier stable across retries
-// ---------------------------------------------------------------------------
-
-describe("BUG-031: keyword classifier stability across retries", () => {
-  test("classification is identical on first attempt and retry with same story", () => {
-    const story = makeStory();
-    const first = keywordStrategy.route(story, ctx);
-    const retry = keywordStrategy.route(story, ctx);
-
-    expect(first!.complexity).toBe(retry!.complexity);
-    expect(first!.testStrategy).toBe(retry!.testStrategy);
-    expect(first!.modelTier).toBe(retry!.modelTier);
-  });
-
-  test("description containing security/auth error text does NOT shift classification", () => {
-    const clean = makeStory({ title: "Add color picker", description: "Simple UI change" });
-    const withErrors = makeStory({
-      title: "Add color picker",
-      description: "Simple UI change\n\nPrior errors:\n- auth token expired\n- security check failed\n- refactor needed",
-    });
-
-    const cleanResult = keywordStrategy.route(clean, ctx);
-    const errorResult = keywordStrategy.route(withErrors, ctx);
-
-    // BUG-031 fix: description excluded from classification — results must be identical
-    expect(cleanResult!.complexity).toBe(errorResult!.complexity);
-    expect(cleanResult!.testStrategy).toBe(errorResult!.testStrategy);
-    expect(cleanResult!.modelTier).toBe(errorResult!.modelTier);
-  });
-
-  test("description with complex keywords does NOT upgrade complexity", () => {
-    const story = makeStory({
-      title: "Fix button label",
-      description: "architecture refactor security migration breaking change",
-      tags: [],
-      acceptanceCriteria: ["Label is correct"],
-    });
-
-    const result = keywordStrategy.route(story, ctx);
-    expect(result!.complexity).toBe("simple");
-    expect(result!.testStrategy).toBe("test-after");
-  });
-
-  test("complexity is driven by title and tags only (not description)", () => {
-    const simple = makeStory({ title: "Update label", description: "auth security jwt rbac encryption" });
-    const complex = makeStory({
-      title: "Add JWT authentication",
-      description: "Simple change",
-      tags: ["auth"],
-      acceptanceCriteria: ["Token issued", "Expiry set", "Refresh works"],
-    });
-
-    expect(keywordStrategy.route(simple, ctx)!.complexity).toBe("simple");
-    expect(keywordStrategy.route(complex, ctx)!.testStrategy).toBe("three-session-tdd");
-  });
-});
-
-// ---------------------------------------------------------------------------
-// BUG-033: LLM routing config — retries and timeout defaults
-// ---------------------------------------------------------------------------
-
-describe("BUG-033: LLM routing config defaults and retry fields", () => {
-  test("LlmRoutingConfig accepts retries and retryDelayMs fields", () => {
-    const config = {
-      ...DEFAULT_CONFIG,
-      routing: {
-        ...DEFAULT_CONFIG.routing,
-        llm: {
-          mode: "per-story" as const,
-          timeoutMs: 30000,
-          retries: 2,
-          retryDelayMs: 500,
-          fallbackToKeywords: true,
-          cacheDecisions: true,
-        },
-      },
-    };
-
-    expect(config.routing.llm!.retries).toBe(2);
-    expect(config.routing.llm!.retryDelayMs).toBe(500);
-  });
-
-  test("retries defaults to undefined — callLlm uses 1 internally", () => {
-    const config = {
-      ...DEFAULT_CONFIG,
-      routing: {
-        ...DEFAULT_CONFIG.routing,
-        llm: { mode: "per-story" as const, fallbackToKeywords: true, cacheDecisions: true },
-      },
-    };
-
-    expect(config.routing.llm!.retries).toBeUndefined();
-  });
-
-  test("effective timeout defaults to 30000ms (raised from 15000)", () => {
-    const config = {
-      ...DEFAULT_CONFIG,
-      routing: {
-        ...DEFAULT_CONFIG.routing,
-        llm: { mode: "per-story" as const, fallbackToKeywords: true, cacheDecisions: true },
-      },
-    };
-
-    // When timeoutMs unset, callLlm applies 30000 default (not old 15000)
-    const effectiveTimeout = config.routing.llm!.timeoutMs ?? 30000;
-    expect(effectiveTimeout).toBe(30000);
-  });
-
-  test("retries: 0 disables retry (single attempt only)", () => {
-    const config = {
-      ...DEFAULT_CONFIG,
-      routing: {
-        ...DEFAULT_CONFIG.routing,
-        llm: { mode: "per-story" as const, retries: 0, fallbackToKeywords: true, cacheDecisions: true },
-      },
-    };
-
-    expect(config.routing.llm!.retries).toBe(0);
-  });
-
-  test("NaxConfigSchema validates retries and retryDelayMs", async () => {
-    const { NaxConfigSchema } = await import("../../../src/config/schemas");
-
-    const result = NaxConfigSchema.safeParse({
-      ...DEFAULT_CONFIG,
-      routing: {
-        strategy: "llm",
-        llm: {
-          mode: "per-story",
-          retries: 2,
-          retryDelayMs: 1000,
-          fallbackToKeywords: true,
-          cacheDecisions: true,
-        },
-      },
-    });
-
-    expect(result.success).toBe(true);
-    if (result.success) {
-      expect(result.data.routing.llm?.retries).toBe(2);
-      expect(result.data.routing.llm?.retryDelayMs).toBe(1000);
-    }
-  });
-
-  test("NaxConfigSchema rejects negative retries", async () => {
-    const { NaxConfigSchema } = await import("../../../src/config/schemas");
-
-    const result = NaxConfigSchema.safeParse({
-      ...DEFAULT_CONFIG,
-      routing: {
-        strategy: "llm",
-        llm: { mode: "per-story", retries: -1, fallbackToKeywords: true, cacheDecisions: true },
-      },
-    });
-
-    expect(result.success).toBe(false);
-  });
-});

package/test/unit/routing/strategies/llm.test.ts

@@ -1,306 +0,0 @@
-// RE-ARCH: keep
-/**
- * LLM Routing Strategy Tests
- *
- * BUG-039: Stream drain fix — stdout/stderr cancelled before proc.kill() on timeout
- */
-
-import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
-import type { NaxConfig } from "../../../../src/config";
-import { DEFAULT_CONFIG } from "../../../../src/config/defaults";
-import { initLogger, resetLogger } from "../../../../src/logger";
-import { type PipedProc, _deps } from "../../../../src/routing/strategies/llm";
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-function makeConfig(overrides: Partial<NaxConfig["routing"]["llm"]> = {}): NaxConfig {
-  return {
-    ...DEFAULT_CONFIG,
-    routing: {
-      ...DEFAULT_CONFIG.routing,
-      strategy: "llm",
-      llm: {
-        mode: "per-story",
-        timeoutMs: 50,
-        retries: 0,
-        retryDelayMs: 0,
-        fallbackToKeywords: false,
-        cacheDecisions: false,
-        model: "fast",
-        ...overrides,
-      },
-    },
-  } as NaxConfig;
-}
-
-/** Creates a fake Bun.spawn proc that never resolves (simulates a hanging LLM call). */
-function makeHangingProc() {
-  const stdoutCancelled = { value: false };
-  const stderrCancelled = { value: false };
-  const killCalled = { value: false };
-  const killCalledAfterCancel = { value: false };
-
-  // ReadableStream that never produces data and never closes
-  const neverStream = () => {
-    const cancelFn: () => void = () => {};
-    const stream = new ReadableStream({
-      start() {},
-      cancel() {
-        cancelFn();
-      },
-    });
-    return stream;
-  };
-
-  const stdout = neverStream();
-  const stderr = neverStream();
-
-  const originalStdoutCancel = stdout.cancel.bind(stdout);
-  const originalStderrCancel = stderr.cancel.bind(stderr);
-
-  // Wrap cancel to track calls
-  const trackedStdout = new Proxy(stdout, {
-    get(target, prop) {
-      if (prop === "cancel") {
-        return () => {
-          stdoutCancelled.value = true;
-          return originalStdoutCancel();
-        };
-      }
-      return (target as unknown as Record<string | symbol, unknown>)[prop as string | symbol];
-    },
-  });
-
-  const trackedStderr = new Proxy(stderr, {
-    get(target, prop) {
-      if (prop === "cancel") {
-        return () => {
-          stderrCancelled.value = true;
-          return originalStderrCancel();
-        };
-      }
-      return (target as unknown as Record<string | symbol, unknown>)[prop as string | symbol];
-    },
-  });
-
-  const proc = {
-    stdout: trackedStdout,
-    stderr: trackedStderr,
-    exited: new Promise<number>(() => {}), // never resolves
-    kill() {
-      killCalledAfterCancel.value = stdoutCancelled.value && stderrCancelled.value;
-      killCalled.value = true;
-    },
-  };
-
-  return { proc, stdoutCancelled, stderrCancelled, killCalled, killCalledAfterCancel };
-}
-
-// ---------------------------------------------------------------------------
-// Tests
-// ---------------------------------------------------------------------------
-
-beforeEach(() => {
-  resetLogger();
-  initLogger({ level: "error", useChalk: false });
-});
-
-afterEach(() => {
-  mock.restore();
-  resetLogger();
-});
-
-describe("BUG-039/BUG-040: stream cleanup on timeout", () => {
-  test("kills process on timeout without calling cancel() on locked streams", async () => {
-    const { proc, stdoutCancelled, stderrCancelled, killCalled } = makeHangingProc();
-
-    const originalSpawn = _deps.spawn;
-    _deps.spawn = mock(() => proc as PipedProc);
-
-    const config = makeConfig({ timeoutMs: 30 });
-
-    const { llmStrategy } = await import("../../../../src/routing/strategies/llm");
-
-    const story = {
-      id: "TEST-001",
-      title: "Test story",
-      description: "Test",
-      acceptanceCriteria: ["AC1"],
-      tags: [],
-      dependencies: [],
-      status: "pending" as const,
-      passes: false,
-      escalations: [],
-      attempts: 0,
-    };
-
-    const startTime = Date.now();
-
-    await expect(llmStrategy.route(story, { config })).rejects.toThrow(/timeout/i);
-
-    const elapsed = Date.now() - startTime;
-
-    // Should resolve promptly — within 500ms of the 30ms timeout
-    expect(elapsed).toBeLessThan(500);
-
-    // BUG-040: cancel() must NOT be called on locked streams — it returns a rejected
-    // Promise (per Web Streams spec) which becomes an unhandled rejection crash.
-    expect(stdoutCancelled.value).toBe(false);
-    expect(stderrCancelled.value).toBe(false);
-    expect(killCalled.value).toBe(true);
-
-    _deps.spawn = originalSpawn;
-  });
-
-  test("no unhandled rejection when Response.text() locks streams and proc is killed", async () => {
-    // Simulate the exact BUG-040 scenario:
-    // 1. Spawn proc with piped streams
-    // 2. Response(proc.stdout).text() locks the streams
-    // 3. Timeout fires, proc.kill() called
-    // 4. No unhandled rejection should occur
-
-    const unhandledRejections: Error[] = [];
-    const handler = (event: PromiseRejectionEvent) => {
-      unhandledRejections.push(event.reason as Error);
-      event.preventDefault();
-    };
-
-    // biome-ignore lint/suspicious/noGlobalAssign: test-only override
-    globalThis.addEventListener("unhandledrejection", handler);
-
-    // Create a proc where streams are locked by Response readers
-    const stdout = new ReadableStream({ start() {} });
-    const stderr = new ReadableStream({ start() {} });
-    const proc = {
-      stdout,
-      stderr,
-      exited: new Promise<number>(() => {}),
-      kill: mock(() => {}),
-    };
-
-    const originalSpawn = _deps.spawn;
-    _deps.spawn = mock(() => proc as PipedProc);
-
-    const config = makeConfig({ timeoutMs: 20, retries: 0 });
-
-    const { llmStrategy } = await import("../../../../src/routing/strategies/llm");
-    const story = {
-      id: "BUG040",
-      title: "Bug test",
-      description: "Test",
-      acceptanceCriteria: ["AC1"],
-      tags: [],
-      dependencies: [],
-      status: "pending" as const,
-      passes: false,
-      escalations: [],
-      attempts: 0,
-    };
-
-    await expect(llmStrategy.route(story, { config })).rejects.toThrow(/timeout/i);
-
-    // Give microtasks time to settle
-    await Bun.sleep(50);
-
-    globalThis.removeEventListener("unhandledrejection", handler);
-    _deps.spawn = originalSpawn;
-
-    // No unhandled rejections should have occurred
-    expect(unhandledRejections).toHaveLength(0);
-  });
-
-  test("clearTimeout is called on success path (no resource leak)", async () => {
-    const originalSpawn = _deps.spawn;
-
-    // proc that resolves immediately with valid output
-    const successProc = {
-      stdout: new ReadableStream({
-        start(ctrl) {
-          ctrl.enqueue(
-            new TextEncoder().encode(
-              JSON.stringify({
-                complexity: "simple",
-                modelTier: "fast",
-                testStrategy: "test-after",
-                reasoning: "Simple test story",
-              }),
-            ),
-          );
-          ctrl.close();
-        },
-      }),
-      stderr: new ReadableStream({
-        start(ctrl) {
-          ctrl.close();
-        },
-      }),
-      exited: Promise.resolve(0),
-      kill: mock(() => {}),
-    };
-
-    _deps.spawn = mock(() => successProc as PipedProc);
-
-    const config = makeConfig({ timeoutMs: 5000 });
-
-    const { llmStrategy, clearCache } = await import("../../../../src/routing/strategies/llm");
-    clearCache();
-
-    const story = {
-      id: "TEST-002",
-      title: "Add login button",
-      description: "Simple button feature",
-      acceptanceCriteria: ["Button renders"],
-      tags: [],
-      dependencies: [],
-      status: "pending" as const,
-      passes: false,
-      escalations: [],
-      attempts: 0,
-    };
-
-    const result = await llmStrategy.route(story, { config });
-
-    expect(result).not.toBeNull();
-    expect(result?.complexity).toBe("simple");
-    // kill() must NOT be called on the success path
-    expect(successProc.kill).not.toHaveBeenCalled();
-
-    _deps.spawn = originalSpawn;
-    clearCache();
-  });
-
-  test("timeout promise rejects and does not hang beyond timeout window", async () => {
-    const originalSpawn = _deps.spawn;
-    const { proc } = makeHangingProc();
-
-    _deps.spawn = mock(() => proc as PipedProc);
-
-    const config = makeConfig({ timeoutMs: 50, retries: 0 });
-
-    const { llmStrategy } = await import("../../../../src/routing/strategies/llm");
-
-    const story = {
-      id: "TEST-003",
-      title: "Hanging story",
-      description: "This will hang",
-      acceptanceCriteria: ["AC"],
-      tags: [],
-      dependencies: [],
-      status: "pending" as const,
-      passes: false,
-      escalations: [],
-      attempts: 0,
-    };
-
-    const before = Date.now();
-    await expect(llmStrategy.route(story, { config })).rejects.toThrow();
-    const after = Date.now();
-
-    // Should complete well under 2s even though proc never exits
-    expect(after - before).toBeLessThan(2000);
-
-    _deps.spawn = originalSpawn;
-  });
-});