@nathapp/nax 0.24.0 → 0.26.0

package/test/unit/interaction-plugins.test.ts

@@ -5,7 +5,8 @@
  * Tests for Telegram, Webhook, and Auto plugins.
  */
 
-import { describe, expect, test } from "bun:test";
+import { createHmac } from "node:crypto";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 import type { InteractionRequest } from "../../src/interaction";
 import { AutoInteractionPlugin } from "../../src/interaction/plugins/auto";
 import { TelegramInteractionPlugin } from "../../src/interaction/plugins/telegram";
@@ -163,3 +164,309 @@ describe("AutoInteractionPlugin", () => {
     expect(plugin.name).toBe("auto");
   });
 });
+
+// ---------------------------------------------------------------------------
+// Telegram send() and poll() flow tests (TC-006)
+// ---------------------------------------------------------------------------
+
+describe("TelegramInteractionPlugin - send() and poll()", () => {
+  const originalFetch = globalThis.fetch;
+
+  afterEach(() => {
+    mock.restore();
+    globalThis.fetch = originalFetch;
+  });
+
+  function makeConfirmRequest(id: string): InteractionRequest {
+    return {
+      id,
+      type: "confirm",
+      featureName: "my-feature",
+      stage: "review",
+      summary: "Proceed with merge?",
+      fallback: "abort",
+      createdAt: Date.now(),
+    };
+  }
+
+  test("send() POSTs to correct Telegram API URL with message text and inline keyboard", async () => {
+    const calls: Array<{ url: string; body: Record<string, unknown> }> = [];
+
+    globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => {
+      const urlStr = url.toString();
+      const body = JSON.parse((init?.body as string) ?? "{}");
+      calls.push({ url: urlStr, body });
+      return new Response(
+        JSON.stringify({ ok: true, result: { message_id: 42, chat: { id: 12345 } } }),
+        { status: 200, headers: { "Content-Type": "application/json" } },
+      );
+    }) as typeof fetch;
+
+    const plugin = new TelegramInteractionPlugin();
+    await plugin.init({ botToken: "bot-abc123", chatId: "99999" });
+
+    await plugin.send(makeConfirmRequest("tg-send-1"));
+
+    expect(calls).toHaveLength(1);
+    const { url, body } = calls[0];
+
+    // Correct API endpoint
+    expect(url).toContain("api.telegram.org/botbot-abc123/sendMessage");
+
+    // Correct chat_id
+    expect(body.chat_id).toBe("99999");
+
+    // Message text present
+    expect(typeof body.text).toBe("string");
+    expect((body.text as string).length).toBeGreaterThan(0);
+
+    // Inline keyboard has approve and reject buttons
+    const keyboard = (body.reply_markup as { inline_keyboard: Array<Array<{ text: string; callback_data: string }>> })
+      .inline_keyboard;
+    expect(Array.isArray(keyboard)).toBe(true);
+    const allButtons = keyboard.flat();
+    const approveBtn = allButtons.find((b) => b.callback_data === "tg-send-1:approve");
+    const rejectBtn = allButtons.find((b) => b.callback_data === "tg-send-1:reject");
+    expect(approveBtn).toBeDefined();
+    expect(rejectBtn).toBeDefined();
+  });
+
+  test("receive() parses callback_query correctly", async () => {
+    globalThis.fetch = mock(async (url: string | URL | Request, init?: RequestInit) => {
+      const urlStr = url.toString();
+
+      if (urlStr.includes("sendMessage")) {
+        return new Response(
+          JSON.stringify({ ok: true, result: { message_id: 10, chat: { id: 99999 } } }),
+          { status: 200, headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (urlStr.includes("getUpdates")) {
+        return new Response(
+          JSON.stringify({
+            ok: true,
+            result: [
+              {
+                update_id: 1,
+                callback_query: {
+                  id: "cq-001",
+                  data: "tg-poll-1:approve",
+                  message: { message_id: 10, chat: { id: 99999 } },
+                },
+              },
+            ],
+          }),
+          { status: 200, headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (urlStr.includes("answerCallbackQuery")) {
+        return new Response(JSON.stringify({ ok: true }), { status: 200 });
+      }
+
+      return new Response("not found", { status: 404 });
+    }) as typeof fetch;
+
+    const plugin = new TelegramInteractionPlugin();
+    await plugin.init({ botToken: "bot-abc123", chatId: "99999" });
+
+    // send() first so message_id is stored (needed for text-message flow, not callback_query)
+    await plugin.send(makeConfirmRequest("tg-poll-1"));
+
+    const response = await plugin.receive("tg-poll-1", 5000);
+
+    expect(response.action).toBe("approve");
+    expect(response.respondedBy).toBe("telegram");
+    expect(response.requestId).toBe("tg-poll-1");
+  });
+
+  test("receive() handles choose callback_query with value", async () => {
+    globalThis.fetch = mock(async (url: string | URL | Request) => {
+      const urlStr = url.toString();
+
+      if (urlStr.includes("sendMessage")) {
+        return new Response(
+          JSON.stringify({ ok: true, result: { message_id: 11, chat: { id: 99999 } } }),
+          { status: 200, headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (urlStr.includes("getUpdates")) {
+        return new Response(
+          JSON.stringify({
+            ok: true,
+            result: [
+              {
+                update_id: 2,
+                callback_query: {
+                  id: "cq-002",
+                  data: "tg-choose-1:choose:option-b",
+                  message: { message_id: 11, chat: { id: 99999 } },
+                },
+              },
+            ],
+          }),
+          { status: 200, headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (urlStr.includes("answerCallbackQuery")) {
+        return new Response(JSON.stringify({ ok: true }), { status: 200 });
+      }
+
+      return new Response("not found", { status: 404 });
+    }) as typeof fetch;
+
+    const plugin = new TelegramInteractionPlugin();
+    await plugin.init({ botToken: "bot-abc123", chatId: "99999" });
+
+    const chooseRequest: InteractionRequest = {
+      id: "tg-choose-1",
+      type: "choose",
+      featureName: "my-feature",
+      stage: "review",
+      summary: "Which option?",
+      fallback: "continue",
+      createdAt: Date.now(),
+      options: [
+        { key: "a", label: "Option A" },
+        { key: "b", label: "Option B" },
+      ],
+    };
+
+    await plugin.send(chooseRequest);
+    const response = await plugin.receive("tg-choose-1", 5000);
+
+    expect(response.action).toBe("choose");
+    expect(response.value).toBe("option-b");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Webhook send() and HMAC validation tests (TC-006)
+// ---------------------------------------------------------------------------
+
+describe("WebhookInteractionPlugin - send() and HMAC validation", () => {
+  afterEach(async () => {
+    mock.restore();
+  });
+
+  function makeWebhookRequest(id: string): InteractionRequest {
+    return {
+      id,
+      type: "confirm",
+      featureName: "wh-feature",
+      stage: "merge",
+      summary: "Approve merge?",
+      fallback: "abort",
+      createdAt: Date.now(),
+    };
+  }
+
+  test("send() POSTs payload with correct Content-Type", async () => {
+    // Start a local server to capture the outgoing request
+    const captured: { contentType: string | null; body: unknown } = { contentType: null, body: null };
+
+    const testServer = Bun.serve({
+      port: 19977,
+      fetch: async (req) => {
+        captured.contentType = req.headers.get("content-type");
+        captured.body = await req.json();
+        return new Response("OK", { status: 200 });
+      },
+    });
+
+    const plugin = new WebhookInteractionPlugin();
+    try {
+      await plugin.init({ url: "http://localhost:19977/hook" });
+
+      await plugin.send(makeWebhookRequest("wh-send-1"));
+
+      expect(captured.contentType).toBe("application/json");
+      expect((captured.body as { id: string }).id).toBe("wh-send-1");
+      // callbackUrl is injected by send()
+      expect(typeof (captured.body as { callbackUrl: string }).callbackUrl).toBe("string");
+    } finally {
+      testServer.stop();
+      await plugin.destroy();
+    }
+  });
+
+  test("send() includes X-Nax-Signature header when secret is configured", async () => {
+    const captured: { signature: string | null; body: string } = { signature: null, body: "" };
+
+    const testServer = Bun.serve({
+      port: 19978,
+      fetch: async (req) => {
+        captured.signature = req.headers.get("x-nax-signature");
+        captured.body = await req.text();
+        return new Response("OK", { status: 200 });
+      },
+    });
+
+    const plugin = new WebhookInteractionPlugin();
+    try {
+      await plugin.init({ url: "http://localhost:19978/hook", secret: "my-secret" });
+
+      await plugin.send(makeWebhookRequest("wh-sig-1"));
+
+      expect(captured.signature).not.toBeNull();
+      // Verify the signature matches expected HMAC
+      const expected = createHmac("sha256", "my-secret").update(captured.body).digest("hex");
+      expect(captured.signature).toBe(expected);
+    } finally {
+      testServer.stop();
+      await plugin.destroy();
+    }
+  });
+
+  test("HMAC validation: tampered payload (no signature) is rejected with 401", async () => {
+    const plugin = new WebhookInteractionPlugin();
+    // url won't be called in this test — we test the callback server
+    await plugin.init({
+      url: "http://localhost:19900/unused",
+      secret: "test-secret",
+      callbackPort: 19988,
+    });
+
+    // Start the callback server by calling receive() in the background
+    const receivePromise = plugin.receive("wh-hmac-1", 4000);
+
+    // Give the server a moment to bind
+    await Bun.sleep(60);
+
+    try {
+      // POST without signature → 401
+      const noSigResp = await fetch("http://localhost:19988/nax/interact/wh-hmac-1", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ requestId: "wh-hmac-1", action: "approve", respondedAt: Date.now() }),
+      });
+      expect(noSigResp.status).toBe(401);
+
+      // POST with wrong signature → 401
+      const badSigResp = await fetch("http://localhost:19988/nax/interact/wh-hmac-1", {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "X-Nax-Signature": "deadbeef" },
+        body: JSON.stringify({ requestId: "wh-hmac-1", action: "approve", respondedAt: Date.now() }),
+      });
+      expect(badSigResp.status).toBe(401);
+
+      // POST with correct HMAC signature → 200, receive() resolves
+      const payload = JSON.stringify({ requestId: "wh-hmac-1", action: "approve", respondedAt: Date.now() });
+      const sig = createHmac("sha256", "test-secret").update(payload).digest("hex");
+      const validResp = await fetch("http://localhost:19988/nax/interact/wh-hmac-1", {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "X-Nax-Signature": sig },
+        body: payload,
+      });
+      expect(validResp.status).toBe(200);
+
+      const response = await receivePromise;
+      expect(response.action).toBe("approve");
+    } finally {
+      await plugin.destroy();
+    }
+  });
+});

package/test/unit/metrics/aggregator.test.ts

@@ -0,0 +1,164 @@
+/**
+ * Metrics Aggregator — RRP-002: complexityAccuracy uses initialComplexity
+ *
+ * AC-6: calculateAggregateMetrics complexityAccuracy compares
+ *       initialComplexity (predicted) vs finalTier (actual), not
+ *       complexity (which may reflect post-escalation state).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { calculateAggregateMetrics } from "../../../src/metrics/aggregator";
+import type { RunMetrics, StoryMetrics } from "../../../src/metrics/types";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeStoryMetrics(overrides: Partial<StoryMetrics> & { storyId: string }): StoryMetrics {
+  return {
+    storyId: overrides.storyId,
+    complexity: "medium",
+    modelTier: "balanced",
+    modelUsed: "claude-sonnet-4-5",
+    attempts: 1,
+    finalTier: "balanced",
+    success: true,
+    cost: 0.01,
+    durationMs: 5000,
+    firstPassSuccess: true,
+    startedAt: "2026-01-01T00:00:00Z",
+    completedAt: "2026-01-01T00:00:05Z",
+    ...overrides,
+  };
+}
+
+function makeRun(stories: StoryMetrics[]): RunMetrics {
+  return {
+    runId: "run-001",
+    feature: "test-feature",
+    startedAt: "2026-01-01T00:00:00Z",
+    completedAt: "2026-01-01T00:01:00Z",
+    totalCost: stories.reduce((sum, s) => sum + s.cost, 0),
+    totalStories: stories.length,
+    storiesCompleted: stories.filter((s) => s.success).length,
+    storiesFailed: stories.filter((s) => !s.success).length,
+    totalDurationMs: 60000,
+    stories,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// AC-6: complexityAccuracy uses initialComplexity as predicted complexity
+// ---------------------------------------------------------------------------
+
+describe("calculateAggregateMetrics - complexityAccuracy uses initialComplexity", () => {
+  test("complexityAccuracy keyed by initialComplexity when present", () => {
+    // Story originally predicted as 'simple' but escalated (finalTier = 'powerful')
+    const story = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",          // post-escalation complexity
+      initialComplexity: "simple",   // original prediction
+      modelTier: "fast",
+      finalTier: "powerful",
+      attempts: 2,
+      firstPassSuccess: false,
+    });
+
+    const runs = [makeRun([story])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    // complexityAccuracy should be keyed by initialComplexity ("simple"), not complexity ("medium")
+    expect(aggregate.complexityAccuracy["simple"]).toBeDefined();
+    expect(aggregate.complexityAccuracy["medium"]).toBeUndefined();
+  });
+
+  test("mismatch detected when initialComplexity tier != finalTier", () => {
+    const escalatedStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",
+      initialComplexity: "simple",
+      modelTier: "fast",
+      finalTier: "powerful",
+      attempts: 2,
+      firstPassSuccess: false,
+    });
+
+    const runs = [makeRun([escalatedStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    // simple -> powerful: mismatch expected
+    expect(aggregate.complexityAccuracy["simple"].mismatchRate).toBeGreaterThan(0);
+  });
+
+  test("no mismatch when initialComplexity tier matches finalTier", () => {
+    const successStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",
+      initialComplexity: "medium",
+      modelTier: "balanced",
+      finalTier: "balanced",
+      attempts: 1,
+      firstPassSuccess: true,
+    });
+
+    const runs = [makeRun([successStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    expect(aggregate.complexityAccuracy["medium"].mismatchRate).toBe(0);
+  });
+
+  test("falls back to complexity when initialComplexity is absent (backward compat)", () => {
+    // Legacy story metrics without initialComplexity
+    const legacyStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "complex",
+      // no initialComplexity
+      modelTier: "powerful",
+      finalTier: "powerful",
+    });
+
+    const runs = [makeRun([legacyStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    // Falls back to complexity as key
+    expect(aggregate.complexityAccuracy["complex"]).toBeDefined();
+  });
+
+  test("mixes initialComplexity-keyed and legacy entries correctly", () => {
+    const modernStory = makeStoryMetrics({
+      storyId: "US-001",
+      complexity: "medium",
+      initialComplexity: "simple",
+      modelTier: "balanced",
+      finalTier: "balanced",
+    });
+    const legacyStory = makeStoryMetrics({
+      storyId: "US-002",
+      complexity: "complex",
+      // no initialComplexity
+      modelTier: "powerful",
+      finalTier: "powerful",
+    });
+
+    const runs = [makeRun([modernStory, legacyStory])];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    expect(aggregate.complexityAccuracy["simple"]).toBeDefined();   // from initialComplexity
+    expect(aggregate.complexityAccuracy["complex"]).toBeDefined();  // from complexity fallback
+    expect(aggregate.complexityAccuracy["medium"]).toBeUndefined(); // NOT used (initialComplexity takes over)
+  });
+
+  test("complexityAccuracy.predicted count matches number of stories with that initialComplexity", () => {
+    const stories = [
+      makeStoryMetrics({ storyId: "US-001", complexity: "medium", initialComplexity: "simple", finalTier: "balanced" }),
+      makeStoryMetrics({ storyId: "US-002", complexity: "medium", initialComplexity: "simple", finalTier: "balanced" }),
+      makeStoryMetrics({ storyId: "US-003", complexity: "complex", initialComplexity: "complex", finalTier: "powerful" }),
+    ];
+
+    const runs = [makeRun(stories)];
+    const aggregate = calculateAggregateMetrics(runs);
+
+    expect(aggregate.complexityAccuracy["simple"].predicted).toBe(2);
+    expect(aggregate.complexityAccuracy["complex"].predicted).toBe(1);
+  });
+});

package/test/unit/metrics/tracker.test.ts

@@ -0,0 +1,186 @@
+/**
+ * Metrics Tracker — RRP-002: initialComplexity in StoryMetrics
+ *
+ * AC-4: StoryMetrics gains initialComplexity?: string field
+ * AC-5: collectStoryMetrics() reads story.routing.initialComplexity,
+ *       falls back to routing.complexity for backward compat
+ */
+
+import { describe, expect, test } from "bun:test";
+import { DEFAULT_CONFIG } from "../../../src/config/defaults";
+import type { NaxConfig } from "../../../src/config";
+import type { PipelineContext } from "../../../src/pipeline/types";
+import type { PRD, UserStory } from "../../../src/prd";
+import type { StoryRouting } from "../../../src/prd/types";
+import { collectStoryMetrics } from "../../../src/metrics/tracker";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeStory(overrides?: Partial<UserStory>): UserStory {
+  return {
+    id: "US-001",
+    title: "Test Story",
+    description: "Test description",
+    acceptanceCriteria: [],
+    tags: [],
+    dependencies: [],
+    status: "passed",
+    passes: true,
+    escalations: [],
+    attempts: 1,
+    ...overrides,
+  };
+}
+
+function makePRD(story: UserStory): PRD {
+  return {
+    project: "test-project",
+    feature: "test-feature",
+    branchName: "feat/test",
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    userStories: [story],
+  };
+}
+
+function makeConfig(): NaxConfig {
+  return { ...DEFAULT_CONFIG };
+}
+
+function makeCtx(story: UserStory, routingOverrides?: Partial<PipelineContext["routing"]>): PipelineContext {
+  return {
+    config: makeConfig(),
+    prd: makePRD(story),
+    story,
+    stories: [story],
+    routing: {
+      complexity: "medium",
+      modelTier: "balanced",
+      testStrategy: "test-after",
+      reasoning: "test",
+      ...routingOverrides,
+    },
+    workdir: "/tmp/nax-tracker-test",
+    hooks: { hooks: {} },
+    agentResult: {
+      success: true,
+      output: "",
+      estimatedCost: 0.01,
+      durationMs: 5000,
+    },
+  } as unknown as PipelineContext;
+}
+
+// ---------------------------------------------------------------------------
+// AC-5: collectStoryMetrics reads initialComplexity from story.routing
+// ---------------------------------------------------------------------------
+
+describe("collectStoryMetrics - initialComplexity field", () => {
+  test("includes initialComplexity from story.routing.initialComplexity", () => {
+    const routing: StoryRouting = {
+      complexity: "medium",
+      initialComplexity: "simple", // original prediction before potential escalation
+      testStrategy: "test-after",
+      reasoning: "test",
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "medium" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("simple");
+  });
+
+  test("initialComplexity differs from complexity when story was escalated", () => {
+    const routing: StoryRouting = {
+      complexity: "medium",         // complexity as classified
+      initialComplexity: "simple",  // original first-classify prediction
+      modelTier: "powerful",        // escalated tier
+      testStrategy: "three-session-tdd",
+      reasoning: "escalated",
+    };
+    const story = makeStory({
+      routing,
+      escalations: [
+        {
+          fromTier: "balanced",
+          toTier: "powerful",
+          reason: "test failure",
+          timestamp: new Date().toISOString(),
+        },
+      ],
+      attempts: 2,
+    });
+    const ctx = makeCtx(story, { complexity: "medium", modelTier: "balanced" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("simple");
+    // complexity field unchanged (backward compat)
+    expect(metrics.complexity).toBe("medium");
+  });
+
+  test("falls back to routing.complexity when story.routing.initialComplexity is absent", () => {
+    // Backward compat: story.routing exists but has no initialComplexity
+    const routing: StoryRouting = {
+      complexity: "complex",
+      testStrategy: "three-session-tdd",
+      reasoning: "legacy routing",
+      // no initialComplexity
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "complex" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("complex");
+  });
+
+  test("falls back to routing.complexity when story.routing is undefined", () => {
+    const story = makeStory({ routing: undefined });
+    const ctx = makeCtx(story, { complexity: "simple" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(metrics.initialComplexity).toBe("simple");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// AC-4: StoryMetrics type has initialComplexity?: string
+// ---------------------------------------------------------------------------
+
+describe("StoryMetrics type - initialComplexity field", () => {
+  test("StoryMetrics includes initialComplexity field", () => {
+    const routing: StoryRouting = {
+      complexity: "medium",
+      initialComplexity: "simple",
+      testStrategy: "test-after",
+      reasoning: "test",
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "medium" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    // TypeScript will error at compile time if initialComplexity is not on StoryMetrics
+    expect("initialComplexity" in metrics).toBe(true);
+  });
+
+  test("initialComplexity is a string when present", () => {
+    const routing: StoryRouting = {
+      complexity: "expert",
+      initialComplexity: "expert",
+      testStrategy: "three-session-tdd",
+      reasoning: "test",
+    };
+    const story = makeStory({ routing });
+    const ctx = makeCtx(story, { complexity: "expert" });
+
+    const metrics = collectStoryMetrics(ctx, new Date().toISOString());
+
+    expect(typeof metrics.initialComplexity).toBe("string");
+  });
+});