@nathapp/nax 0.18.1

package/test/execution/post-verify.test.ts

@@ -0,0 +1,224 @@
+/**
+ * Unit tests for post-verify regression gate (BUG-009)
+ *
+ * Tests the logic for:
+ * - Running regression gate after scoped verification passes
+ * - Skipping regression gate when scoped verification already ran full suite
+ * - Feeding regression failures into rectification loop
+ */
+
+import { describe, expect, test } from "bun:test";
+import type { RegressionGateConfig } from "../../src/config/schema";
+
+describe("RegressionGateConfig", () => {
+  test("should have correct default values", () => {
+    const defaultConfig: RegressionGateConfig = {
+      enabled: true,
+      timeoutSeconds: 120,
+    };
+
+    expect(defaultConfig.enabled).toBe(true);
+    expect(defaultConfig.timeoutSeconds).toBe(120);
+  });
+
+  test("should allow disabling regression gate", () => {
+    const config: RegressionGateConfig = {
+      enabled: false,
+      timeoutSeconds: 120,
+    };
+
+    expect(config.enabled).toBe(false);
+  });
+
+  test("should allow custom timeout", () => {
+    const config: RegressionGateConfig = {
+      enabled: true,
+      timeoutSeconds: 180,
+    };
+
+    expect(config.timeoutSeconds).toBe(180);
+  });
+});
+
+describe("Regression Gate Logic", () => {
+  test("should run regression gate when scoped tests were run (changed files > 0)", () => {
+    const changedTestFiles = ["test/foo.test.ts", "test/bar.test.ts"];
+    const regressionGateEnabled = true;
+    const scopedTestsWereRun = changedTestFiles.length > 0;
+
+    // Logic: regression gate should run
+    const shouldRunRegressionGate = regressionGateEnabled && scopedTestsWereRun;
+    expect(shouldRunRegressionGate).toBe(true);
+  });
+
+  test("should skip regression gate when scoped tests ran full suite (changed files = 0)", () => {
+    const changedTestFiles: string[] = [];
+    const regressionGateEnabled = true;
+    const scopedTestsWereRun = changedTestFiles.length > 0;
+
+    // Logic: regression gate should NOT run (full suite already ran)
+    const shouldRunRegressionGate = regressionGateEnabled && scopedTestsWereRun;
+    expect(shouldRunRegressionGate).toBe(false);
+  });
+
+  test("should skip regression gate when disabled in config", () => {
+    const changedTestFiles = ["test/foo.test.ts"];
+    const regressionGateEnabled = false;
+    const scopedTestsWereRun = changedTestFiles.length > 0;
+
+    // Logic: regression gate should NOT run (disabled)
+    const shouldRunRegressionGate = regressionGateEnabled && scopedTestsWereRun;
+    expect(shouldRunRegressionGate).toBe(false);
+  });
+
+  test("should skip regression gate when both disabled AND no changed files", () => {
+    const changedTestFiles: string[] = [];
+    const regressionGateEnabled = false;
+    const scopedTestsWereRun = changedTestFiles.length > 0;
+
+    // Logic: regression gate should NOT run
+    const shouldRunRegressionGate = regressionGateEnabled && scopedTestsWereRun;
+    expect(shouldRunRegressionGate).toBe(false);
+  });
+});
+
+describe("Regression Failure Handling", () => {
+  test("should prefix regression errors with REGRESSION:", () => {
+    const regressionStatus = "TEST_FAILURE";
+    const diagnosticContext = `REGRESSION: ${regressionStatus}`;
+
+    expect(diagnosticContext).toBe("REGRESSION: TEST_FAILURE");
+    expect(diagnosticContext).toContain("REGRESSION:");
+  });
+
+  test("should handle different regression failure statuses", () => {
+    const statuses = ["TEST_FAILURE", "TIMEOUT", "ENVIRONMENTAL_FAILURE", "ASSET_CHECK_FAILED"];
+
+    for (const status of statuses) {
+      const diagnosticContext = `REGRESSION: ${status}`;
+      expect(diagnosticContext).toContain("REGRESSION:");
+      expect(diagnosticContext).toContain(status);
+    }
+  });
+});
+
+describe("Rectification Prompt for Regression", () => {
+  test("should include REGRESSION prefix in rectification prompt", () => {
+    const basePrompt = `# Rectification Required
+
+Your changes caused test regressions. Fix these without breaking existing logic.`;
+
+    const regressionPrompt = `# REGRESSION: Cross-Story Test Failures
+
+Your changes passed scoped tests but broke unrelated tests. Fix these regressions.
+
+${basePrompt}`;
+
+    expect(regressionPrompt).toContain("# REGRESSION:");
+    expect(regressionPrompt).toContain("passed scoped tests but broke unrelated tests");
+    expect(regressionPrompt).toContain(basePrompt);
+  });
+
+  test("regression prompt should emphasize cross-story nature", () => {
+    const regressionPrompt =
+      "# REGRESSION: Cross-Story Test Failures\n\nYour changes passed scoped tests but broke unrelated tests.";
+
+    expect(regressionPrompt).toContain("Cross-Story");
+    expect(regressionPrompt).toContain("unrelated tests");
+  });
+});
+
+describe("Regression Gate Timeout", () => {
+  test("should use config.execution.regressionGate.timeoutSeconds", () => {
+    const regressionGateConfig: RegressionGateConfig = {
+      enabled: true,
+      timeoutSeconds: 120,
+    };
+
+    expect(regressionGateConfig.timeoutSeconds).toBe(120);
+  });
+
+  test("should allow different timeout from verification timeout", () => {
+    const verificationTimeoutSeconds = 300;
+    const regressionGateTimeoutSeconds = 120;
+
+    // Regression gate can have different timeout (usually shorter)
+    expect(regressionGateTimeoutSeconds).not.toBe(verificationTimeoutSeconds);
+    expect(regressionGateTimeoutSeconds).toBeLessThan(verificationTimeoutSeconds);
+  });
+});
+
+describe("Story State After Regression Failure", () => {
+  test("should revert story to pending status", () => {
+    const story = {
+      id: "US-001",
+      status: "passed" as const,
+      passes: true,
+      priorErrors: [] as string[],
+      attempts: 0,
+    };
+
+    // After regression failure
+    const updatedStory = {
+      ...story,
+      status: "pending" as const,
+      passes: false,
+      priorErrors: [...story.priorErrors, "REGRESSION: TEST_FAILURE"],
+    };
+
+    expect(updatedStory.status).toBe("pending");
+    expect(updatedStory.passes).toBe(false);
+    expect(updatedStory.priorErrors).toContain("REGRESSION: TEST_FAILURE");
+  });
+
+  test("should increment attempts when countsTowardEscalation is true", () => {
+    const story = { id: "US-001", attempts: 0 };
+    const countsTowardEscalation = true;
+
+    // After regression failure that counts toward escalation
+    const updatedAttempts = countsTowardEscalation ? story.attempts + 1 : story.attempts;
+
+    expect(updatedAttempts).toBe(1);
+  });
+
+  test("should NOT increment attempts when countsTowardEscalation is false", () => {
+    const story = { id: "US-001", attempts: 0 };
+    const countsTowardEscalation = false;
+
+    // After regression failure that doesn't count toward escalation (e.g., timeout)
+    const updatedAttempts = countsTowardEscalation ? story.attempts + 1 : story.attempts;
+
+    expect(updatedAttempts).toBe(0);
+  });
+});
+
+describe("Story Metrics Removal", () => {
+  test("should remove story metrics on regression failure", () => {
+    const allStoryMetrics = [
+      { storyId: "US-001", cost: 0.5 },
+      { storyId: "US-002", cost: 0.3 },
+    ];
+    const failedStoryIds = new Set(["US-001"]);
+
+    // Remove metrics for failed stories
+    const remainingMetrics = allStoryMetrics.filter((m) => !failedStoryIds.has(m.storyId));
+
+    expect(remainingMetrics.length).toBe(1);
+    expect(remainingMetrics[0].storyId).toBe("US-002");
+  });
+
+  test("should remove metrics for all stories in batch", () => {
+    const allStoryMetrics = [
+      { storyId: "US-001", cost: 0.5 },
+      { storyId: "US-002", cost: 0.3 },
+      { storyId: "US-003", cost: 0.2 },
+    ];
+    const storyIds = new Set(["US-001", "US-002"]);
+
+    // Remove metrics for entire batch
+    const remainingMetrics = allStoryMetrics.filter((m) => !storyIds.has(m.storyId));
+
+    expect(remainingMetrics.length).toBe(1);
+    expect(remainingMetrics[0].storyId).toBe("US-003");
+  });
+});

package/test/helpers/timeout.ts

@@ -0,0 +1,42 @@
+/**
+ * Test Timeout Helpers
+ *
+ * Utilities to prevent tests from hanging indefinitely.
+ */
+
+/**
+ * Wraps a promise with a hard timeout.
+ * If the promise doesn't resolve within the timeout, rejects with a timeout error.
+ *
+ * @param promise The promise to wrap
+ * @param timeoutMs Timeout in milliseconds
+ * @param operation Description of the operation (for error messages)
+ * @returns The promise result if it completes in time
+ * @throws TimeoutError if the timeout is exceeded
+ */
+export async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, operation = "Operation"): Promise<T> {
+  return Promise.race([
+    promise,
+    new Promise<T>((_, reject) =>
+      setTimeout(() => reject(new Error(`${operation} timed out after ${timeoutMs}ms`)), timeoutMs),
+    ),
+  ]);
+}
+
+/**
+ * Wraps a function call with a hard timeout.
+ * Useful for wrapping synchronous or async functions that might hang.
+ *
+ * @param fn The function to execute
+ * @param timeoutMs Timeout in milliseconds
+ * @param operation Description of the operation (for error messages)
+ * @returns The function result if it completes in time
+ * @throws TimeoutError if the timeout is exceeded
+ */
+export async function executeWithTimeout<T>(
+  fn: () => Promise<T> | T,
+  timeoutMs: number,
+  operation = "Operation",
+): Promise<T> {
+  return withTimeout(Promise.resolve(fn()), timeoutMs, operation);
+}

package/test/integration/US-002-TEST-SUMMARY.md

@@ -0,0 +1,107 @@
+# US-002 Test Summary: Context Provider Injection
+
+## Overview
+Created comprehensive test suite for US-002 that verifies context providers inject external data into agent prompts with proper token budget management.
+
+## Test File
+- **Location**: `test/integration/context-provider-injection.test.ts`
+- **Total Tests**: 20
+- **Passing**: 14 (features already implemented)
+- **Failing**: 6 (features not yet implemented)
+
+## Test Coverage by Acceptance Criteria
+
+### ✅ AC1: All registered context providers are called before agent execution
+**Status**: All tests passing (feature implemented)
+- ✓ Calls all registered context providers
+- ✓ Providers receive the current story
+- ✓ Works with no providers registered
+
+### ✅ AC2: Provider content appended under markdown section with label
+**Status**: All tests passing (feature implemented)
+- ✓ Appends provider content under labeled markdown section
+- ✓ Multiple providers create separate labeled sections
+- ✓ Provider content appended to existing context markdown
+
+### ❌ AC3: Total injected tokens respect token budget
+**Status**: 4 tests failing (feature NOT implemented correctly)
+
+**Issue**: Current implementation uses hardcoded `PLUGIN_CONTEXT_MAX_TOKENS = 20_000` instead of reading from `config.execution.contextProviderTokenBudget`
+
+Failing tests:
+- ✗ Respects default token budget of 2000 tokens when not configured
+- ✗ Respects custom token budget from config
+- ✗ Providers added in order until budget exhausted
+- ✗ Single provider exceeding budget is skipped
+
+### ✅ AC4: Provider errors caught, logged, and skipped
+**Status**: All tests passing (feature implemented)
+- ✓ Continues when a provider throws error
+- ✓ Handles all providers failing gracefully
+- ✓ Error in one provider doesn't affect others
+
+### ❌ AC5: Token budget configurable via execution.contextProviderTokenBudget
+**Status**: 2 tests failing (feature NOT implemented)
+
+**Issue**:
+1. `ExecutionConfig` type doesn't include `contextProviderTokenBudget` field
+2. `DEFAULT_CONFIG` doesn't set default value of 2000 tokens
+3. Context stage uses hardcoded value instead of reading from config
+
+Failing tests:
+- ✗ Default config includes contextProviderTokenBudget with default of 2000
+- ✗ Different projects can have different token budgets
+
+## Implementation Gaps
+
+### 1. Config Schema Missing Field
+**File**: `src/config/schema.ts`
+- Add `contextProviderTokenBudget: number` to `ExecutionConfig` interface
+- Add validation in `ExecutionConfigSchema` (Zod)
+- Set default value of 2000 in `DEFAULT_CONFIG.execution`
+
+### 2. Context Stage Uses Hardcoded Value
+**File**: `src/pipeline/stages/context.ts`
+- Line 32: `const PLUGIN_CONTEXT_MAX_TOKENS = 20_000;` (hardcoded)
+- Should read from: `ctx.config.execution.contextProviderTokenBudget`
+- Lines 62, 72: Replace `PLUGIN_CONTEXT_MAX_TOKENS` with config value
+
+## Test Execution
+
+```bash
+# Run US-002 tests only
+bun test ./test/integration/context-provider-injection.test.ts
+
+# Current results:
+# 14 pass, 6 fail, 46 expect() calls
+```
+
+## Next Steps for Implementer
+
+1. **Update ExecutionConfig interface** (src/config/schema.ts):
+   - Add `contextProviderTokenBudget: number` field
+   - Add Zod validation: `z.number().int().min(100).max(100000).default(2000)`
+   - Add to DEFAULT_CONFIG: `contextProviderTokenBudget: 2000`
+
+2. **Update context stage** (src/pipeline/stages/context.ts):
+   - Remove hardcoded `PLUGIN_CONTEXT_MAX_TOKENS` constant
+   - Read budget from `ctx.config.execution.contextProviderTokenBudget`
+   - Use configured value in budget checks (lines 62, 72)
+
+3. **Verify all tests pass**:
+   ```bash
+   bun test ./test/integration/context-provider-injection.test.ts
+   ```
+
+## Coverage Notes
+
+The test suite covers:
+- ✓ Provider registration and invocation
+- ✓ Markdown formatting with labels
+- ✓ Error handling and soft failures
+- ✓ Token budget enforcement (with config)
+- ✓ Multi-provider orchestration
+- ✓ Integration with existing PRD context
+- ✓ Built context element tracking
+
+All edge cases are covered per acceptance criteria.

package/test/integration/US-003-TEST-SUMMARY.md

@@ -0,0 +1,149 @@
+# US-003 Test Summary: Review Plugins Run After Built-in Verification
+
+**Story ID:** US-003
+**Date:** 2026-02-27
+**Status:** ✅ PASSED
+**Test File:** `test/integration/review-plugin-integration.test.ts`
+
+## Overview
+
+This test suite verifies that plugin reviewers are correctly integrated into the review pipeline stage, running after built-in checks and triggering appropriate retry/escalation on failure.
+
+## Test Results
+
+**Total Tests:** 19
+**Passed:** 19
+**Failed:** 0
+**Success Rate:** 100%
+
+## Acceptance Criteria Coverage
+
+### ✅ AC1: Plugin reviewers run after built-in checks pass
+
+| Test | Status |
+|------|--------|
+| Plugin reviewers execute when built-in checks pass | ✅ PASS |
+| Plugin reviewers do not run if built-in checks fail | ✅ PASS |
+| No plugin reviewers registered - continues normally | ✅ PASS |
+
+**Verification:** Plugin reviewers only execute after built-in checks succeed, preventing unnecessary work when code quality gates fail.
+
+### ✅ AC2: Each reviewer receives workdir and changed files
+
+| Test | Status |
+|------|--------|
+| Reviewer receives correct workdir | ✅ PASS |
+| Reviewer receives list of changed files | ✅ PASS |
+| Reviewer receives empty array when no files changed | ✅ PASS |
+
+**Verification:** Reviewers receive accurate context about the working directory and which files were modified, enabling targeted analysis.
+
+### ✅ AC3: Reviewer failure triggers retry/escalation
+
+| Test | Status |
+|------|--------|
+| Failing reviewer returns fail action | ✅ PASS |
+| Reviewer failure includes plugin name in reason | ✅ PASS |
+
+**Verification:** When a plugin reviewer fails, the pipeline returns a `fail` action with the plugin name in the failure reason, triggering the same retry/escalation logic as built-in check failures.
+
+### ✅ AC4: Reviewer output included in story result
+
+| Test | Status |
+|------|--------|
+| Passing reviewer output is captured | ✅ PASS |
+| Failing reviewer output is captured | ✅ PASS |
+
+**Verification:** All reviewer outputs (success and failure) are stored in `ctx.reviewResult.pluginReviewers`, providing debugging information and audit trail.
+
+### ✅ AC5: Exceptions count as failures
+
+| Test | Status |
+|------|--------|
+| Reviewer throwing exception counts as failure | ✅ PASS |
+| Exception message captured in output | ✅ PASS |
+| Non-Error exception converted to string | ✅ PASS |
+
+**Verification:** When a reviewer throws an exception, it's treated as a failure with the error message captured for debugging. The pipeline correctly handles both Error objects and primitive throws.
+
+### ✅ AC6: Multiple reviewers run sequentially with short-circuiting
+
+| Test | Status |
+|------|--------|
+| Multiple reviewers run in order when all pass | ✅ PASS |
+| First failure short-circuits remaining reviewers | ✅ PASS |
+| Exception short-circuits remaining reviewers | ✅ PASS |
+
+**Verification:** Reviewers execute sequentially in registration order. When one fails (or throws), subsequent reviewers are skipped, providing fail-fast behavior.
+
+### ✅ Edge Cases
+
+| Test | Status |
+|------|--------|
+| No plugins context - continues normally | ✅ PASS |
+| Reviewer returns empty output | ✅ PASS |
+| Reviewer without exitCode works | ✅ PASS |
+
+**Verification:** The implementation handles edge cases gracefully: missing plugin context, empty output strings, and optional exitCode field.
+
+## Implementation Verification
+
+### Key Files Modified
+
+1. **`src/pipeline/stages/review.ts`**
+   - Lines 77-155: Plugin reviewer execution logic
+   - Lines 35-53: `getChangedFiles()` helper function
+   - Correctly integrates plugin reviewers after built-in checks
+
+2. **`src/review/types.ts`**
+   - Lines 26-38: `PluginReviewerResult` interface
+   - Line 51: Extended `ReviewResult` with `pluginReviewers` field
+
+3. **`test/integration/review-plugin-integration.test.ts`**
+   - 722 lines of comprehensive test coverage
+   - Mock plugins and reviewers for isolated testing
+   - Git repository setup for realistic changed file detection
+
+### Type Safety
+
+- ✅ All TypeScript types correctly defined
+- ✅ No type errors (`bun run typecheck` passes)
+- ✅ Proper type guards and assertions
+
+### Error Handling
+
+- ✅ Exceptions caught and converted to failures
+- ✅ Error messages preserved for debugging
+- ✅ Non-Error throws handled correctly
+- ✅ Missing optional fields handled safely
+
+### Integration Points
+
+- ✅ Integrates with `PluginRegistry.getReviewers()`
+- ✅ Uses existing pipeline context structure
+- ✅ Follows established patterns from built-in checks
+- ✅ Compatible with retry/escalation logic
+
+## Performance Considerations
+
+- Reviewers run sequentially (not parallel) to prevent resource contention
+- Fail-fast behavior minimizes wasted computation
+- Changed files retrieved once and reused for all reviewers
+- No unnecessary git operations or file system scans
+
+## Conclusion
+
+**US-003 is fully implemented and verified.** All acceptance criteria are met with comprehensive test coverage. The implementation follows the codebase patterns, handles edge cases gracefully, and integrates seamlessly with the existing plugin system architecture.
+
+## Test Execution
+
+```bash
+$ bun test test/integration/review-plugin-integration.test.ts
+
+ 19 pass
+ 0 fail
+ 51 expect() calls
+Ran 19 tests across 1 file. [1.71s]
+```
+
+**Final Status:** ✅ READY FOR PRODUCTION

package/test/integration/US-004-TEST-SUMMARY.md

@@ -0,0 +1,106 @@
+# US-004: Reporter plugins receive lifecycle events — Test Summary
+
+**Status:** ✅ PASSED
+**Date:** 2026-02-27
+**Commit:** 26181a1
+
+## Overview
+
+This story implements reporter lifecycle events that fire at appropriate points in the runner loop. All reporter calls are fire-and-forget (errors logged, never block pipeline).
+
+## Implementation Summary
+
+### Changes Made
+
+1. **Moved PRD initialization** (runner.ts:205)
+   - Moved `prd` declaration before try block to make it accessible in finally block
+   - Ensures `prd` is available for onRunEnd event even on failure/abort
+
+2. **Consolidated onRunEnd calls** (runner.ts:1417-1439)
+   - Moved onRunEnd reporter events to finally block
+   - Removed duplicate calls from success paths (parallel and sequential)
+   - Guarantees onRunEnd fires even when run fails or is aborted
+
+3. **Added dry-run onStoryComplete events** (runner.ts:666-684)
+   - Added missing onStoryComplete events for dry-run mode
+   - Ensures reporters receive events consistently across all execution modes
+
+### Key Design Decisions
+
+- **Finally block placement**: onRunEnd must fire even on exceptions, so it's placed in the finally block before teardown and lock release
+- **Error isolation**: Each reporter call is wrapped in try/catch to prevent one reporter's failure from affecting others
+- **Event ordering**: onRunEnd fires before plugin teardown to ensure reporters can still access plugin state
+
+## Test Results
+
+All 9 tests in `test/integration/reporter-lifecycle.test.ts` pass:
+
+### AC1: onRunStart fires once at run start ✅
+- Verified event contains: runId, feature, totalStories, startTime
+- Verified event fires exactly once per run
+
+### AC2: onStoryComplete fires after each story ✅
+- Verified event contains: runId, storyId, status, durationMs, cost, tier, testStrategy
+- Verified event fires for each story execution (including dry-run)
+- Verified correct status values (completed, failed, skipped, paused)
+
+### AC3: onRunEnd fires once at run end ✅
+- Verified event contains: runId, totalDurationMs, totalCost, storySummary
+- Verified storySummary contains: completed, failed, skipped, paused counts
+- Verified correct counts match PRD state
+
+### AC4: Reporter errors never block execution ✅
+- Verified failing reporter doesn't abort run
+- Verified run completes successfully despite reporter errors
+- Verified errors are logged (not thrown)
+
+### AC5: Multiple reporters all receive events ✅
+- Verified two reporters both receive onRunStart, onStoryComplete, onRunEnd
+- Verified second reporter receives events even if first reporter fails
+- Verified no short-circuiting on error (all reporters always execute)
+
+### AC6: Events fire even when run fails or is aborted ✅
+- Verified onRunStart and onRunEnd fire when stories are pre-failed
+- Verified onRunEnd fires in finally block (even on exception)
+- Verified storySummary reflects actual failure state
+
+## Additional Test Coverage
+
+- **onStoryComplete for different outcomes**: Verified events for completed, failed, skipped, paused stories
+- **Multiple stories**: Verified consistent runId across all events in same run
+- **Dry-run mode**: Verified reporters receive events in dry-run mode
+
+## Verification Command
+
+```bash
+bun test test/integration/reporter-lifecycle.test.ts
+```
+
+**Result:** 9 pass, 0 fail, 48 expect() calls
+
+## Integration with Existing Code
+
+- **US-001 (Plugin loading)**: Uses pluginRegistry.getReporters() to retrieve all loaded reporters
+- **US-002 (Context provider injection)**: No conflicts, reporters operate independently
+- **US-003 (Review plugins)**: No conflicts, different lifecycle hooks
+
+## Notes
+
+- Reporter events are fire-and-forget by design
+- All reporter methods are optional (IReporter interface)
+- Reporter errors are logged at WARN level (not ERROR) since they're non-critical
+- onRunEnd always fires in finally block, even if try block throws
+- PRD must be accessible in finally block, so it's initialized before try
+
+## Acceptance Criteria Status
+
+| AC | Description | Status |
+|----|-------------|--------|
+| 1  | onRunStart fires once at run start with runId, feature, totalStories, startTime | ✅ |
+| 2  | onStoryComplete fires after each story with storyId, status, durationMs, cost, tier, testStrategy | ✅ |
+| 3  | onRunEnd fires once at run end with runId, totalDurationMs, totalCost, storySummary counts | ✅ |
+| 4  | Reporter errors are caught and logged but never block execution | ✅ |
+| 5  | Multiple reporters all receive events (not short-circuited on error) | ✅ |
+| 6  | Events fire even when the run fails or is aborted (onRunEnd still fires) | ✅ |
+
+**Overall Status:** ✅ ALL ACCEPTANCE CRITERIA MET