@nathapp/nax 0.50.2 → 0.51.0

package/src/metrics/tracker.ts

@@ -1,200 +0,0 @@
-/**
- * Metrics Tracker
- *
- * Collects and persists per-story and per-run metrics.
- */
-
-import path from "node:path";
-import { resolveModel } from "../config/schema";
-import type { PipelineContext } from "../pipeline/types";
-import { loadJsonFile, saveJsonFile } from "../utils/json-file";
-import type { RunMetrics, StoryMetrics } from "./types";
-
-/**
- * Collect metrics for a single story execution.
- *
- * Extracts timing, cost, model usage, and escalation data from the pipeline context.
- *
- * @param ctx - Pipeline context with execution results
- * @param storyStartTime - Story start timestamp (ISO string)
- * @returns Story metrics object
- *
- * @example
- * ```ts
- * const metrics = collectStoryMetrics(ctx, "2026-02-17T10:30:00.000Z");
- * // {
- * //   storyId: "US-001",
- * //   complexity: "medium",
- * //   modelTier: "balanced",
- * //   modelUsed: "claude-sonnet-4.5",
- * //   attempts: 1,
- * //   finalTier: "balanced",
- * //   success: true,
- * //   cost: 0.0234,
- * //   durationMs: 45000,
- * //   firstPassSuccess: true,
- * //   ...
- * // }
- * ```
- */
-export function collectStoryMetrics(ctx: PipelineContext, storyStartTime: string): StoryMetrics {
-  const story = ctx.story;
-  const routing = ctx.routing;
-  const agentResult = ctx.agentResult;
-
-  // Calculate attempts (initial + escalations)
-  // BUG-067: priorFailures captures cross-tier attempts that story.escalations never records
-  const escalationCount = story.escalations?.length || 0;
-  const priorFailureCount = story.priorFailures?.length || 0;
-  const attempts = priorFailureCount + Math.max(1, story.attempts || 1);
-
-  // Determine final tier (from last escalation or initial routing)
-  const finalTier = escalationCount > 0 ? story.escalations[escalationCount - 1].toTier : routing.modelTier;
-
-  // First pass success = succeeded with no prior failures and no escalations (BUG-067)
-  const firstPassSuccess = agentResult?.success === true && escalationCount === 0 && priorFailureCount === 0;
-
-  // Extract model name from config
-  const modelEntry = ctx.config.models[routing.modelTier];
-  const modelDef = modelEntry ? resolveModel(modelEntry) : null;
-  const modelUsed = modelDef?.model || routing.modelTier;
-
-  // initialComplexity: prefer story.routing.initialComplexity (first classify),
-  // fall back to routing.complexity for backward compat
-  const initialComplexity = story.routing?.initialComplexity ?? routing.complexity;
-
-  // fullSuiteGatePassed: true only for TDD strategies when gate passes
-  const isTddStrategy =
-    routing.testStrategy === "three-session-tdd" || routing.testStrategy === "three-session-tdd-lite";
-  const fullSuiteGatePassed = isTddStrategy ? (ctx.fullSuiteGatePassed ?? false) : false;
-
-  return {
-    storyId: story.id,
-    complexity: routing.complexity,
-    initialComplexity,
-    modelTier: routing.modelTier,
-    modelUsed,
-    attempts,
-    finalTier,
-    success: agentResult?.success || false,
-    cost: (ctx.accumulatedAttemptCost ?? 0) + (agentResult?.estimatedCost || 0),
-    durationMs: agentResult?.durationMs || 0,
-    firstPassSuccess,
-    startedAt: storyStartTime,
-    completedAt: new Date().toISOString(),
-    fullSuiteGatePassed,
-    runtimeCrashes: ctx.storyRuntimeCrashes ?? 0,
-  };
-}
-
-/**
- * Collect metrics for a batch of stories.
- *
- * Creates individual story metrics for each story in the batch,
- * distributing the total cost and duration proportionally.
- *
- * @param ctx - Pipeline context with batch execution results
- * @param storyStartTime - Batch start timestamp (ISO string)
- * @returns Array of story metrics (one per story in batch)
- *
- * @example
- * ```ts
- * const batchMetrics = collectBatchMetrics(ctx, "2026-02-17T10:30:00.000Z");
- * // [
- * //   { storyId: "US-001", cost: 0.0078, ... },
- * //   { storyId: "US-002", cost: 0.0078, ... },
- * //   { storyId: "US-003", cost: 0.0078, ... },
- * // ]
- * ```
- */
-export function collectBatchMetrics(ctx: PipelineContext, storyStartTime: string): StoryMetrics[] {
-  const stories = ctx.stories;
-  const routing = ctx.routing;
-  const agentResult = ctx.agentResult;
-
-  const totalCost = agentResult?.estimatedCost || 0;
-  const totalDuration = agentResult?.durationMs || 0;
-  const costPerStory = totalCost / stories.length;
-  const durationPerStory = totalDuration / stories.length;
-
-  const modelEntry = ctx.config.models[routing.modelTier];
-  const modelDef = modelEntry ? resolveModel(modelEntry) : null;
-  const modelUsed = modelDef?.model || routing.modelTier;
-
-  return stories.map((story) => {
-    // initialComplexity: prefer story.routing.initialComplexity (if individual routing exists),
-    // fall back to shared routing.complexity (batch stories classified together)
-    const initialComplexity = story.routing?.initialComplexity ?? routing.complexity;
-
-    return {
-      storyId: story.id,
-      complexity: routing.complexity,
-      initialComplexity,
-      modelTier: routing.modelTier,
-      modelUsed,
-      attempts: 1, // batch stories don't escalate individually
-      finalTier: routing.modelTier,
-      success: true, // if batch succeeded, all stories succeeded
-      cost: costPerStory,
-      durationMs: durationPerStory,
-      firstPassSuccess: true, // batch = first pass success
-      startedAt: storyStartTime,
-      completedAt: new Date().toISOString(),
-      fullSuiteGatePassed: false, // batches are not TDD-gated
-      runtimeCrashes: 0, // batch stories don't have individual crash tracking
-    };
-  });
-}
-
-/**
- * Save run metrics to nax/metrics.json.
- *
- * Appends the run metrics to the existing metrics file (or creates it if missing).
- * Each run is a separate entry in the JSON array.
- *
- * @param workdir - Project root directory
- * @param runMetrics - Run metrics to persist
- *
- * @example
- * ```ts
- * await saveRunMetrics("/home/user/project", {
- *   runId: "run-20260217-103045",
- *   feature: "auth-system",
- *   totalCost: 0.1234,
- *   stories: [...],
- *   ...
- * });
- * ```
- */
-export async function saveRunMetrics(workdir: string, runMetrics: RunMetrics): Promise<void> {
-  const metricsPath = path.join(workdir, "nax", "metrics.json");
-
-  // Load existing metrics (returns empty array if file doesn't exist or is invalid)
-  const existing = await loadJsonFile<RunMetrics[]>(metricsPath, "metrics");
-  const allMetrics = Array.isArray(existing) ? existing : [];
-
-  // Append new run
-  allMetrics.push(runMetrics);
-
-  // Write back
-  await saveJsonFile(metricsPath, allMetrics, "metrics");
-}
-
-/**
- * Load all run metrics from nax/metrics.json.
- *
- * @param workdir - Project root directory
- * @returns Array of run metrics, or empty array if file doesn't exist
- *
- * @example
- * ```ts
- * const runs = await loadRunMetrics("/home/user/project");
- * console.log(`Total runs: ${runs.length}`);
- * ```
- */
-export async function loadRunMetrics(workdir: string): Promise<RunMetrics[]> {
-  const metricsPath = path.join(workdir, "nax", "metrics.json");
-
-  const content = await loadJsonFile<RunMetrics[]>(metricsPath, "metrics");
-  return Array.isArray(content) ? content : [];
-}

package/src/metrics/types.ts

@@ -1,115 +0,0 @@
-/**
- * Metrics Tracking Types
- *
- * Structured cost and performance metrics for tracking agent execution.
- */
-
-/**
- * Per-story execution metrics
- */
-export interface StoryMetrics {
-  /** Story ID */
-  storyId: string;
-  /** Classified complexity */
-  complexity: string;
-  /** Initial complexity from first classification — preserved across escalations */
-  initialComplexity?: string;
-  /** Initial model tier */
-  modelTier: string;
-  /** Actual model used (e.g., "claude-sonnet-4.5") */
-  modelUsed: string;
-  /** Number of attempts (includes escalations) */
-  attempts: number;
-  /** Final tier that succeeded */
-  finalTier: string;
-  /** Whether the story succeeded */
-  success: boolean;
-  /** Total cost for this story (including all attempts) */
-  cost: number;
-  /** Total duration in milliseconds */
-  durationMs: number;
-  /** Whether it passed on the first attempt */
-  firstPassSuccess: boolean;
-  /** Timestamp when started */
-  startedAt: string;
-  /** Timestamp when completed */
-  completedAt: string;
-  /** Number of runtime crashes (RUNTIME_CRASH verify status) encountered for this story (BUG-070) */
-  runtimeCrashes?: number;
-  /** Whether TDD full-suite gate passed (only true for TDD strategies when gate passes) */
-  fullSuiteGatePassed?: boolean;
-}
-
-/**
- * Per-run execution metrics
- */
-export interface RunMetrics {
-  /** Unique run ID */
-  runId: string;
-  /** Feature name */
-  feature: string;
-  /** Run start timestamp */
-  startedAt: string;
-  /** Run completion timestamp */
-  completedAt: string;
-  /** Total cost for the run */
-  totalCost: number;
-  /** Total number of stories in the run */
-  totalStories: number;
-  /** Number of stories completed successfully */
-  storiesCompleted: number;
-  /** Number of stories that failed */
-  storiesFailed: number;
-  /** Total duration in milliseconds */
-  totalDurationMs: number;
-  /** Per-story metrics */
-  stories: StoryMetrics[];
-}
-
-/**
- * Aggregate metrics across all runs
- */
-export interface AggregateMetrics {
-  /** Total number of runs */
-  totalRuns: number;
-  /** Total cost across all runs */
-  totalCost: number;
-  /** Total stories across all runs */
-  totalStories: number;
-  /** Percentage of stories passing on first attempt */
-  firstPassRate: number;
-  /** Percentage of stories needing escalation */
-  escalationRate: number;
-  /** Average cost per story */
-  avgCostPerStory: number;
-  /** Average cost per feature run */
-  avgCostPerFeature: number;
-  /** Per-model efficiency metrics */
-  modelEfficiency: Record<
-    string,
-    {
-      /** Total attempts with this model */
-      attempts: number;
-      /** Successful attempts */
-      successes: number;
-      /** Success rate (0-1) */
-      passRate: number;
-      /** Average cost per story */
-      avgCost: number;
-      /** Total cost for this model */
-      totalCost: number;
-    }
-  >;
-  /** Complexity prediction accuracy */
-  complexityAccuracy: Record<
-    string,
-    {
-      /** Number of stories predicted at this complexity */
-      predicted: number;
-      /** Most common final tier used */
-      actualTierUsed: string;
-      /** Rate at which prediction didn't match actual tier needed */
-      mismatchRate: number;
-    }
-  >;
-}

package/src/optimizer/index.ts

@@ -1,63 +0,0 @@
-/**
- * Prompt Optimizer
- *
- * Exports optimizer types, implementations, and factory function.
- */
-
-export type {
-  IPromptOptimizer,
-  PromptOptimizerInput,
-  PromptOptimizerResult,
-} from "./types.js";
-export { estimateTokens } from "./types.js";
-export { NoopOptimizer } from "./noop.optimizer.js";
-export { RuleBasedOptimizer } from "./rule-based.optimizer.js";
-
-import type { NaxConfig } from "../config/schema.js";
-import { getSafeLogger } from "../logger/index.js";
-import type { PluginRegistry } from "../plugins/registry.js";
-import { NoopOptimizer } from "./noop.optimizer.js";
-import { RuleBasedOptimizer } from "./rule-based.optimizer.js";
-import type { IPromptOptimizer } from "./types.js";
-
-/**
- * Resolve the prompt optimizer to use for this run.
- *
- * Resolution order:
- * 1. Plugin-provided optimizer (if any plugins provide "optimizer")
- * 2. Built-in strategy from config (rule-based, noop)
- * 3. Fallback to NoopOptimizer
- *
- * @param config - Nax configuration
- * @param pluginRegistry - Plugin registry (optional, for plugin-provided optimizers)
- * @returns Resolved optimizer instance
- */
-export function resolveOptimizer(config: NaxConfig, pluginRegistry?: PluginRegistry): IPromptOptimizer {
-  // Check if optimizer is disabled
-  if (!config.optimizer?.enabled) {
-    return new NoopOptimizer();
-  }
-
-  // 1. Check plugin registry first
-  if (pluginRegistry) {
-    const pluginOptimizers = pluginRegistry.getOptimizers();
-    if (pluginOptimizers.length > 0) {
-      // Use first plugin optimizer (plugin optimizers use the same interface)
-      return pluginOptimizers[0];
-    }
-  }
-
-  // 2. Use built-in strategy from config
-  const strategy = config.optimizer.strategy ?? "noop";
-
-  switch (strategy) {
-    case "rule-based":
-      return new RuleBasedOptimizer();
-    case "noop":
-      return new NoopOptimizer();
-    default:
-      // Unknown strategy, fallback to noop
-      getSafeLogger()?.warn("optimizer", `Unknown optimizer strategy '${strategy}', using noop`);
-      return new NoopOptimizer();
-  }
-}

package/src/optimizer/noop.optimizer.ts

@@ -1,24 +0,0 @@
-import type { IPromptOptimizer, PromptOptimizerInput, PromptOptimizerResult } from "./types.js";
-import { estimateTokens } from "./types.js";
-
-/**
- * Passthrough optimizer that returns the prompt unchanged.
- *
- * Used as the default when optimization is disabled or
- * when the configured strategy is unrecognized.
- */
-export class NoopOptimizer implements IPromptOptimizer {
-  public readonly name = "noop";
-
-  async optimize(input: PromptOptimizerInput): Promise<PromptOptimizerResult> {
-    const tokens = estimateTokens(input.prompt);
-
-    return {
-      prompt: input.prompt,
-      originalTokens: tokens,
-      optimizedTokens: tokens,
-      savings: 0,
-      appliedRules: [],
-    };
-  }
-}

package/src/optimizer/rule-based.optimizer.ts

@@ -1,248 +0,0 @@
-import type { IPromptOptimizer, PromptOptimizerInput, PromptOptimizerResult } from "./types.js";
-import { estimateTokens } from "./types.js";
-
-interface RuleBasedConfig {
-  stripWhitespace?: boolean;
-  compactCriteria?: boolean;
-  deduplicateContext?: boolean;
-  maxPromptTokens?: number;
-}
-
-const DEFAULT_CONFIG: RuleBasedConfig = {
-  stripWhitespace: true,
-  compactCriteria: true,
-  deduplicateContext: true,
-  maxPromptTokens: 8000,
-};
-
-/**
- * Rule-based optimizer that applies deterministic transformations
- * to reduce token usage without external dependencies.
- */
-export class RuleBasedOptimizer implements IPromptOptimizer {
-  public readonly name = "rule-based";
-
-  async optimize(input: PromptOptimizerInput): Promise<PromptOptimizerResult> {
-    const originalTokens = estimateTokens(input.prompt);
-    const appliedRules: string[] = [];
-
-    let optimized = input.prompt;
-
-    // Get config with defaults
-    const config = {
-      ...DEFAULT_CONFIG,
-      ...input.config.optimizer?.strategies?.["rule-based"],
-    };
-
-    // Rule 1: Strip whitespace
-    if (config.stripWhitespace) {
-      const before = optimized;
-      optimized = this.stripWhitespace(optimized);
-      if (optimized !== before) {
-        appliedRules.push("stripWhitespace");
-      }
-    }
-
-    // Rule 2: Compact acceptance criteria
-    if (config.compactCriteria) {
-      const before = optimized;
-      optimized = this.compactCriteria(optimized);
-      if (optimized !== before) {
-        appliedRules.push("compactCriteria");
-      }
-    }
-
-    // Rule 3: Deduplicate context
-    if (config.deduplicateContext && input.contextMarkdown) {
-      const before = optimized;
-      optimized = this.deduplicateContext(optimized, input.contextMarkdown);
-      if (optimized !== before) {
-        appliedRules.push("deduplicateContext");
-      }
-    }
-
-    // Rule 4: Enforce max prompt tokens
-    if (config.maxPromptTokens) {
-      const currentTokens = estimateTokens(optimized);
-      if (currentTokens > config.maxPromptTokens) {
-        optimized = this.trimToMaxTokens(optimized, config.maxPromptTokens);
-        appliedRules.push("maxPromptTokens");
-      }
-    }
-
-    const optimizedTokens = estimateTokens(optimized);
-    const savings = originalTokens > 0 ? (originalTokens - optimizedTokens) / originalTokens : 0;
-
-    return {
-      prompt: optimized,
-      originalTokens,
-      optimizedTokens,
-      savings,
-      appliedRules,
-    };
-  }
-
-  /**
-   * Collapse multiple blank lines to single blank line and trim trailing whitespace.
-   */
-  private stripWhitespace(prompt: string): string {
-    return (
-      prompt
-        // Trim trailing whitespace from each line
-        .split("\n")
-        .map((line) => line.trimEnd())
-        .join("\n")
-        // Collapse 3+ consecutive newlines to 2
-        .replace(/\n{3,}/g, "\n\n")
-    );
-  }
-
-  /**
-   * Convert verbose acceptance criteria to terse bullet format.
-   */
-  private compactCriteria(prompt: string): string {
-    return (
-      prompt
-        // Remove verbose prefixes
-        .replace(/The user should be able to /gi, "")
-        .replace(/The system must /gi, "")
-        .replace(/The system should /gi, "")
-        .replace(/When the /gi, "")
-        .replace(/When a /gi, "")
-        // Compact common verbose patterns
-        .replace(/it should validate all fields/gi, "validate all fields")
-        .replace(/display an error message/gi, "show error")
-        .replace(/error message/gi, "error")
-    );
-  }
-
-  /**
-   * Remove context sections that duplicate constitution content.
-   */
-  private deduplicateContext(prompt: string, contextMarkdown: string): string {
-    // Find context section
-    const contextSectionMatch = prompt.match(/# Context\n([\s\S]*?)(?=\n#|$)/i);
-    if (!contextSectionMatch) {
-      return prompt;
-    }
-
-    const contextSection = contextSectionMatch[1];
-    const contextLines = contextSection.split("\n");
-
-    // Remove lines that appear in constitution
-    const dedupedLines = contextLines.filter((line) => {
-      const trimmed = line.trim();
-      if (!trimmed) return true; // Keep blank lines for now
-      // Check if this line appears in the context markdown (which may duplicate constitution)
-      return !contextMarkdown.includes(trimmed);
-    });
-
-    // If we removed content, replace the context section
-    if (dedupedLines.length < contextLines.length) {
-      const newContextSection = dedupedLines.join("\n");
-      return prompt.replace(contextSectionMatch[0], `# Context\n${newContextSection}`);
-    }
-
-    return prompt;
-  }
-
-  /**
-   * Trim context aggressively if prompt exceeds max tokens.
-   * Preserve Task and Acceptance Criteria sections.
-   */
-  private trimToMaxTokens(prompt: string, maxTokens: number): string {
-    const currentTokens = estimateTokens(prompt);
-    if (currentTokens <= maxTokens) {
-      return prompt;
-    }
-
-    // Extract sections
-    const sections = this.extractSections(prompt);
-    const targetChars = maxTokens * 4; // Reverse of token estimation
-    const trimmedMessage = "\n... (context trimmed)";
-
-    // Preserve task and AC, trim context
-    let result = "";
-    let remainingChars = targetChars;
-
-    // Add task section (always preserve)
-    if (sections.task) {
-      result += sections.task;
-      remainingChars -= sections.task.length;
-    }
-
-    // Add AC section (always preserve)
-    if (sections.acceptanceCriteria) {
-      result += sections.acceptanceCriteria;
-      remainingChars -= sections.acceptanceCriteria.length;
-    }
-
-    // Add as much context as fits
-    if (sections.context && remainingChars > 0) {
-      // Reserve space for the trimmed message if we're going to add it
-      const reserveForMessage = sections.context.length > remainingChars ? trimmedMessage.length : 0;
-      const maxContextChars = Math.max(0, remainingChars - reserveForMessage);
-      const trimmedContext = sections.context.substring(0, maxContextChars);
-      result += trimmedContext;
-      if (trimmedContext.length < sections.context.length) {
-        result += trimmedMessage;
-      }
-    }
-
-    // Add other sections if there's room
-    if (sections.other && remainingChars > sections.other.length) {
-      result += sections.other;
-    }
-
-    return result;
-  }
-
-  /**
-   * Extract common prompt sections for targeted trimming.
-   */
-  private extractSections(prompt: string): {
-    task?: string;
-    context?: string;
-    acceptanceCriteria?: string;
-    other?: string;
-  } {
-    const sections: {
-      task?: string;
-      context?: string;
-      acceptanceCriteria?: string;
-      other?: string;
-    } = {};
-
-    const taskMatch = prompt.match(/# Task\n([\s\S]*?)(?=\n#|$)/i);
-    if (taskMatch) {
-      sections.task = taskMatch[0];
-    }
-
-    const contextMatch = prompt.match(/# Context\n([\s\S]*?)(?=\n#|$)/i);
-    if (contextMatch) {
-      sections.context = contextMatch[0];
-    }
-
-    const acMatch = prompt.match(/# Acceptance Criteria\n([\s\S]*?)(?=\n#|$)/i);
-    if (acMatch) {
-      sections.acceptanceCriteria = acMatch[0];
-    }
-
-    // Collect everything else
-    let other = prompt;
-    if (sections.task) {
-      other = other.replace(sections.task, "");
-    }
-    if (sections.context) {
-      other = other.replace(sections.context, "");
-    }
-    if (sections.acceptanceCriteria) {
-      other = other.replace(sections.acceptanceCriteria, "");
-    }
-    if (other.trim()) {
-      sections.other = other;
-    }
-
-    return sections;
-  }
-}

package/src/optimizer/types.ts

@@ -1,53 +0,0 @@
-import type { NaxConfig } from "../config/schema.js";
-import type { UserStory } from "../prd/types.js";
-
-/**
- * Interface for prompt optimizers.
- *
- * Optimizers transform assembled prompts to reduce token usage while
- * preserving semantic meaning and all technical requirements.
- */
-export interface IPromptOptimizer {
-  /** Unique optimizer name */
-  name: string;
-
-  /**
-   * Optimize a prompt before it is sent to the coding agent.
-   *
-   * Implementations MUST preserve all technical requirements,
-   * acceptance criteria semantics, and code references.
-   */
-  optimize(input: PromptOptimizerInput): Promise<PromptOptimizerResult>;
-}
-
-export interface PromptOptimizerInput {
-  /** Assembled prompt from promptStage */
-  prompt: string;
-  /** Stories being executed (for context) */
-  stories: UserStory[];
-  /** Raw context markdown (pre-assembly, for dedup detection) */
-  contextMarkdown?: string;
-  /** Nax configuration */
-  config: NaxConfig;
-}
-
-export interface PromptOptimizerResult {
-  /** Optimized prompt */
-  prompt: string;
-  /** Estimated token count before optimization */
-  originalTokens: number;
-  /** Estimated token count after optimization */
-  optimizedTokens: number;
-  /** Savings percentage (0-1) */
-  savings: number;
-  /** List of applied optimization rules/passes */
-  appliedRules: string[];
-}
-
-/**
- * Estimate token count using simple heuristic.
- * ~4 chars per token for English text (rough estimate).
- */
-export function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}