@nathapp/nax 0.18.1

package/src/logging/index.ts

@@ -0,0 +1,22 @@
+/**
+ * Logging formatter module
+ *
+ * Provides human-friendly log formatting with multiple verbosity levels
+ */
+
+export {
+  formatLogEntry,
+  formatRunSummary,
+  formatTimestamp,
+  formatDuration,
+  formatCost,
+  type FormattedEntry,
+} from "./formatter.js";
+export {
+  EMOJI,
+  type VerbosityMode,
+  type FormatterOptions,
+  type RunSummary,
+  type StoryStartData,
+  type StageResultData,
+} from "./types.js";

package/src/logging/types.ts

@@ -0,0 +1,93 @@
+/**
+ * Logging formatter types for human-readable output
+ */
+
+import type { LogEntry } from "../logger/types.js";
+
+/**
+ * Verbosity mode for log formatting
+ */
+export type VerbosityMode = "quiet" | "normal" | "verbose" | "json";
+
+/**
+ * Emoji indicators for different log events
+ */
+export const EMOJI = {
+  // Status indicators
+  success: "✓",
+  failure: "✗",
+  warning: "⚠",
+  info: "ℹ",
+  skip: "⊘",
+
+  // Stage/process indicators
+  routing: "🎯",
+  execution: "⚙️",
+  review: "🔍",
+  tdd: "🔄",
+  agent: "🤖",
+  cost: "💰",
+  duration: "⏱️",
+
+  // Story progress
+  storyStart: "▶",
+  storyComplete: "●",
+  retry: "↻",
+} as const;
+
+/**
+ * Run summary statistics
+ */
+export interface RunSummary {
+  /** Total stories in run */
+  total: number;
+  /** Stories that passed */
+  passed: number;
+  /** Stories that failed */
+  failed: number;
+  /** Stories that were skipped */
+  skipped: number;
+  /** Total run duration in milliseconds */
+  durationMs: number;
+  /** Total cost in dollars */
+  totalCost: number;
+  /** Run start timestamp */
+  startedAt: string;
+  /** Run completion timestamp */
+  completedAt?: string;
+}
+
+/**
+ * Story start event data
+ */
+export interface StoryStartData {
+  storyId: string;
+  title: string;
+  complexity?: string;
+  modelTier?: string;
+  attempt?: number;
+}
+
+/**
+ * Stage result event data
+ */
+export interface StageResultData {
+  stage: string;
+  success: boolean;
+  action?: "continue" | "skip" | "fail" | "escalate" | "pause";
+  reason?: string;
+  cost?: number;
+  durationMs?: number;
+}
+
+/**
+ * Formatter options
+ */
+export interface FormatterOptions {
+  /** Verbosity mode */
+  mode: VerbosityMode;
+  /** Whether to use color/emoji (default: true) */
+  useColor?: boolean;
+  /** Timezone for timestamp formatting (default: system timezone) */
+  timezone?: string;
+}

package/src/metrics/aggregator.ts

@@ -0,0 +1,190 @@
+/**
+ * Metrics Aggregator
+ *
+ * Calculates aggregate metrics across all runs.
+ */
+
+import type { AggregateMetrics, RunMetrics, StoryMetrics } from "./types";
+
+/**
+ * Calculate aggregate metrics across all runs.
+ *
+ * Analyzes historical data to compute:
+ * - Overall success rates
+ * - Per-model efficiency
+ * - Complexity prediction accuracy
+ * - Cost statistics
+ *
+ * @param runs - Array of all historical run metrics
+ * @returns Aggregate metrics
+ *
+ * @example
+ * ```ts
+ * const runs = await loadRunMetrics(workdir);
+ * const aggregate = calculateAggregateMetrics(runs);
+ * console.log(`First pass rate: ${(aggregate.firstPassRate * 100).toFixed(1)}%`);
+ * console.log(`Avg cost per story: $${aggregate.avgCostPerStory.toFixed(4)}`);
+ * ```
+ */
+export function calculateAggregateMetrics(runs: RunMetrics[]): AggregateMetrics {
+  if (runs.length === 0) {
+    return {
+      totalRuns: 0,
+      totalCost: 0,
+      totalStories: 0,
+      firstPassRate: 0,
+      escalationRate: 0,
+      avgCostPerStory: 0,
+      avgCostPerFeature: 0,
+      modelEfficiency: {},
+      complexityAccuracy: {},
+    };
+  }
+
+  // Flatten all story metrics
+  const allStories: StoryMetrics[] = runs.flatMap((run) => run.stories);
+
+  const totalRuns = runs.length;
+  const totalCost = runs.reduce((sum, run) => sum + run.totalCost, 0);
+  const totalStories = allStories.length;
+
+  // Calculate first pass rate
+  const firstPassSuccesses = allStories.filter((s) => s.firstPassSuccess).length;
+  const firstPassRate = totalStories > 0 ? firstPassSuccesses / totalStories : 0;
+
+  // Calculate escalation rate (stories that needed more than 1 attempt)
+  const escalatedStories = allStories.filter((s) => s.attempts > 1).length;
+  const escalationRate = totalStories > 0 ? escalatedStories / totalStories : 0;
+
+  // Average costs
+  const avgCostPerStory = totalStories > 0 ? totalCost / totalStories : 0;
+  const avgCostPerFeature = totalRuns > 0 ? totalCost / totalRuns : 0;
+
+  // Per-model efficiency
+  const modelStats = new Map<
+    string,
+    {
+      attempts: number;
+      successes: number;
+      totalCost: number;
+    }
+  >();
+
+  for (const story of allStories) {
+    const modelKey = story.modelUsed;
+    const existing = modelStats.get(modelKey) || {
+      attempts: 0,
+      successes: 0,
+      totalCost: 0,
+    };
+
+    modelStats.set(modelKey, {
+      attempts: existing.attempts + story.attempts,
+      successes: existing.successes + (story.success ? 1 : 0),
+      totalCost: existing.totalCost + story.cost,
+    });
+  }
+
+  const modelEfficiency: AggregateMetrics["modelEfficiency"] = {};
+  for (const [modelKey, stats] of modelStats) {
+    const passRate = stats.attempts > 0 ? stats.successes / stats.attempts : 0;
+    const avgCost = stats.successes > 0 ? stats.totalCost / stats.successes : 0;
+
+    modelEfficiency[modelKey] = {
+      attempts: stats.attempts,
+      successes: stats.successes,
+      passRate,
+      avgCost,
+      totalCost: stats.totalCost,
+    };
+  }
+
+  // Complexity prediction accuracy
+  const complexityStats = new Map<
+    string,
+    {
+      predicted: number;
+      tierCounts: Map<string, number>;
+      mismatches: number;
+    }
+  >();
+
+  for (const story of allStories) {
+    const complexity = story.complexity;
+    const existing = complexityStats.get(complexity) || {
+      predicted: 0,
+      tierCounts: new Map<string, number>(),
+      mismatches: 0,
+    };
+
+    existing.predicted += 1;
+
+    // Track which tier was actually used
+    const finalTier = story.finalTier;
+    existing.tierCounts.set(finalTier, (existing.tierCounts.get(finalTier) || 0) + 1);
+
+    // Count as mismatch if escalated (initial tier != final tier)
+    if (story.modelTier !== story.finalTier) {
+      existing.mismatches += 1;
+    }
+
+    complexityStats.set(complexity, existing);
+  }
+
+  const complexityAccuracy: AggregateMetrics["complexityAccuracy"] = {};
+  for (const [complexity, stats] of complexityStats) {
+    // Find most common final tier
+    let maxCount = 0;
+    let mostCommonTier = "unknown";
+    for (const [tier, count] of stats.tierCounts) {
+      if (count > maxCount) {
+        maxCount = count;
+        mostCommonTier = tier;
+      }
+    }
+
+    const mismatchRate = stats.predicted > 0 ? stats.mismatches / stats.predicted : 0;
+
+    complexityAccuracy[complexity] = {
+      predicted: stats.predicted,
+      actualTierUsed: mostCommonTier,
+      mismatchRate,
+    };
+  }
+
+  return {
+    totalRuns,
+    totalCost,
+    totalStories,
+    firstPassRate,
+    escalationRate,
+    avgCostPerStory,
+    avgCostPerFeature,
+    modelEfficiency,
+    complexityAccuracy,
+  };
+}
+
+/**
+ * Get the last run metrics from the list.
+ *
+ * @param runs - Array of all run metrics
+ * @returns Most recent run, or null if no runs
+ *
+ * @example
+ * ```ts
+ * const runs = await loadRunMetrics(workdir);
+ * const lastRun = getLastRun(runs);
+ * if (lastRun) {
+ *   console.log(`Last run: ${lastRun.feature} (${lastRun.storiesCompleted}/${lastRun.totalStories} stories)`);
+ * }
+ * ```
+ */
+export function getLastRun(runs: RunMetrics[]): RunMetrics | null {
+  if (runs.length === 0) {
+    return null;
+  }
+
+  // Runs are appended chronologically, so last element is most recent
+  return runs[runs.length - 1];
+}

package/src/metrics/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Metrics Tracking
+ *
+ * Per-story and per-run cost tracking for data-driven routing optimization.
+ */
+
+export type { StoryMetrics, RunMetrics, AggregateMetrics } from "./types";
+export {
+  collectStoryMetrics,
+  collectBatchMetrics,
+  saveRunMetrics,
+  loadRunMetrics,
+} from "./tracker";
+export { calculateAggregateMetrics, getLastRun } from "./aggregator";

package/src/metrics/tracker.ts

@@ -0,0 +1,200 @@
+/**
+ * Metrics Tracker
+ *
+ * Collects and persists per-story and per-run metrics.
+ */
+
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { resolveModel } from "../config/schema";
+import { getLogger } from "../logger";
+import type { PipelineContext } from "../pipeline/types";
+import type { RunMetrics, StoryMetrics } from "./types";
+
+/**
+ * Collect metrics for a single story execution.
+ *
+ * Extracts timing, cost, model usage, and escalation data from the pipeline context.
+ *
+ * @param ctx - Pipeline context with execution results
+ * @param storyStartTime - Story start timestamp (ISO string)
+ * @returns Story metrics object
+ *
+ * @example
+ * ```ts
+ * const metrics = collectStoryMetrics(ctx, "2026-02-17T10:30:00.000Z");
+ * // {
+ * //   storyId: "US-001",
+ * //   complexity: "medium",
+ * //   modelTier: "balanced",
+ * //   modelUsed: "claude-sonnet-4.5",
+ * //   attempts: 1,
+ * //   finalTier: "balanced",
+ * //   success: true,
+ * //   cost: 0.0234,
+ * //   durationMs: 45000,
+ * //   firstPassSuccess: true,
+ * //   ...
+ * // }
+ * ```
+ */
+export function collectStoryMetrics(ctx: PipelineContext, storyStartTime: string): StoryMetrics {
+  const story = ctx.story;
+  const routing = ctx.routing;
+  const agentResult = ctx.agentResult;
+
+  // Calculate attempts (initial + escalations)
+  const escalationCount = story.escalations?.length || 0;
+  const attempts = Math.max(1, story.attempts || 1);
+
+  // Determine final tier (from last escalation or initial routing)
+  const finalTier = escalationCount > 0 ? story.escalations[escalationCount - 1].toTier : routing.modelTier;
+
+  // First pass success = succeeded with no escalations
+  const firstPassSuccess = agentResult?.success === true && escalationCount === 0;
+
+  // Extract model name from config
+  const modelEntry = ctx.config.models[routing.modelTier];
+  const modelDef = modelEntry ? resolveModel(modelEntry) : null;
+  const modelUsed = modelDef?.model || routing.modelTier;
+
+  return {
+    storyId: story.id,
+    complexity: routing.complexity,
+    modelTier: routing.modelTier,
+    modelUsed,
+    attempts,
+    finalTier,
+    success: agentResult?.success || false,
+    cost: agentResult?.estimatedCost || 0,
+    durationMs: agentResult?.durationMs || 0,
+    firstPassSuccess,
+    startedAt: storyStartTime,
+    completedAt: new Date().toISOString(),
+  };
+}
+
+/**
+ * Collect metrics for a batch of stories.
+ *
+ * Creates individual story metrics for each story in the batch,
+ * distributing the total cost and duration proportionally.
+ *
+ * @param ctx - Pipeline context with batch execution results
+ * @param storyStartTime - Batch start timestamp (ISO string)
+ * @returns Array of story metrics (one per story in batch)
+ *
+ * @example
+ * ```ts
+ * const batchMetrics = collectBatchMetrics(ctx, "2026-02-17T10:30:00.000Z");
+ * // [
+ * //   { storyId: "US-001", cost: 0.0078, ... },
+ * //   { storyId: "US-002", cost: 0.0078, ... },
+ * //   { storyId: "US-003", cost: 0.0078, ... },
+ * // ]
+ * ```
+ */
+export function collectBatchMetrics(ctx: PipelineContext, storyStartTime: string): StoryMetrics[] {
+  const stories = ctx.stories;
+  const routing = ctx.routing;
+  const agentResult = ctx.agentResult;
+
+  const totalCost = agentResult?.estimatedCost || 0;
+  const totalDuration = agentResult?.durationMs || 0;
+  const costPerStory = totalCost / stories.length;
+  const durationPerStory = totalDuration / stories.length;
+
+  const modelEntry = ctx.config.models[routing.modelTier];
+  const modelDef = modelEntry ? resolveModel(modelEntry) : null;
+  const modelUsed = modelDef?.model || routing.modelTier;
+
+  return stories.map((story) => ({
+    storyId: story.id,
+    complexity: routing.complexity,
+    modelTier: routing.modelTier,
+    modelUsed,
+    attempts: 1, // batch stories don't escalate individually
+    finalTier: routing.modelTier,
+    success: true, // if batch succeeded, all stories succeeded
+    cost: costPerStory,
+    durationMs: durationPerStory,
+    firstPassSuccess: true, // batch = first pass success
+    startedAt: storyStartTime,
+    completedAt: new Date().toISOString(),
+  }));
+}
+
+/**
+ * Save run metrics to nax/metrics.json.
+ *
+ * Appends the run metrics to the existing metrics file (or creates it if missing).
+ * Each run is a separate entry in the JSON array.
+ *
+ * @param workdir - Project root directory
+ * @param runMetrics - Run metrics to persist
+ *
+ * @example
+ * ```ts
+ * await saveRunMetrics("/home/user/project", {
+ *   runId: "run-20260217-103045",
+ *   feature: "auth-system",
+ *   totalCost: 0.1234,
+ *   stories: [...],
+ *   ...
+ * });
+ * ```
+ */
+export async function saveRunMetrics(workdir: string, runMetrics: RunMetrics): Promise<void> {
+  const metricsPath = path.join(workdir, "nax", "metrics.json");
+
+  let allMetrics: RunMetrics[] = [];
+
+  // Load existing metrics if file exists
+  const logger = getLogger();
+  if (existsSync(metricsPath)) {
+    try {
+      const file = Bun.file(metricsPath);
+      const content = await file.json();
+      allMetrics = Array.isArray(content) ? content : [];
+    } catch (err) {
+      logger.warn("metrics", "Could not parse metrics file, starting fresh", { metricsPath });
+      allMetrics = [];
+    }
+  }
+
+  // Append new run
+  allMetrics.push(runMetrics);
+
+  // Write back
+  await Bun.write(metricsPath, JSON.stringify(allMetrics, null, 2));
+}
+
+/**
+ * Load all run metrics from nax/metrics.json.
+ *
+ * @param workdir - Project root directory
+ * @returns Array of run metrics, or empty array if file doesn't exist
+ *
+ * @example
+ * ```ts
+ * const runs = await loadRunMetrics("/home/user/project");
+ * console.log(`Total runs: ${runs.length}`);
+ * ```
+ */
+export async function loadRunMetrics(workdir: string): Promise<RunMetrics[]> {
+  const metricsPath = path.join(workdir, "nax", "metrics.json");
+
+  if (!existsSync(metricsPath)) {
+    return [];
+  }
+
+  try {
+    const file = Bun.file(metricsPath);
+    const content = await file.json();
+    return Array.isArray(content) ? content : [];
+  } catch (err) {
+    const logger = getLogger();
+    logger.warn("metrics", "Could not parse metrics file", { metricsPath });
+    return [];
+  }
+}

package/src/metrics/types.ts

@@ -0,0 +1,109 @@
+/**
+ * Metrics Tracking Types
+ *
+ * Structured cost and performance metrics for tracking agent execution.
+ */
+
+/**
+ * Per-story execution metrics
+ */
+export interface StoryMetrics {
+  /** Story ID */
+  storyId: string;
+  /** Classified complexity */
+  complexity: string;
+  /** Initial model tier */
+  modelTier: string;
+  /** Actual model used (e.g., "claude-sonnet-4.5") */
+  modelUsed: string;
+  /** Number of attempts (includes escalations) */
+  attempts: number;
+  /** Final tier that succeeded */
+  finalTier: string;
+  /** Whether the story succeeded */
+  success: boolean;
+  /** Total cost for this story (including all attempts) */
+  cost: number;
+  /** Total duration in milliseconds */
+  durationMs: number;
+  /** Whether it passed on the first attempt */
+  firstPassSuccess: boolean;
+  /** Timestamp when started */
+  startedAt: string;
+  /** Timestamp when completed */
+  completedAt: string;
+}
+
+/**
+ * Per-run execution metrics
+ */
+export interface RunMetrics {
+  /** Unique run ID */
+  runId: string;
+  /** Feature name */
+  feature: string;
+  /** Run start timestamp */
+  startedAt: string;
+  /** Run completion timestamp */
+  completedAt: string;
+  /** Total cost for the run */
+  totalCost: number;
+  /** Total number of stories in the run */
+  totalStories: number;
+  /** Number of stories completed successfully */
+  storiesCompleted: number;
+  /** Number of stories that failed */
+  storiesFailed: number;
+  /** Total duration in milliseconds */
+  totalDurationMs: number;
+  /** Per-story metrics */
+  stories: StoryMetrics[];
+}
+
+/**
+ * Aggregate metrics across all runs
+ */
+export interface AggregateMetrics {
+  /** Total number of runs */
+  totalRuns: number;
+  /** Total cost across all runs */
+  totalCost: number;
+  /** Total stories across all runs */
+  totalStories: number;
+  /** Percentage of stories passing on first attempt */
+  firstPassRate: number;
+  /** Percentage of stories needing escalation */
+  escalationRate: number;
+  /** Average cost per story */
+  avgCostPerStory: number;
+  /** Average cost per feature run */
+  avgCostPerFeature: number;
+  /** Per-model efficiency metrics */
+  modelEfficiency: Record<
+    string,
+    {
+      /** Total attempts with this model */
+      attempts: number;
+      /** Successful attempts */
+      successes: number;
+      /** Success rate (0-1) */
+      passRate: number;
+      /** Average cost per story */
+      avgCost: number;
+      /** Total cost for this model */
+      totalCost: number;
+    }
+  >;
+  /** Complexity prediction accuracy */
+  complexityAccuracy: Record<
+    string,
+    {
+      /** Number of stories predicted at this complexity */
+      predicted: number;
+      /** Most common final tier used */
+      actualTierUsed: string;
+      /** Rate at which prediction didn't match actual tier needed */
+      mismatchRate: number;
+    }
+  >;
+}

package/src/optimizer/index.ts

@@ -0,0 +1,62 @@
+/**
+ * Prompt Optimizer
+ *
+ * Exports optimizer types, implementations, and factory function.
+ */
+
+export type {
+  IPromptOptimizer,
+  PromptOptimizerInput,
+  PromptOptimizerResult,
+} from "./types.js";
+export { estimateTokens } from "./types.js";
+export { NoopOptimizer } from "./noop.optimizer.js";
+export { RuleBasedOptimizer } from "./rule-based.optimizer.js";
+
+import type { NaxConfig } from "../config/schema.js";
+import type { PluginRegistry } from "../plugins/registry.js";
+import { NoopOptimizer } from "./noop.optimizer.js";
+import { RuleBasedOptimizer } from "./rule-based.optimizer.js";
+import type { IPromptOptimizer } from "./types.js";
+
+/**
+ * Resolve the prompt optimizer to use for this run.
+ *
+ * Resolution order:
+ * 1. Plugin-provided optimizer (if any plugins provide "optimizer")
+ * 2. Built-in strategy from config (rule-based, noop)
+ * 3. Fallback to NoopOptimizer
+ *
+ * @param config - Nax configuration
+ * @param pluginRegistry - Plugin registry (optional, for plugin-provided optimizers)
+ * @returns Resolved optimizer instance
+ */
+export function resolveOptimizer(config: NaxConfig, pluginRegistry?: PluginRegistry): IPromptOptimizer {
+  // Check if optimizer is disabled
+  if (!config.optimizer?.enabled) {
+    return new NoopOptimizer();
+  }
+
+  // 1. Check plugin registry first
+  if (pluginRegistry) {
+    const pluginOptimizers = pluginRegistry.getOptimizers();
+    if (pluginOptimizers.length > 0) {
+      // Use first plugin optimizer (plugin optimizers use the same interface)
+      return pluginOptimizers[0];
+    }
+  }
+
+  // 2. Use built-in strategy from config
+  const strategy = config.optimizer.strategy ?? "noop";
+
+  switch (strategy) {
+    case "rule-based":
+      return new RuleBasedOptimizer();
+    case "noop":
+      return new NoopOptimizer();
+    default:
+      // Unknown strategy, fallback to noop
+      console.warn(`[nax] Unknown optimizer strategy '${strategy}', using noop`);
+      return new NoopOptimizer();
+  }
+}