@bhargavvc/sdd-cc 1.30.0 → 1.35.0

package/sdd/bin/sdd-tools.cjs

@@ -70,6 +70,16 @@
  *   audit-uat                           Scan all phases for unresolved UAT/verification items
  *   uat render-checkpoint --file <path> Render the current UAT checkpoint block
  *
+ * Intel:
+ *   intel query <term>             Query intel files for a term
+ *   intel status                   Show intel file freshness
+ *   intel update                   Trigger intel refresh (returns agent spawn hint)
+ *   intel diff                     Show changed intel entries since last snapshot
+ *   intel snapshot                 Save current intel state as diff baseline
+ *   intel patch-meta <file>        Update _meta.updated_at in an intel file
+ *   intel validate                 Validate intel file structure
+ *   intel extract-exports <file>   Extract exported symbols from a source file
+ *
  * Scaffolding:
  *   scaffold context --phase <N>       Create CONTEXT.md template
  *   scaffold uat --phase <N>           Create UAT.md template
@@ -93,6 +103,7 @@
  *   verify commits <h1> [h2] ...      Batch verify commit hashes
  *   verify artifacts <plan-file>       Check must_haves.artifacts
  *   verify key-links <plan-file>       Check must_haves.key_links
+ *   verify schema-drift <phase> [--skip]  Detect schema file changes without push
  *
  * Template Fill:
  *   template fill summary --phase N    Create pre-filled SUMMARY.md
@@ -133,6 +144,20 @@
  *   init milestone-op                  All context for milestone operations
  *   init map-codebase                  All context for map-codebase workflow
  *   init progress                      All context for progress workflow
+ *
+ * Documentation:
+ *   docs-init                            Project context for docs-update workflow
+ *
+ * Learnings:
+ *   learnings list                       List all global learnings (JSON)
+ *   learnings query --tag <tag>          Query learnings by tag
+ *   learnings copy                       Copy from current project's LEARNINGS.md
+ *   learnings prune --older-than <dur>   Remove entries older than duration (e.g. 90d)
+ *   learnings delete <id>                Delete a learning by ID
+ *
+ * SDD-2 Migration:
+ *   from-sdd2 [--path <dir>] [--force] [--dry-run]
+ *             Import a SDD-2 (.sdd/) project back to SDD v1 (.planning/) format
  */
 
 const fs = require('fs');
@@ -152,6 +177,8 @@ const frontmatter = require('./lib/frontmatter.cjs');
 const profilePipeline = require('./lib/profile-pipeline.cjs');
 const profileOutput = require('./lib/profile-output.cjs');
 const workstream = require('./lib/workstream.cjs');
+const docs = require('./lib/docs.cjs');
+const learnings = require('./lib/learnings.cjs');
 
 // ─── Arg parsing helpers ──────────────────────────────────────────────────────
 
@@ -230,7 +257,7 @@ async function main() {
   }
 
   // Optional workstream override for parallel milestone work.
-  // Priority: --ws flag > SDD_WORKSTREAM env var > active-workstream file > null (flat mode)
+  // Priority: --ws flag > SDD_WORKSTREAM env var > session-scoped pointer > shared legacy pointer > null
   const wsEqArg = args.find(arg => arg.startsWith('--ws='));
   const wsIdx = args.indexOf('--ws');
   let ws = null;
@@ -271,10 +298,31 @@ async function main() {
     args.splice(pickIdx, 2);
   }
 
+  // --default <value>: for config-get, return this value instead of erroring
+  // when the key is absent. Allows workflows to express optional config reads
+  // without defensive `2>/dev/null || true` boilerplate (#1893).
+  const defaultIdx = args.indexOf('--default');
+  let defaultValue = undefined;
+  if (defaultIdx !== -1) {
+    defaultValue = args[defaultIdx + 1];
+    if (defaultValue === undefined) defaultValue = '';
+    args.splice(defaultIdx, 2);
+  }
+
   const command = args[0];
 
   if (!command) {
-    error('Usage: sdd-tools <command> [args] [--raw] [--pick <field>] [--cwd <path>] [--ws <name>]\nCommands: state, resolve-model, find-phase, commit, verify-summary, verify, frontmatter, template, generate-slug, current-timestamp, list-todos, verify-path-exists, config-ensure-section, config-new-project, init, workstream');
+    error('Usage: sdd-tools <command> [args] [--raw] [--pick <field>] [--cwd <path>] [--ws <name>]\nCommands: state, resolve-model, find-phase, commit, verify-summary, verify, frontmatter, template, generate-slug, current-timestamp, list-todos, verify-path-exists, config-ensure-section, config-new-project, init, workstream, docs-init');
+  }
+
+  // Reject flags that are never valid for any sdd-tools command. AI agents
+  // sometimes hallucinate --help or --version on tool invocations; silently
+  // ignoring them can cause destructive operations to proceed unchecked.
+  const NEVER_VALID_FLAGS = new Set(['-h', '--help', '-?', '--h', '--version', '-v', '--usage']);
+  for (const arg of args) {
+    if (NEVER_VALID_FLAGS.has(arg)) {
+      error(`Unknown flag: ${arg}\nsdd-tools does not accept help or version flags. Run "sdd-tools" with no arguments for usage.`);
+    }
   }
 
   // Multi-repo guard: resolve project root for commands that read/write .planning/.
@@ -313,7 +361,7 @@ async function main() {
       }
     };
     try {
-      await runCommand(command, args, cwd, raw);
+      await runCommand(command, args, cwd, raw, defaultValue);
       cleanup();
     } catch (e) {
       fs.writeSync = origWriteSync;
@@ -322,7 +370,27 @@ async function main() {
     return;
   }
 
-  await runCommand(command, args, cwd, raw);
+  // Intercept stdout to transparently resolve @file: references (#1891).
+  // core.cjs output() writes @file:<path> when JSON > 50KB. The --pick path
+  // already resolves this, but the normal path wrote @file: to stdout, forcing
+  // every workflow to have a bash-specific `if [[ "$INIT" == @file:* ]]` check
+  // that breaks on PowerShell and other non-bash shells.
+  const origWriteSync2 = fs.writeSync;
+  const outChunks = [];
+  fs.writeSync = function (fd, data, ...rest) {
+    if (fd === 1) { outChunks.push(String(data)); return; }
+    return origWriteSync2.call(fs, fd, data, ...rest);
+  };
+  try {
+    await runCommand(command, args, cwd, raw, defaultValue);
+  } finally {
+    fs.writeSync = origWriteSync2;
+  }
+  let captured = outChunks.join('');
+  if (captured.startsWith('@file:')) {
+    captured = fs.readFileSync(captured.slice(6), 'utf-8');
+  }
+  origWriteSync2.call(fs, 1, captured);
 }
 
 /**
@@ -348,7 +416,7 @@ function extractField(obj, fieldPath) {
   return current;
 }
 
-async function runCommand(command, args, cwd, raw) {
+async function runCommand(command, args, cwd, raw, defaultValue) {
   switch (command) {
     case 'state': {
       const subcommand = args[1];
@@ -394,6 +462,14 @@ async function runCommand(command, args, cwd, raw) {
         state.cmdSignalWaiting(cwd, type, question, options, p, raw);
       } else if (subcommand === 'signal-resume') {
         state.cmdSignalResume(cwd, raw);
+      } else if (subcommand === 'planned-phase') {
+        const { phase: p, name, plans } = parseNamedArgs(args, ['phase', 'name', 'plans']);
+        state.cmdStatePlannedPhase(cwd, p, plans !== null ? parseInt(plans, 10) : null, raw);
+      } else if (subcommand === 'validate') {
+        state.cmdStateValidate(cwd, raw);
+      } else if (subcommand === 'sync') {
+        const { verify } = parseNamedArgs(args, [], ['verify']);
+        state.cmdStateSync(cwd, { verify }, raw);
       } else {
         state.cmdStateLoad(cwd, raw);
       }
@@ -425,6 +501,11 @@ async function runCommand(command, args, cwd, raw) {
       break;
     }
 
+    case 'check-commit': {
+      commands.cmdCheckCommit(cwd, raw);
+      break;
+    }
+
     case 'commit-to-subrepo': {
       const message = args[1];
       const filesIndex = args.indexOf('--files');
@@ -498,8 +579,11 @@ async function runCommand(command, args, cwd, raw) {
         verify.cmdVerifyArtifacts(cwd, args[2], raw);
       } else if (subcommand === 'key-links') {
         verify.cmdVerifyKeyLinks(cwd, args[2], raw);
+      } else if (subcommand === 'schema-drift') {
+        const skipFlag = args.includes('--skip');
+        verify.cmdVerifySchemaDrift(cwd, args[2], skipFlag, raw);
       } else {
-        error('Unknown verify subcommand. Available: plan-structure, phase-completeness, references, commits, artifacts, key-links');
+        error('Unknown verify subcommand. Available: plan-structure, phase-completeness, references, commits, artifacts, key-links, schema-drift');
       }
       break;
     }
@@ -540,7 +624,7 @@ async function runCommand(command, args, cwd, raw) {
     }
 
     case 'config-get': {
-      config.cmdConfigGet(cwd, args[1], raw);
+      config.cmdConfigGet(cwd, args[1], raw, defaultValue);
       break;
     }
 
@@ -570,8 +654,10 @@ async function runCommand(command, args, cwd, raw) {
           includeArchived: args.includes('--include-archived'),
         };
         phase.cmdPhasesList(cwd, options, raw);
+      } else if (subcommand === 'clear') {
+        milestone.cmdPhasesClear(cwd, raw, args.slice(2));
       } else {
-        error('Unknown phases subcommand. Available: list');
+        error('Unknown phases subcommand. Available: list, clear');
       }
       break;
     }
@@ -712,12 +798,16 @@ async function runCommand(command, args, cwd, raw) {
     case 'init': {
       const workflow = args[1];
       switch (workflow) {
-        case 'execute-phase':
-          init.cmdInitExecutePhase(cwd, args[2], raw);
+        case 'execute-phase': {
+          const { validate: epValidate } = parseNamedArgs(args, [], ['validate']);
+          init.cmdInitExecutePhase(cwd, args[2], raw, { validate: epValidate });
           break;
-        case 'plan-phase':
-          init.cmdInitPlanPhase(cwd, args[2], raw);
+        }
+        case 'plan-phase': {
+          const { validate: ppValidate } = parseNamedArgs(args, [], ['validate']);
+          init.cmdInitPlanPhase(cwd, args[2], raw, { validate: ppValidate });
           break;
+        }
         case 'new-project':
           init.cmdInitNewProject(cwd, raw);
           break;
@@ -910,6 +1000,88 @@ async function runCommand(command, args, cwd, raw) {
       break;
     }
 
+    // ─── Intel ────────────────────────────────────────────────────────────
+
+    case 'intel': {
+      const intel = require('./lib/intel.cjs');
+      const subcommand = args[1];
+      if (subcommand === 'query') {
+        const term = args[2];
+        if (!term) error('Usage: sdd-tools intel query <term>');
+        const planningDir = path.join(cwd, '.planning');
+        core.output(intel.intelQuery(term, planningDir), raw);
+      } else if (subcommand === 'status') {
+        const planningDir = path.join(cwd, '.planning');
+        core.output(intel.intelStatus(planningDir), raw);
+      } else if (subcommand === 'diff') {
+        const planningDir = path.join(cwd, '.planning');
+        core.output(intel.intelDiff(planningDir), raw);
+      } else if (subcommand === 'snapshot') {
+        const planningDir = path.join(cwd, '.planning');
+        core.output(intel.intelSnapshot(planningDir), raw);
+      } else if (subcommand === 'patch-meta') {
+        const filePath = args[2];
+        if (!filePath) error('Usage: sdd-tools intel patch-meta <file-path>');
+        core.output(intel.intelPatchMeta(path.resolve(cwd, filePath)), raw);
+      } else if (subcommand === 'validate') {
+        const planningDir = path.join(cwd, '.planning');
+        core.output(intel.intelValidate(planningDir), raw);
+      } else if (subcommand === 'extract-exports') {
+        const filePath = args[2];
+        if (!filePath) error('Usage: sdd-tools intel extract-exports <file-path>');
+        core.output(intel.intelExtractExports(path.resolve(cwd, filePath)), raw);
+      } else if (subcommand === 'update') {
+        const planningDir = path.join(cwd, '.planning');
+        core.output(intel.intelUpdate(planningDir), raw);
+      } else {
+        error('Unknown intel subcommand. Available: query, status, update, diff, snapshot, patch-meta, validate, extract-exports');
+      }
+      break;
+    }
+
+    // ─── Documentation ────────────────────────────────────────────────────
+
+    case 'docs-init': {
+      docs.cmdDocsInit(cwd, raw);
+      break;
+    }
+
+    // ─── Learnings ─────────────────────────────────────────────────────────
+
+    case 'learnings': {
+      const subcommand = args[1];
+      if (subcommand === 'list') {
+        learnings.cmdLearningsList(raw);
+      } else if (subcommand === 'query') {
+        const tagIdx = args.indexOf('--tag');
+        const tag = tagIdx !== -1 ? args[tagIdx + 1] : null;
+        if (!tag) error('Usage: sdd-tools learnings query --tag <tag>');
+        learnings.cmdLearningsQuery(tag, raw);
+      } else if (subcommand === 'copy') {
+        learnings.cmdLearningsCopy(cwd, raw);
+      } else if (subcommand === 'prune') {
+        const olderIdx = args.indexOf('--older-than');
+        const olderThan = olderIdx !== -1 ? args[olderIdx + 1] : null;
+        if (!olderThan) error('Usage: sdd-tools learnings prune --older-than <duration>');
+        learnings.cmdLearningsPrune(olderThan, raw);
+      } else if (subcommand === 'delete') {
+        const id = args[2];
+        if (!id) error('Usage: sdd-tools learnings delete <id>');
+        learnings.cmdLearningsDelete(id, raw);
+      } else {
+        error('Unknown learnings subcommand. Available: list, query, copy, prune, delete');
+      }
+      break;
+    }
+
+    // ─── SDD-2 Reverse Migration ───────────────────────────────────────────
+
+    case 'from-sdd2': {
+      const sdd2Import = require('./lib/sdd2-import.cjs');
+      sdd2Import.cmdFromSdd2(args.slice(1), cwd, raw);
+      break;
+    }
+
     default:
       error(`Unknown command: ${command}`);
   }

package/sdd/contexts/dev.md

@@ -0,0 +1,21 @@
+# Dev Context Profile
+
+Agent output guidance for dev mode. Loaded when `context: dev` is set in config.json.
+
+## Output Style
+
+- Concise, action-oriented responses
+- Lead with the code change or command, follow with brief rationale
+- Skip preamble — assume the developer has full context
+- Use inline code references (`file:line`) over prose descriptions
+
+## Focus Areas
+
+- Working code that compiles and passes tests
+- Minimal diff — change only what is necessary
+- Flag side effects or breaking changes immediately
+- Surface the next actionable step at the end of every response
+
+## Verbosity
+
+Low. One-liner explanations unless the change is non-obvious. Omit background theory, alternative approaches, and caveats that do not affect the current task.

package/sdd/contexts/research.md

@@ -0,0 +1,22 @@
+# Research Context Profile
+
+Agent output guidance for research mode. Loaded when `context: research` is set in config.json.
+
+## Output Style
+
+- Verbose, exploratory responses that surface trade-offs and alternatives
+- Present multiple approaches with pros and cons before recommending one
+- Include links, references, and citations where available
+- Use structured headings and bullet lists for scan-ability
+
+## Focus Areas
+
+- Breadth of options — enumerate before narrowing
+- Prior art and ecosystem conventions
+- Risks, edge cases, and failure modes
+- Dependencies and compatibility implications
+- Long-term maintainability of each approach
+
+## Verbosity
+
+High. Explain reasoning, show evidence, and document assumptions. Include background context even if the developer likely knows it — research artifacts are read by future contributors who may not.

package/sdd/contexts/review.md

@@ -0,0 +1,22 @@
+# Review Context Profile
+
+Agent output guidance for review mode. Loaded when `context: review` is set in config.json.
+
+## Output Style
+
+- Critical, detail-focused responses that prioritize correctness
+- Organize findings by severity: blocking, important, nit
+- Reference specific lines and files for every finding
+- State what is correct as well as what needs change — confirm the good parts
+
+## Focus Areas
+
+- Correctness — logic errors, off-by-ones, missing edge cases
+- Security — input validation, injection vectors, secret exposure
+- Performance — unnecessary allocations, O(n^2) patterns, missing caching
+- Style and consistency — naming, formatting, import order
+- Test coverage — untested branches, missing assertions, flaky patterns
+
+## Verbosity
+
+Medium. Be thorough on findings but terse in explanation. Each issue should be one to three sentences: what is wrong, why it matters, and how to fix it.

package/sdd/references/agent-contracts.md

@@ -0,0 +1,79 @@
+# Agent Contracts
+
+Completion markers and handoff schemas for all SDD agents. Workflows use these markers to detect agent completion and route accordingly.
+
+This doc describes what IS, not what should be. Casing inconsistencies are documented as they appear in agent source files.
+
+---
+
+## Agent Registry
+
+| Agent | Role | Completion Markers |
+|-------|------|--------------------|
+| sdd-planner | Plan creation | `## PLANNING COMPLETE` |
+| sdd-executor | Plan execution | `## PLAN COMPLETE`, `## CHECKPOINT REACHED` |
+| sdd-phase-researcher | Phase-scoped research | `## RESEARCH COMPLETE`, `## RESEARCH BLOCKED` |
+| sdd-project-researcher | Project-wide research | `## RESEARCH COMPLETE`, `## RESEARCH BLOCKED` |
+| sdd-plan-checker | Plan validation | `## VERIFICATION PASSED`, `## ISSUES FOUND` |
+| sdd-research-synthesizer | Multi-research synthesis | `## SYNTHESIS COMPLETE`, `## SYNTHESIS BLOCKED` |
+| sdd-debugger | Debug investigation | `## DEBUG COMPLETE`, `## ROOT CAUSE FOUND`, `## CHECKPOINT REACHED` |
+| sdd-roadmapper | Roadmap creation/revision | `## ROADMAP CREATED`, `## ROADMAP REVISED`, `## ROADMAP BLOCKED` |
+| sdd-ui-auditor | UI review | `## UI REVIEW COMPLETE` |
+| sdd-ui-checker | UI validation | `## ISSUES FOUND` |
+| sdd-ui-researcher | UI spec creation | `## UI-SPEC COMPLETE`, `## UI-SPEC BLOCKED` |
+| sdd-verifier | Post-execution verification | `## Verification Complete` (title case) |
+| sdd-integration-checker | Cross-phase integration check | `## Integration Check Complete` (title case) |
+| sdd-nyquist-auditor | Sampling audit | `## PARTIAL`, `## ESCALATE` (non-standard) |
+| sdd-security-auditor | Security audit | `## OPEN_THREATS`, `## ESCALATE` (non-standard) |
+| sdd-codebase-mapper | Codebase analysis | No marker (writes docs directly) |
+| sdd-assumptions-analyzer | Assumption extraction | No marker (returns `## Assumptions` sections) |
+| sdd-doc-verifier | Doc validation | No marker (writes JSON to `.planning/tmp/`) |
+| sdd-doc-writer | Doc generation | No marker (writes docs directly) |
+| sdd-advisor-researcher | Advisory research | No marker (utility agent) |
+| sdd-user-profiler | User profiling | No marker (returns JSON in analysis tags) |
+| sdd-intel-updater | Codebase intelligence analysis | `## INTEL UPDATE COMPLETE`, `## INTEL UPDATE FAILED` |
+
+## Marker Rules
+
+1. **ALL-CAPS markers** (e.g., `## PLANNING COMPLETE`) are the standard convention
+2. **Title-case markers** (e.g., `## Verification Complete`) exist in sdd-verifier and sdd-integration-checker -- these are intentional as-is, not bugs
+3. **Non-standard markers** (e.g., `## PARTIAL`, `## ESCALATE`) in audit agents indicate partial results requiring orchestrator judgment
+4. **Agents without markers** either write artifacts directly to disk or return structured data (JSON/sections) that the caller parses
+5. Markers must appear as H2 headings (`## `) at the start of a line in the agent's final output
+
+## Key Handoff Contracts
+
+### Planner -> Executor (via PLAN.md)
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| Frontmatter | Yes | phase, plan, type, wave, depends_on, files_modified, autonomous, requirements |
+| `<objective>` | Yes | What the plan achieves |
+| `<tasks>` | Yes | Ordered task list with type, files, action, verify, acceptance_criteria |
+| `<verification>` | Yes | Overall verification steps |
+| `<success_criteria>` | Yes | Measurable completion criteria |
+
+### Executor -> Verifier (via SUMMARY.md)
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| Frontmatter | Yes | phase, plan, subsystem, tags, key-files, metrics |
+| Commits table | Yes | Per-task commit hashes and descriptions |
+| Deviations section | Yes | Auto-fixed issues or "None" |
+| Self-Check | Yes | PASSED or FAILED with details |
+
+## Workflow Regex Patterns
+
+Workflows match these markers to detect agent completion:
+
+**plan-phase.md matches:**
+- `## RESEARCH COMPLETE` / `## RESEARCH BLOCKED` (researcher output)
+- `## PLANNING COMPLETE` (planner output)
+- `## CHECKPOINT REACHED` (planner/executor pause)
+- `## VERIFICATION PASSED` / `## ISSUES FOUND` (plan-checker output)
+
+**execute-phase.md matches:**
+- `## PHASE COMPLETE` (all plans in phase done)
+- `## Self-Check: FAILED` (summary self-check)
+
+> **NOTE:** `## PLAN COMPLETE` is the sdd-executor's completion marker but execute-phase.md does not regex-match it. Instead, it detects executor completion via spot-checks (SUMMARY.md existence, git commit state). This is intentional behavior, not a mismatch.

package/sdd/references/ai-evals.md

@@ -0,0 +1,156 @@
+# AI Evaluation Reference
+
+> Reference used by `sdd-eval-planner` and `sdd-eval-auditor`.
+> Based on "AI Evals for Everyone" course (Reganti & Badam) + industry practice.
+
+---
+
+## Core Concepts
+
+### Why Evals Exist
+AI systems are non-deterministic. Input X does not reliably produce output Y across runs, users, or edge cases. Evals are the continuous process of assessing whether your system's behavior meets expectations under real-world conditions — unit tests and integration tests alone are insufficient.
+
+### Model vs. Product Evaluation
+- **Model evals** (MMLU, HumanEval, GSM8K) — measure general capability in standardized conditions. Use as initial filter only.
+- **Product evals** — measure behavior inside your specific system, with your data, your users, your domain rules. This is where 80% of eval effort belongs.
+
+### The Three Components of Every Eval
+- **Input** — everything affecting the system: query, history, retrieved docs, system prompt, config
+- **Expected** — what good behavior looks like, defined through rubrics
+- **Actual** — what the system produced, including intermediate steps, tool calls, and reasoning traces
+
+### Three Measurement Approaches
+1. **Code-based metrics** — deterministic checks: JSON validation, required disclaimers, performance thresholds, classification flags. Fast, cheap, reliable. Use first.
+2. **LLM judges** — one model evaluates another against a rubric. Powerful for subjective qualities (tone, reasoning, escalation). Requires calibration against human judgment before trusting.
+3. **Human evaluation** — gold standard for nuanced judgment. Doesn't scale. Use for calibration, edge cases, periodic sampling, and high-stakes decisions.
+
+Most effective systems combine all three.
+
+---
+
+## Evaluation Dimensions
+
+### Pre-Deployment (Development Phase)
+
+| Dimension | What It Measures | When It Matters |
+|-----------|-----------------|-----------------|
+| **Factual accuracy** | Correctness of claims against ground truth | RAG, knowledge bases, any factual assertions |
+| **Context faithfulness** | Response grounded in provided context vs. fabricated | RAG pipelines, document Q&A, retrieval-augmented systems |
+| **Hallucination detection** | Plausible but unsupported claims | All generative systems, high-stakes domains |
+| **Escalation accuracy** | Correct identification of when human intervention needed | Customer service, healthcare, financial advisory |
+| **Policy compliance** | Adherence to business rules, legal requirements, disclaimers | Regulated industries, enterprise deployments |
+| **Tone/style appropriateness** | Match with brand voice, audience expectations, emotional context | Customer-facing systems, content generation |
+| **Output structure validity** | Schema compliance, required fields, format correctness | Structured extraction, API integrations, data pipelines |
+| **Task completion** | Whether the system accomplished the stated goal | Agentic workflows, multi-step tasks |
+| **Tool use correctness** | Correct selection and invocation of tools | Agent systems with tool calls |
+| **Safety** | Absence of harmful, biased, or inappropriate outputs | All user-facing systems |
+
+### Production Monitoring
+
+| Dimension | Monitoring Approach |
+|-----------|---------------------|
+| **Safety violations** | Online guardrail — real-time, immediate intervention |
+| **Compliance failures** | Online guardrail — block or escalate before user sees output |
+| **Quality degradation trends** | Offline flywheel — batch analysis of sampled interactions |
+| **Emerging failure modes** | Signal-metric divergence — when user behavior signals diverge from metric scores, investigate manually |
+| **Cost/latency drift** | Code-based metrics — automated threshold alerts |
+
+---
+
+## The Guardrail vs. Flywheel Decision
+
+Ask: "If this behavior goes wrong, would it be catastrophic for my business?"
+
+- **Yes → Guardrail** — run online, real-time, with immediate intervention (block, escalate, hand off). Be selective: guardrails add latency.
+- **No → Flywheel** — run offline as batch analysis feeding system refinements over time.
+
+---
+
+## Rubric Design
+
+Generic metrics are meaningless without context. "Helpfulness" in real estate means summarizing listings clearly. In healthcare it means knowing when *not* to answer.
+
+A rubric must define:
+1. The dimension being measured
+2. What scores 1, 3, and 5 on a 5-point scale (or pass/fail criteria)
+3. Domain-specific examples of acceptable vs. unacceptable behavior
+
+Without rubrics, LLM judges produce noise rather than signal.
+
+---
+
+## Reference Dataset Guidelines
+
+- Start with **10-20 high-quality examples** — not 200 mediocre ones
+- Cover: critical success scenarios, common user workflows, known edge cases, historical failure modes
+- Have domain experts label the examples (not just engineers)
+- Expand based on what you learn in production — don't build for hypothetical coverage
+
+---
+
+## Eval Tooling Guide
+
+| Tool | Type | Best For | Key Strength |
+|------|------|----------|-------------|
+| **RAGAS** | Python library | RAG evaluation | Purpose-built metrics: faithfulness, answer relevance, context precision/recall |
+| **Langfuse** | Platform (open-source, self-hostable) | All system types | Strong tracing, prompt management, good for teams wanting infrastructure control |
+| **LangSmith** | Platform (commercial) | LangChain/LangGraph ecosystems | Tightest integration with LangChain; best if already in that ecosystem |
+| **Arize Phoenix** | Platform (open-source + hosted) | RAG + multi-agent tracing | Strong RAG eval + trace visualization; open-source with hosted option |
+| **Braintrust** | Platform (commercial) | Model-agnostic evaluation | Dataset and experiment management; good for comparing across frameworks |
+| **Promptfoo** | CLI tool (open-source) | Prompt testing, CI/CD | CLI-first, excellent for CI/CD prompt regression testing |
+
+### Tool Selection by System Type
+
+| System Type | Recommended Tooling |
+|-------------|---------------------|
+| RAG / Knowledge Q&A | RAGAS + Arize Phoenix or Braintrust |
+| Multi-agent systems | Langfuse + Arize Phoenix |
+| Conversational / single-model | Promptfoo + Braintrust |
+| Structured extraction | Promptfoo + code-based validators |
+| LangChain/LangGraph projects | LangSmith (native integration) |
+| Production monitoring (all types) | Langfuse, Arize Phoenix, or LangSmith |
+
+---
+
+## Evals in the Development Lifecycle
+
+### Plan Phase (Evaluation-Aware Design)
+Before writing code, define:
+1. What type of AI system is being built → determines framework and dominant eval concerns
+2. Critical failure modes (3-5 behaviors that cannot go wrong)
+3. Rubrics — explicit definitions of acceptable/unacceptable behavior per dimension
+4. Evaluation strategy — which dimensions use code metrics, LLM judges, or human review
+5. Reference dataset requirements — size, composition, labeling approach
+6. Eval tooling selection
+
+Output: EVALS-SPEC section of AI-SPEC.md
+
+### Execute Phase (Instrument While Building)
+- Add tracing from day one (Langfuse, Arize Phoenix, or LangSmith)
+- Build reference dataset concurrently with implementation
+- Implement code-based checks first; add LLM judges only for subjective dimensions
+- Run evals in CI/CD via Promptfoo or Braintrust
+
+### Verify Phase (Pre-Deployment Validation)
+- Run full reference dataset against all metrics
+- Conduct human review of edge cases and LLM judge disagreements
+- Calibrate LLM judges against human scores (target ≥ 0.7 correlation before trusting)
+- Define and configure production guardrails
+- Establish monitoring baseline
+
+### Monitor Phase (Production Evaluation Loop)
+- Smart sampling — weight toward interactions with concerning signals (retries, unusual length, explicit escalations)
+- Online guardrails on every interaction
+- Offline flywheel on sampled batch
+- Watch for signal-metric divergence — the early warning system for evaluation gaps
+
+---
+
+## Common Pitfalls
+
+1. **Assuming benchmarks predict product success** — they don't; model evals are a filter, not a verdict
+2. **Engineering evals in isolation** — domain experts must co-define rubrics; engineers alone miss critical nuances
+3. **Building comprehensive coverage on day one** — start small (10-20 examples), expand from real failure modes
+4. **Trusting uncalibrated LLM judges** — validate against human judgment before relying on them
+5. **Measuring everything** — only track metrics that drive decisions; "collect it all" produces noise
+6. **Treating evaluation as one-time setup** — user behavior evolves, requirements change, failure modes emerge; evaluation is continuous