@opengsd/get-shit-done-redux 1.0.0

package/get-shit-done/bin/verify-reapply-patches.cjs

@@ -0,0 +1,336 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Deterministic verifier for the /gsd-reapply-patches Step 5 "Hunk Verification
+ * Gate". For each backed-up patch file, asserts that the user's added lines
+ * (computed from a real diff against the pristine baseline, not from the
+ * LLM's prose summary) survive into the merged output.
+ *
+ * Usage:
+ *   node scripts/verify-reapply-patches.cjs \
+ *     --patches-dir <path>        \  # gsd-local-patches/
+ *     --config-dir <path>         \  # ~/.claude (or runtime equivalent)
+ *     [--pristine-dir <path>]        # gsd-pristine/; if absent, falls back to
+ *                                    # treating every significant backup line as
+ *                                    # required (over-broad but safe for #2969:
+ *                                    # false-positive halts beat silent successes
+ *                                    # on lost content)
+ *     [--json]                       # emit JSON report instead of human text
+ *
+ * Exit codes:
+ *   0 — every user-added line is present in the merged file (gate passes)
+ *   1 — at least one missing line in at least one file (gate fails)
+ *   2 — usage / structural error (e.g. patches dir missing)
+ *
+ * Bug #2969: the Step 5 gate previously trusted Claude's free-text "verified:
+ * yes/no" reporting per hunk. The LLM was filling in `yes` even when content
+ * had been silently dropped. Moving the check to a deterministic script is the
+ * durability fix.
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+const SIGNIFICANT_MIN_CHARS = 12;
+
+function parseArgs(argv) {
+  const opts = { patchesDir: null, configDir: null, pristineDir: null, json: false };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg === '--patches-dir') opts.patchesDir = argv[++i];
+    else if (arg === '--config-dir') opts.configDir = argv[++i];
+    else if (arg === '--pristine-dir') opts.pristineDir = argv[++i];
+    else if (arg === '--json') opts.json = true;
+    else if (arg === '--help' || arg === '-h') {
+      process.stdout.write(
+        'usage: verify-reapply-patches.cjs --patches-dir <path> --config-dir <path> [--pristine-dir <path>] [--json]\n',
+      );
+      process.exit(0);
+    } else {
+      process.stderr.write(`unknown argument: ${arg}\n`);
+      process.exit(2);
+    }
+  }
+  return opts;
+}
+
+function isSignificantLine(line) {
+  const trimmed = line.trim();
+  if (trimmed.length < SIGNIFICANT_MIN_CHARS) return false;
+  // Pure punctuation / closing brackets carry too little structural info to
+  // reliably distinguish a survived hunk from incidental similarity.
+  if (/^[\s})\];,]+$/.test(trimmed)) return false;
+  // Generic decorative comments like `// ----` similarly fail the test.
+  if (/^[\s\-=#*/]+$/.test(trimmed)) return false;
+  return true;
+}
+
+/**
+ * Compute the SHA-256 hex digest of a string (UTF-8 encoded).
+ */
+function sha256(content) {
+  return crypto.createHash('sha256').update(content, 'utf8').digest('hex');
+}
+
+/**
+ * Read the `pristine_hashes` map from backup-meta.json in the patches dir.
+ * Returns an empty object if backup-meta.json is absent, unreadable, or has no
+ * `pristine_hashes` field — callers must treat an empty map as "no recorded
+ * hash for any file" (no hash-validation possible, not an error).
+ */
+function readPristineHashes(patchesDir) {
+  const metaPath = path.join(patchesDir, 'backup-meta.json');
+  try {
+    const raw = fs.readFileSync(metaPath, 'utf8');
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed.pristine_hashes === 'object' && parsed.pristine_hashes !== null) {
+      return parsed.pristine_hashes;
+    }
+  } catch {
+    // absent or unreadable — not an error, just no recorded hashes
+  }
+  return {};
+}
+
+/**
+ * Walk a directory, returning every file's path relative to the root.
+ */
+function walk(rootDir, relPrefix = '') {
+  const out = [];
+  if (!fs.existsSync(rootDir)) return out;
+  for (const entry of fs.readdirSync(rootDir, { withFileTypes: true })) {
+    const rel = relPrefix ? path.join(relPrefix, entry.name) : entry.name;
+    const abs = path.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      out.push(...walk(abs, rel));
+    } else if (entry.isFile()) {
+      out.push(rel);
+    }
+  }
+  return out;
+}
+
+/**
+ * Compute the set of "user-added" lines: lines present in the backup but
+ * absent from the pristine baseline. If no pristine is provided, falls back
+ * to using every significant line in the backup (over-broad but safe — favours
+ * false-positive failures over silent successes, which is the right side to
+ * err on for #2969).
+ */
+function computeUserAddedLines(backupContent, pristineContent) {
+  const backupLines = backupContent.split(/\r?\n/);
+  if (!pristineContent) {
+    return backupLines.filter(isSignificantLine);
+  }
+  const pristineSet = new Set(pristineContent.split(/\r?\n/));
+  return backupLines.filter((line) => isSignificantLine(line) && !pristineSet.has(line));
+}
+
+/**
+ * Stable reason codes for the per-file result. Tests assert via
+ * `assert.equal(result.reason, REASON.X)` rather than regex-matching prose,
+ * so the diagnostic surface is a typed enum, not free text.
+ *
+ * Adding a new reason requires updating the REASON map AND the tests'
+ * shape assertion that locks the documented set of codes.
+ */
+const REASON = Object.freeze({
+  OK_NO_USER_LINES_VS_PRISTINE: 'ok_no_user_lines_vs_pristine',
+  OK_NO_SIGNIFICANT_BACKUP_LINES: 'ok_no_significant_backup_lines',
+  // Bug #3657: the on-disk gsd-pristine/ file's SHA-256 does not match the
+  // hash recorded in backup-meta.json.pristine_hashes.  This means the
+  // pristine snapshot was refreshed to a newer GSD version after the backup
+  // was captured.  Using the wrong-version pristine as the diff baseline would
+  // invert the delta (upstream removals appear as "missing user lines").
+  // The verifier skips this file rather than false-failing it.  A separate
+  // re-anchor step (or the git-aware fallback in the workflow) is needed to
+  // resolve this file; the guard here ensures the gate does not report spurious
+  // failures in the meantime.
+  OK_PRISTINE_DRIFT_DETECTED: 'ok_pristine_drift_detected',
+  FAIL_INSTALLED_MISSING: 'fail_installed_missing',
+  FAIL_INSTALLED_NOT_REGULAR_FILE: 'fail_installed_not_regular_file',
+  FAIL_READ_ERROR: 'fail_read_error',
+  FAIL_USER_LINES_MISSING: 'fail_user_lines_missing',
+});
+
+function verifyFile({ relPath, patchesDir, configDir, pristineDir, pristineHashes }) {
+  const backupPath = path.join(patchesDir, relPath);
+  const installedPath = path.join(configDir, relPath);
+  const result = { file: relPath, status: 'ok', missing: [], reason: null };
+
+  if (!fs.existsSync(backupPath) || !fs.statSync(backupPath).isFile()) {
+    return result; // walked entry no longer exists — non-fatal
+  }
+
+  // Installed path checks: must exist, must be a regular file, must be
+  // readable. Anything else is a fail-with-diagnostic, not a crash that
+  // aborts the whole gate run and drops structured output.
+  let installedStat;
+  try {
+    installedStat = fs.statSync(installedPath);
+  } catch {
+    result.status = 'fail';
+    result.reason = REASON.FAIL_INSTALLED_MISSING;
+    return result;
+  }
+  if (!installedStat.isFile()) {
+    result.status = 'fail';
+    result.reason = REASON.FAIL_INSTALLED_NOT_REGULAR_FILE;
+    return result;
+  }
+
+  let backupContent;
+  let installedContent;
+  try {
+    backupContent = fs.readFileSync(backupPath, 'utf8');
+    installedContent = fs.readFileSync(installedPath, 'utf8');
+  } catch {
+    result.status = 'fail';
+    result.reason = REASON.FAIL_READ_ERROR;
+    return result;
+  }
+
+  let pristineContent = null;
+  if (pristineDir) {
+    const pristinePath = path.join(pristineDir, relPath);
+    try {
+      const stat = fs.statSync(pristinePath);
+      if (stat.isFile()) {
+        const candidate = fs.readFileSync(pristinePath, 'utf8');
+        // Bug #3657: if backup-meta.json recorded a pristine_hash for this
+        // file, validate that the on-disk pristine matches it.  A mismatch
+        // means the installer refreshed gsd-pristine/ to a newer GSD version
+        // after the backup was captured.  Using the wrong-version pristine as
+        // the diff baseline inverts the delta: upstream removals appear as
+        // "user-added lines that must survive", causing FAIL_USER_LINES_MISSING
+        // false positives.  When the hash is stale, skip the pristine and fall
+        // through to over-broad mode (every significant backup line is checked).
+        // Over-broad mode never false-fails for a different reason because all
+        // backup lines that are genuinely user-added will still be present in a
+        // correctly merged install.
+        // Normalize to forward slashes so the key lookup matches on Windows
+        // where path.join produces backslash-separated relPath values but
+        // backup-meta.json stores keys written with forward slashes.
+        const hashKey = relPath.replace(/\\/g, '/');
+        const recordedHash = pristineHashes && pristineHashes[hashKey];
+        if (recordedHash) {
+          if (sha256(candidate) === recordedHash) {
+            // Hash matches: the on-disk pristine is the correct baseline.
+            pristineContent = candidate;
+          } else {
+            // Hash mismatch: the on-disk gsd-pristine/ was refreshed to a newer
+            // GSD version after the backup was captured.  Using it as the diff
+            // baseline would invert the delta and produce false FAIL_USER_LINES_MISSING
+            // reports (Bug #3657).  Report the file as ok with a diagnostic code
+            // so the gate does not false-fail; a re-anchor or git-aware baseline
+            // step is required to verify this file correctly.
+            result.reason = REASON.OK_PRISTINE_DRIFT_DETECTED;
+            return result;
+          }
+        } else {
+          // No recorded hash for this file (older installer or absent
+          // backup-meta) — use the on-disk pristine as-is (pre-fix behaviour).
+          pristineContent = candidate;
+        }
+      }
+    } catch {
+      // Pristine missing or unreadable — fall through to over-broad mode.
+    }
+  }
+
+  const userAdded = computeUserAddedLines(backupContent, pristineContent);
+  if (userAdded.length === 0) {
+    // Backup and pristine match exactly (or no significant content) — nothing
+    // to verify but also nothing to lose. Report as ok with diagnostic code.
+    result.reason = pristineContent
+      ? REASON.OK_NO_USER_LINES_VS_PRISTINE
+      : REASON.OK_NO_SIGNIFICANT_BACKUP_LINES;
+    return result;
+  }
+
+  for (const line of userAdded) {
+    if (!installedContent.includes(line)) {
+      result.missing.push(line.trim());
+    }
+  }
+  if (result.missing.length > 0) {
+    result.status = 'fail';
+    result.reason = REASON.FAIL_USER_LINES_MISSING;
+  }
+  return result;
+}
+
+function main() {
+  const opts = parseArgs(process.argv.slice(2));
+  if (!opts.patchesDir || !opts.configDir) {
+    process.stderr.write('--patches-dir and --config-dir are required\n');
+    process.exit(2);
+  }
+  if (!fs.existsSync(opts.patchesDir)) {
+    process.stderr.write(`patches dir not found: ${opts.patchesDir}\n`);
+    process.exit(2);
+  }
+  if (!fs.existsSync(opts.configDir)) {
+    process.stderr.write(`config dir not found: ${opts.configDir}\n`);
+    process.exit(2);
+  }
+
+  const files = walk(opts.patchesDir).filter((f) => !f.endsWith('backup-meta.json'));
+  // Bug #3657: read pristine_hashes from backup-meta.json once and share
+  // across all per-file verifications so each can detect drift independently.
+  const pristineHashes = readPristineHashes(opts.patchesDir);
+  const results = files.map((relPath) =>
+    verifyFile({
+      relPath,
+      patchesDir: opts.patchesDir,
+      configDir: opts.configDir,
+      pristineDir: opts.pristineDir,
+      pristineHashes,
+    }),
+  );
+
+  const failures = results.filter((r) => r.status === 'fail');
+
+  // Bug #3657 (Finding 1): aggregate drifted files into top-level report fields
+  // so workflow Step 5a can gate on drift distinctly from failures.  Drift is
+  // NOT a failure (exit code stays 0) but the workflow now has structured data
+  // to decide whether to proceed.  Per-file shape is unchanged: each drifted
+  // result still has status:'ok' + reason:OK_PRISTINE_DRIFT_DETECTED for
+  // backward compat.  The new top-level fields are purely additive.
+  const driftedResults = results.filter((r) => r.reason === REASON.OK_PRISTINE_DRIFT_DETECTED);
+  const drifted = driftedResults.length;
+  const drifted_files = driftedResults.map((r) => r.file);
+
+  if (opts.json) {
+    process.stdout.write(
+      JSON.stringify({ checked: results.length, failures: failures.length, drifted, drifted_files, results }, null, 2) + '\n',
+    );
+  } else {
+    process.stdout.write(`# Hunk Verification Gate (#2969)\n\n`);
+    process.stdout.write(`Checked: ${results.length} file(s)\n`);
+    process.stdout.write(`Failures: ${failures.length}\n\n`);
+    if (failures.length > 0) {
+      process.stdout.write(`## Files with missing user-added content\n\n`);
+      for (const r of failures) {
+        process.stdout.write(`- ${r.file}\n`);
+        if (r.reason) process.stdout.write(`  reason: ${r.reason}\n`);
+        for (const line of r.missing.slice(0, 5)) {
+          process.stdout.write(`  missing: ${line}\n`);
+        }
+        if (r.missing.length > 5) {
+          process.stdout.write(`  …and ${r.missing.length - 5} more line(s)\n`);
+        }
+      }
+    }
+  }
+
+  process.exit(failures.length > 0 ? 1 : 0);
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = { computeUserAddedLines, isSignificantLine, verifyFile, walk, REASON, readPristineHashes, sha256 };

package/get-shit-done/contexts/dev.md

@@ -0,0 +1,21 @@
+# Dev Context Profile
+
+Agent output guidance for dev mode. Loaded when `context: dev` is set in config.json.
+
+## Output Style
+
+- Concise, action-oriented responses
+- Lead with the code change or command, follow with brief rationale
+- Skip preamble — assume the developer has full context
+- Use inline code references (`file:line`) over prose descriptions
+
+## Focus Areas
+
+- Working code that compiles and passes tests
+- Minimal diff — change only what is necessary
+- Flag side effects or breaking changes immediately
+- Surface the next actionable step at the end of every response
+
+## Verbosity
+
+Low. One-liner explanations unless the change is non-obvious. Omit background theory, alternative approaches, and caveats that do not affect the current task.

package/get-shit-done/contexts/research.md

@@ -0,0 +1,22 @@
+# Research Context Profile
+
+Agent output guidance for research mode. Loaded when `context: research` is set in config.json.
+
+## Output Style
+
+- Verbose, exploratory responses that surface trade-offs and alternatives
+- Present multiple approaches with pros and cons before recommending one
+- Include links, references, and citations where available
+- Use structured headings and bullet lists for scan-ability
+
+## Focus Areas
+
+- Breadth of options — enumerate before narrowing
+- Prior art and ecosystem conventions
+- Risks, edge cases, and failure modes
+- Dependencies and compatibility implications
+- Long-term maintainability of each approach
+
+## Verbosity
+
+High. Explain reasoning, show evidence, and document assumptions. Include background context even if the developer likely knows it — research artifacts are read by future contributors who may not.

package/get-shit-done/contexts/review.md

@@ -0,0 +1,23 @@
+# Review Context Profile
+
+Agent output guidance for review mode. Loaded when `context: review` is set in config.json.
+
+## Output Style
+
+- Critical, detail-focused responses that prioritize correctness
+- Organize findings by severity: blocking, important, nit
+- Reference specific lines and files for every finding
+- State what is correct as well as what needs change — confirm the good parts
+
+## Focus Areas
+
+- Correctness — logic errors, off-by-ones, missing edge cases
+- Security — input validation, injection vectors, secret exposure
+- Performance — unnecessary allocations, O(n^2) patterns, missing caching
+- Style and consistency — naming, formatting, import order
+- Test coverage — untested branches, missing assertions, flaky patterns
+- Structural Findings (fallow) — machine-derived cross-module facts injected as `<structural_findings>{ findings: [...], summary: { total, unusedExports, duplicates, circularDependencies } }</structural_findings>`. Render in REVIEW.md under `## Structural Findings (fallow)` as a separate section from narrative findings. Treat as ground truth for cross-module facts; do not re-derive.
+
+## Verbosity
+
+Medium. Be thorough on findings but terse in explanation. Each issue should be one to three sentences: what is wrong, why it matters, and how to fix it.

package/get-shit-done/references/agent-contracts.md

@@ -0,0 +1,79 @@
+# Agent Contracts
+
+Completion markers and handoff schemas for all GSD agents. Workflows use these markers to detect agent completion and route accordingly.
+
+This doc describes what IS, not what should be. Casing inconsistencies are documented as they appear in agent source files.
+
+---
+
+## Agent Registry
+
+| Agent | Role | Completion Markers |
+|-------|------|--------------------|
+| gsd-planner | Plan creation | `## PLANNING COMPLETE` |
+| gsd-executor | Plan execution | `## PLAN COMPLETE`, `## CHECKPOINT REACHED` |
+| gsd-phase-researcher | Phase-scoped research | `## RESEARCH COMPLETE`, `## RESEARCH BLOCKED` |
+| gsd-project-researcher | Project-wide research | `## RESEARCH COMPLETE`, `## RESEARCH BLOCKED` |
+| gsd-plan-checker | Plan validation | `## VERIFICATION PASSED`, `## ISSUES FOUND` |
+| gsd-research-synthesizer | Multi-research synthesis | `## SYNTHESIS COMPLETE`, `## SYNTHESIS BLOCKED` |
+| gsd-debugger | Debug investigation | `## DEBUG COMPLETE`, `## ROOT CAUSE FOUND`, `## CHECKPOINT REACHED` |
+| gsd-roadmapper | Roadmap creation/revision | `## ROADMAP CREATED`, `## ROADMAP REVISED`, `## ROADMAP BLOCKED` |
+| gsd-ui-auditor | UI review | `## UI REVIEW COMPLETE` |
+| gsd-ui-checker | UI validation | `## ISSUES FOUND` |
+| gsd-ui-researcher | UI spec creation | `## UI-SPEC COMPLETE`, `## UI-SPEC BLOCKED` |
+| gsd-verifier | Post-execution verification | `## Verification Complete` (title case) |
+| gsd-integration-checker | Cross-phase integration check | `## Integration Check Complete` (title case) |
+| gsd-nyquist-auditor | Sampling audit | `## PARTIAL`, `## ESCALATE` (non-standard) |
+| gsd-security-auditor | Security audit | `## OPEN_THREATS`, `## ESCALATE` (non-standard) |
+| gsd-codebase-mapper | Codebase analysis | No marker (writes docs directly) |
+| gsd-assumptions-analyzer | Assumption extraction | No marker (returns `## Assumptions` sections) |
+| gsd-doc-verifier | Doc validation | No marker (writes JSON to `.planning/tmp/`) |
+| gsd-doc-writer | Doc generation | No marker (writes docs directly) |
+| gsd-advisor-researcher | Advisory research | No marker (utility agent) |
+| gsd-user-profiler | User profiling | No marker (returns JSON in analysis tags) |
+| gsd-intel-updater | Codebase intelligence analysis | `## INTEL UPDATE COMPLETE`, `## INTEL UPDATE FAILED` |
+
+## Marker Rules
+
+1. **ALL-CAPS markers** (e.g., `## PLANNING COMPLETE`) are the standard convention
+2. **Title-case markers** (e.g., `## Verification Complete`) exist in gsd-verifier and gsd-integration-checker -- these are intentional as-is, not bugs
+3. **Non-standard markers** (e.g., `## PARTIAL`, `## ESCALATE`) in audit agents indicate partial results requiring orchestrator judgment
+4. **Agents without markers** either write artifacts directly to disk or return structured data (JSON/sections) that the caller parses
+5. Markers must appear as H2 headings (`## `) at the start of a line in the agent's final output
+
+## Key Handoff Contracts
+
+### Planner -> Executor (via PLAN.md)
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| Frontmatter | Yes | phase, plan, type, wave, depends_on, files_modified, autonomous, requirements |
+| `<objective>` | Yes | What the plan achieves |
+| `<tasks>` | Yes | Ordered task list with type, files, action, verify, acceptance_criteria |
+| `<verification>` | Yes | Overall verification steps |
+| `<success_criteria>` | Yes | Measurable completion criteria |
+
+### Executor -> Verifier (via SUMMARY.md)
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| Frontmatter | Yes | phase, plan, subsystem, tags, key-files, metrics |
+| Commits table | Yes | Per-task commit hashes and descriptions |
+| Deviations section | Yes | Auto-fixed issues or "None" |
+| Self-Check | Yes | PASSED or FAILED with details |
+
+## Workflow Regex Patterns
+
+Workflows match these markers to detect agent completion:
+
+**plan-phase.md matches:**
+- `## RESEARCH COMPLETE` / `## RESEARCH BLOCKED` (researcher output)
+- `## PLANNING COMPLETE` (planner output)
+- `## CHECKPOINT REACHED` (planner/executor pause)
+- `## VERIFICATION PASSED` / `## ISSUES FOUND` (plan-checker output)
+
+**execute-phase.md matches:**
+- `## PHASE COMPLETE` (all plans in phase done)
+- `## Self-Check: FAILED` (summary self-check)
+
+> **NOTE:** `## PLAN COMPLETE` is the gsd-executor's completion marker but execute-phase.md does not regex-match it. Instead, it detects executor completion via spot-checks (SUMMARY.md existence, git commit state). This is intentional behavior, not a mismatch.

package/get-shit-done/references/ai-evals.md

@@ -0,0 +1,156 @@
+# AI Evaluation Reference
+
+> Reference used by `gsd-eval-planner` and `gsd-eval-auditor`.
+> Based on "AI Evals for Everyone" course (Reganti & Badam) + industry practice.
+
+---
+
+## Core Concepts
+
+### Why Evals Exist
+AI systems are non-deterministic. Input X does not reliably produce output Y across runs, users, or edge cases. Evals are the continuous process of assessing whether your system's behavior meets expectations under real-world conditions — unit tests and integration tests alone are insufficient.
+
+### Model vs. Product Evaluation
+- **Model evals** (MMLU, HumanEval, GSM8K) — measure general capability in standardized conditions. Use as initial filter only.
+- **Product evals** — measure behavior inside your specific system, with your data, your users, your domain rules. This is where 80% of eval effort belongs.
+
+### The Three Components of Every Eval
+- **Input** — everything affecting the system: query, history, retrieved docs, system prompt, config
+- **Expected** — what good behavior looks like, defined through rubrics
+- **Actual** — what the system produced, including intermediate steps, tool calls, and reasoning traces
+
+### Three Measurement Approaches
+1. **Code-based metrics** — deterministic checks: JSON validation, required disclaimers, performance thresholds, classification flags. Fast, cheap, reliable. Use first.
+2. **LLM judges** — one model evaluates another against a rubric. Powerful for subjective qualities (tone, reasoning, escalation). Requires calibration against human judgment before trusting.
+3. **Human evaluation** — gold standard for nuanced judgment. Doesn't scale. Use for calibration, edge cases, periodic sampling, and high-stakes decisions.
+
+Most effective systems combine all three.
+
+---
+
+## Evaluation Dimensions
+
+### Pre-Deployment (Development Phase)
+
+| Dimension | What It Measures | When It Matters |
+|-----------|-----------------|-----------------|
+| **Factual accuracy** | Correctness of claims against ground truth | RAG, knowledge bases, any factual assertions |
+| **Context faithfulness** | Response grounded in provided context vs. fabricated | RAG pipelines, document Q&A, retrieval-augmented systems |
+| **Hallucination detection** | Plausible but unsupported claims | All generative systems, high-stakes domains |
+| **Escalation accuracy** | Correct identification of when human intervention needed | Customer service, healthcare, financial advisory |
+| **Policy compliance** | Adherence to business rules, legal requirements, disclaimers | Regulated industries, enterprise deployments |
+| **Tone/style appropriateness** | Match with brand voice, audience expectations, emotional context | Customer-facing systems, content generation |
+| **Output structure validity** | Schema compliance, required fields, format correctness | Structured extraction, API integrations, data pipelines |
+| **Task completion** | Whether the system accomplished the stated goal | Agentic workflows, multi-step tasks |
+| **Tool use correctness** | Correct selection and invocation of tools | Agent systems with tool calls |
+| **Safety** | Absence of harmful, biased, or inappropriate outputs | All user-facing systems |
+
+### Production Monitoring
+
+| Dimension | Monitoring Approach |
+|-----------|---------------------|
+| **Safety violations** | Online guardrail — real-time, immediate intervention |
+| **Compliance failures** | Online guardrail — block or escalate before user sees output |
+| **Quality degradation trends** | Offline flywheel — batch analysis of sampled interactions |
+| **Emerging failure modes** | Signal-metric divergence — when user behavior signals diverge from metric scores, investigate manually |
+| **Cost/latency drift** | Code-based metrics — automated threshold alerts |
+
+---
+
+## The Guardrail vs. Flywheel Decision
+
+Ask: "If this behavior goes wrong, would it be catastrophic for my business?"
+
+- **Yes → Guardrail** — run online, real-time, with immediate intervention (block, escalate, hand off). Be selective: guardrails add latency.
+- **No → Flywheel** — run offline as batch analysis feeding system refinements over time.
+
+---
+
+## Rubric Design
+
+Generic metrics are meaningless without context. "Helpfulness" in real estate means summarizing listings clearly. In healthcare it means knowing when *not* to answer.
+
+A rubric must define:
+1. The dimension being measured
+2. What scores 1, 3, and 5 on a 5-point scale (or pass/fail criteria)
+3. Domain-specific examples of acceptable vs. unacceptable behavior
+
+Without rubrics, LLM judges produce noise rather than signal.
+
+---
+
+## Reference Dataset Guidelines
+
+- Start with **10-20 high-quality examples** — not 200 mediocre ones
+- Cover: critical success scenarios, common user workflows, known edge cases, historical failure modes
+- Have domain experts label the examples (not just engineers)
+- Expand based on what you learn in production — don't build for hypothetical coverage
+
+---
+
+## Eval Tooling Guide
+
+| Tool | Type | Best For | Key Strength |
+|------|------|----------|-------------|
+| **RAGAS** | Python library | RAG evaluation | Purpose-built metrics: faithfulness, answer relevance, context precision/recall |
+| **Langfuse** | Platform (open-source, self-hostable) | All system types | Strong tracing, prompt management, good for teams wanting infrastructure control |
+| **LangSmith** | Platform (commercial) | LangChain/LangGraph ecosystems | Tightest integration with LangChain; best if already in that ecosystem |
+| **Arize Phoenix** | Platform (open-source + hosted) | RAG + multi-agent tracing | Strong RAG eval + trace visualization; open-source with hosted option |
+| **Braintrust** | Platform (commercial) | Model-agnostic evaluation | Dataset and experiment management; good for comparing across frameworks |
+| **Promptfoo** | CLI tool (open-source) | Prompt testing, CI/CD | CLI-first, excellent for CI/CD prompt regression testing |
+
+### Tool Selection by System Type
+
+| System Type | Recommended Tooling |
+|-------------|---------------------|
+| RAG / Knowledge Q&A | RAGAS + Arize Phoenix or Braintrust |
+| Multi-agent systems | Langfuse + Arize Phoenix |
+| Conversational / single-model | Promptfoo + Braintrust |
+| Structured extraction | Promptfoo + code-based validators |
+| LangChain/LangGraph projects | LangSmith (native integration) |
+| Production monitoring (all types) | Langfuse, Arize Phoenix, or LangSmith |
+
+---
+
+## Evals in the Development Lifecycle
+
+### Plan Phase (Evaluation-Aware Design)
+Before writing code, define:
+1. What type of AI system is being built → determines framework and dominant eval concerns
+2. Critical failure modes (3-5 behaviors that cannot go wrong)
+3. Rubrics — explicit definitions of acceptable/unacceptable behavior per dimension
+4. Evaluation strategy — which dimensions use code metrics, LLM judges, or human review
+5. Reference dataset requirements — size, composition, labeling approach
+6. Eval tooling selection
+
+Output: EVALS-SPEC section of AI-SPEC.md
+
+### Execute Phase (Instrument While Building)
+- Add tracing from day one (Langfuse, Arize Phoenix, or LangSmith)
+- Build reference dataset concurrently with implementation
+- Implement code-based checks first; add LLM judges only for subjective dimensions
+- Run evals in CI/CD via Promptfoo or Braintrust
+
+### Verify Phase (Pre-Deployment Validation)
+- Run full reference dataset against all metrics
+- Conduct human review of edge cases and LLM judge disagreements
+- Calibrate LLM judges against human scores (target ≥ 0.7 correlation before trusting)
+- Define and configure production guardrails
+- Establish monitoring baseline
+
+### Monitor Phase (Production Evaluation Loop)
+- Smart sampling — weight toward interactions with concerning signals (retries, unusual length, explicit escalations)
+- Online guardrails on every interaction
+- Offline flywheel on sampled batch
+- Watch for signal-metric divergence — the early warning system for evaluation gaps
+
+---
+
+## Common Pitfalls
+
+1. **Assuming benchmarks predict product success** — they don't; model evals are a filter, not a verdict
+2. **Engineering evals in isolation** — domain experts must co-define rubrics; engineers alone miss critical nuances
+3. **Building comprehensive coverage on day one** — start small (10-20 examples), expand from real failure modes
+4. **Trusting uncalibrated LLM judges** — validate against human judgment before relying on them
+5. **Measuring everything** — only track metrics that drive decisions; "collect it all" produces noise
+6. **Treating evaluation as one-time setup** — user behavior evolves, requirements change, failure modes emerge; evaluation is continuous