@hone-ai/cli 1.6.0 → 1.7.1

package/hone-cli.js

@@ -1274,7 +1274,8 @@ program
 
     try {
       const health = await axios.get(`${config.apiUrl}/health`);
-      console.log(`✓ API health: ${health.data.status} (v${health.data.version})`);
+      const ver = health.data.version ? ` (v${health.data.version})` : '';
+      console.log(`✓ API health: ${health.data.status}${ver}`);
     } catch (e) {
       console.error(`✗ API health: ${e.message}`);
     }
@@ -3927,12 +3928,12 @@ program
   .description('Validate .github/pipeline/<STORY-ID>/metadata.yml against the framework JSON schema. Implements SC-010 §10 (metadata.yml as wire protocol).')
   .option('--all', 'validate every metadata.yml in .github/pipeline/')
   .option('--repo-root <path>', 'repo root (default: process.cwd())')
-  .option('--schema <path>', 'override schema path (default: enterprise-assets/.github/schema/metadata.schema.json)')
+  .option('--schema <path>', 'override schema path (default: bundled metadata.schema.json)')
   .option('--json', 'emit findings as JSON')
   .action((storyId, opts) => {
     const { validateMetadata, validateAllMetadata } = require('./lib/validate-metadata');
     const repoRoot = opts.repoRoot || process.cwd();
-    const schemaPath = opts.schema || require('node:path').join(repoRoot, 'enterprise-assets/.github/schema/metadata.schema.json');
+    const schemaPath = opts.schema || require('node:path').join(__dirname, 'schema', 'metadata.schema.json');
 
     if (opts.all) {
       const result = validateAllMetadata({ repoRoot, schemaPath });

package/lib/auto-detect.js

@@ -174,23 +174,58 @@ function detectE2EConvention(signals) {
  * we recommend. Returns either status:'ok' or status:'drift' with an
  * actionable suggested fix.
  */
+/**
+ * Check if configuredPattern is a superset of recommendedPattern.
+ * A broader configured pattern is intentional (not drift).
+ * Only flag when configured is NARROWER than recommended.
+ */
+function isPatternBroader(configured, recommended) {
+  try {
+    const recRe = new RegExp(recommended);
+    const cfgRe = new RegExp(configured);
+    // Test a set of sample strings that the recommended pattern matches
+    // If configured matches all of them, it's at least as broad
+    const samples = [];
+    // Generate simple test strings from recommended pattern components
+    const recStr = recommended.replace(/\\/g, '');
+    if (recStr.includes('|')) {
+      // Pattern has alternations — extract them
+      const alts = recommended.split('|').map(a => a.replace(/[()^$]/g, ''));
+      for (const alt of alts) {
+        // Generate a plausible match for each alternative
+        const sample = alt.replace(/\[0-9\]\+/g, '123').replace(/\[A-Z\]\+?/g, 'A').replace(/\[A-Za-z0-9\]\+/g, 'abc');
+        samples.push(sample);
+      }
+    }
+    // If configured matches everything recommended matches, it's broader or equal
+    return samples.length > 0 && samples.every(s => cfgRe.test(s));
+  } catch {
+    return false;
+  }
+}
+
 function checkPatternDrift({ configured, recommended }) {
   const findings = [];
   if (configured?.story_id_pattern && configured.story_id_pattern !== recommended.story.pattern) {
-    findings.push({
-      key: 'story_id_pattern',
-      configured: configured.story_id_pattern,
-      recommended: recommended.story.pattern,
-      reason: `repo looks like ${recommended.story.shape} (${recommended.story.confidence} confidence)`,
-    });
+    // Only flag if configured is NARROWER — broader is intentional
+    if (!isPatternBroader(configured.story_id_pattern, recommended.story.pattern)) {
+      findings.push({
+        key: 'story_id_pattern',
+        configured: configured.story_id_pattern,
+        recommended: recommended.story.pattern,
+        reason: `repo looks like ${recommended.story.shape} (${recommended.story.confidence} confidence)`,
+      });
+    }
   }
   if (configured?.e2e_spec_pattern && configured.e2e_spec_pattern !== recommended.e2e.pattern) {
-    findings.push({
-      key: 'e2e_spec_pattern',
-      configured: configured.e2e_spec_pattern,
-      recommended: recommended.e2e.pattern,
-      reason: `detected ${recommended.e2e.framework} under ${recommended.e2e.dir}/e2e/ (${recommended.e2e.confidence} confidence)`,
-    });
+    if (!isPatternBroader(configured.e2e_spec_pattern, recommended.e2e.pattern)) {
+      findings.push({
+        key: 'e2e_spec_pattern',
+        configured: configured.e2e_spec_pattern,
+        recommended: recommended.e2e.pattern,
+        reason: `detected ${recommended.e2e.framework} under ${recommended.e2e.dir}/e2e/ (${recommended.e2e.confidence} confidence)`,
+      });
+    }
   }
 
   if (findings.length === 0) {

package/lib/eval-ab-testing.js

@@ -0,0 +1,113 @@
+'use strict';
+/**
+ * eval-ab-testing.js — HC-019k Agent A/B testing.
+ *
+ * Compare two prompt variants using the same eval scenarios.
+ * Reports which variant performs better across all checks.
+ *
+ * Usage: provide two versions of an agent prompt (A and B),
+ * run the same deterministic evals against both, compare results.
+ *
+ * Pure helper — no I/O, no LLM calls.
+ */
+const { runScenario } = require('./eval-runner');
+const { wrapDeterministic } = require('./eval-three-valued');
+
+/**
+ * Run A/B comparison for a set of scenarios against two prompt variants.
+ *
+ * @param {Array<object>} scenarios — eval scenarios (filtered for one agent)
+ * @param {string} promptA — current prompt text (control)
+ * @param {string} promptB — new prompt text (variant)
+ * @param {object} [opts]
+ * @param {string} [opts.labelA='A (current)']
+ * @param {string} [opts.labelB='B (variant)']
+ * @returns {{ agent, labelA, labelB, scenarios: Array, summary }}
+ */
+function comparePrompts(scenarios, promptA, promptB, opts = {}) {
+  const { labelA = 'A (current)', labelB = 'B (variant)' } = opts;
+  const results = [];
+
+  for (const scenario of scenarios) {
+    const resultA = wrapDeterministic(runScenario(scenario, promptA));
+    const resultB = wrapDeterministic(runScenario(scenario, promptB));
+
+    const winner =
+      resultA.verdict === 'pass' && resultB.verdict !== 'pass' ? 'A' :
+      resultB.verdict === 'pass' && resultA.verdict !== 'pass' ? 'B' :
+      resultA.checks_passed > resultB.checks_passed ? 'A' :
+      resultB.checks_passed > resultA.checks_passed ? 'B' :
+      'tie';
+
+    results.push({
+      id: scenario.id,
+      name: scenario.name || scenario.id,
+      a: { verdict: resultA.verdict, checks_passed: resultA.checks_passed, checks: resultA.checks },
+      b: { verdict: resultB.verdict, checks_passed: resultB.checks_passed, checks: resultB.checks },
+      winner,
+    });
+  }
+
+  const aWins = results.filter(r => r.winner === 'A').length;
+  const bWins = results.filter(r => r.winner === 'B').length;
+  const ties = results.filter(r => r.winner === 'tie').length;
+
+  const aTotal = results.reduce((s, r) => s + r.a.checks_passed, 0);
+  const bTotal = results.reduce((s, r) => s + r.b.checks_passed, 0);
+  const maxChecks = results.reduce((s, r) => s + r.a.checks, 0);
+
+  return {
+    agent: scenarios[0]?.evalAgent || scenarios[0]?.agent || 'unknown',
+    labelA,
+    labelB,
+    scenarios: results,
+    summary: {
+      total: results.length,
+      a_wins: aWins,
+      b_wins: bWins,
+      ties,
+      a_score: maxChecks > 0 ? Math.round((aTotal / maxChecks) * 100) : 0,
+      b_score: maxChecks > 0 ? Math.round((bTotal / maxChecks) * 100) : 0,
+      recommendation: aWins > bWins ? 'keep_a' : bWins > aWins ? 'use_b' : 'no_difference',
+    },
+  };
+}
+
+/**
+ * Format A/B comparison results.
+ */
+function formatComparison(result, format = 'pretty') {
+  if (format === 'json') return JSON.stringify(result, null, 2);
+
+  const lines = ['', 'Hone AI — A/B Prompt Comparison', '================================', ''];
+  lines.push(`Agent:     ${result.agent}`);
+  lines.push(`Variant A: ${result.labelA}`);
+  lines.push(`Variant B: ${result.labelB}`);
+  lines.push('');
+
+  lines.push('  Scenario                          A        B     Winner');
+  lines.push('  --------                          -        -     ------');
+  for (const s of result.scenarios) {
+    const name = (s.name || s.id).padEnd(32).slice(0, 32);
+    const a = `${s.a.checks_passed}/${s.a.checks}`.padStart(6);
+    const b = `${s.b.checks_passed}/${s.b.checks}`.padStart(6);
+    const winner = s.winner === 'tie' ? ' tie' : s.winner === 'A' ? ' <-A' : ' B->';
+    lines.push(`  ${name} ${a}   ${b}   ${winner}`);
+  }
+
+  lines.push('');
+  lines.push('----------------------------------');
+  lines.push(`Score: A=${result.summary.a_score}% | B=${result.summary.b_score}%`);
+  lines.push(`Wins:  A=${result.summary.a_wins} | B=${result.summary.b_wins} | Ties=${result.summary.ties}`);
+
+  const rec = result.summary.recommendation;
+  const msg = rec === 'keep_a' ? 'Keep current prompt (A wins)' :
+    rec === 'use_b' ? 'Switch to variant B (B wins)' :
+    'No significant difference';
+  lines.push(`Recommendation: ${msg}`);
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+module.exports = { comparePrompts, formatComparison };

package/lib/eval-llm-judge.js

@@ -0,0 +1,213 @@
+'use strict';
+/**
+ * eval-llm-judge.js — HC-019i LLM-as-judge evaluator.
+ *
+ * Uses an LLM to assess agent prompt quality against criteria that
+ * deterministic graders can't check (semantic meaning, completeness,
+ * reasoning quality).
+ *
+ * Pure helper with injected LLM call function.
+ * Integrates with HC-019j three-valued outcomes for non-deterministic results.
+ */
+const { classify, wrapDeterministic } = require('./eval-three-valued');
+
+/**
+ * Judge criteria for LLM evaluation.
+ * Each criterion is a question the LLM answers YES/NO about the agent output.
+ *
+ * Eval scenario format for LLM-judge mode:
+ * ```yaml
+ * grading:
+ *   mode: llm-judge
+ *   criteria:
+ *     - "Does the prompt clearly define the agent's role and responsibilities?"
+ *     - "Does the prompt include error handling guidance?"
+ *     - "Is the output format specification unambiguous?"
+ *   runs: 3  # optional, default 1 for cost savings
+ * ```
+ */
+
+const JUDGE_SYSTEM_PROMPT = `You are an eval judge for AI agent prompts. You will be given an agent prompt and a list of criteria. For each criterion, answer YES or NO with a brief explanation.
+
+Rules:
+- Answer ONLY YES or NO for each criterion, followed by a one-sentence explanation
+- Be strict — if the criterion is not clearly met, answer NO
+- Format each answer on its own line as: "CRITERION_N: YES|NO — explanation"
+- Do not add commentary outside the criterion answers`;
+
+/**
+ * Build the judge prompt.
+ * @param {string} agentPrompt — the agent prompt text to evaluate
+ * @param {string[]} criteria — list of criteria to judge against
+ * @returns {string}
+ */
+function buildJudgePrompt(agentPrompt, criteria) {
+  const criteriaList = criteria
+    .map((c, i) => `CRITERION_${i + 1}: ${c}`)
+    .join('\n');
+
+  return `## Agent Prompt to Evaluate
+
+${agentPrompt.slice(0, 8000)}
+
+## Criteria to Judge
+
+${criteriaList}
+
+## Your Judgement
+
+For each criterion, answer YES or NO with a brief explanation:`;
+}
+
+/**
+ * Parse the LLM judge response into structured results.
+ * @param {string} response — LLM output
+ * @param {number} criteriaCount — expected number of criteria
+ * @returns {Array<{ criterion: number, passed: boolean, explanation: string }>}
+ */
+function parseJudgeResponse(response, criteriaCount) {
+  const results = [];
+
+  for (let i = 1; i <= criteriaCount; i++) {
+    const pattern = new RegExp(`CRITERION_${i}:\\s*(YES|NO)\\s*[-—]\\s*(.+)`, 'i');
+    const match = response.match(pattern);
+
+    if (match) {
+      results.push({
+        criterion: i,
+        passed: match[1].toUpperCase() === 'YES',
+        explanation: match[2].trim(),
+      });
+    } else {
+      // Try looser pattern
+      const loosePattern = new RegExp(`(?:CRITERION_${i}|#${i}|${i}\\.)\\s*:?\\s*(YES|NO)`, 'i');
+      const looseMatch = response.match(loosePattern);
+      results.push({
+        criterion: i,
+        passed: looseMatch ? looseMatch[1].toUpperCase() === 'YES' : false,
+        explanation: looseMatch ? 'parsed from loose format' : 'could not parse response',
+      });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Run a single LLM-judge evaluation.
+ * @param {object} opts
+ * @param {string} opts.agentPrompt — prompt text to evaluate
+ * @param {string[]} opts.criteria — list of criteria
+ * @param {(systemPrompt: string, userPrompt: string) => Promise<string>} opts.callLLM — injected LLM call
+ * @returns {Promise<{ passed: boolean, criteriaResults: Array, rawResponse: string }>}
+ */
+async function runJudge({ agentPrompt, criteria, callLLM }) {
+  const userPrompt = buildJudgePrompt(agentPrompt, criteria);
+  const response = await callLLM(JUDGE_SYSTEM_PROMPT, userPrompt);
+  const criteriaResults = parseJudgeResponse(response, criteria.length);
+  const allPassed = criteriaResults.every(r => r.passed);
+
+  return {
+    passed: allPassed,
+    criteriaResults,
+    rawResponse: response,
+  };
+}
+
+/**
+ * Run LLM-judge evaluation with optional multiple runs for three-valued outcomes.
+ * @param {object} opts
+ * @param {object} opts.scenario — eval scenario with grading.mode = 'llm-judge'
+ * @param {string} opts.agentPrompt — prompt text
+ * @param {(systemPrompt: string, userPrompt: string) => Promise<string>} opts.callLLM
+ * @param {object} [opts.thresholds] — pass/fail thresholds for classify()
+ * @returns {Promise<object>} — scenario result with verdict + confidence
+ */
+async function runJudgeScenario({ scenario, agentPrompt, callLLM, thresholds }) {
+  const criteria = scenario.grading?.criteria || [];
+  const runs = scenario.grading?.runs || 1;
+
+  if (criteria.length === 0) {
+    return wrapDeterministic({
+      id: scenario.id,
+      agent: scenario.evalAgent || scenario.agent,
+      name: scenario.name || scenario.id,
+      result: 'error',
+      checks: 0,
+      checks_passed: 0,
+      failures: [{ type: 'config', passed: false, detail: 'no criteria defined for llm-judge' }],
+    });
+  }
+
+  // Single run — return deterministic result
+  if (runs <= 1) {
+    try {
+      const judgeResult = await runJudge({ agentPrompt, criteria, callLLM });
+      return wrapDeterministic({
+        id: scenario.id,
+        agent: scenario.evalAgent || scenario.agent,
+        name: scenario.name || scenario.id,
+        result: judgeResult.passed ? 'pass' : 'fail',
+        checks: criteria.length,
+        checks_passed: judgeResult.criteriaResults.filter(r => r.passed).length,
+        failures: judgeResult.criteriaResults
+          .filter(r => !r.passed)
+          .map(r => ({ type: `criterion_${r.criterion}`, passed: false, detail: r.explanation })),
+        judgeDetails: judgeResult.criteriaResults,
+      });
+    } catch (e) {
+      return wrapDeterministic({
+        id: scenario.id,
+        agent: scenario.evalAgent || scenario.agent,
+        name: scenario.name || scenario.id,
+        result: 'error',
+        checks: 0,
+        checks_passed: 0,
+        failures: [{ type: 'llm_error', passed: false, detail: e.message }],
+      });
+    }
+  }
+
+  // Multiple runs — use three-valued classification
+  const outcomes = [];
+  const allDetails = [];
+
+  for (let i = 0; i < runs; i++) {
+    try {
+      const judgeResult = await runJudge({ agentPrompt, criteria, callLLM });
+      outcomes.push(judgeResult.passed);
+      allDetails.push(judgeResult);
+    } catch (e) {
+      outcomes.push(false);
+      allDetails.push({ passed: false, error: e.message });
+    }
+  }
+
+  const classification = classify(outcomes, thresholds);
+
+  return {
+    id: scenario.id,
+    agent: scenario.evalAgent || scenario.agent,
+    name: scenario.name || scenario.id,
+    result: classification.verdict,
+    verdict: classification.verdict,
+    confidence: classification.confidence,
+    deterministic: false,
+    checks: criteria.length,
+    checks_passed: classification.verdict === 'pass' ? criteria.length : 0,
+    runs_passed: classification.passed,
+    failures: classification.verdict === 'fail'
+      ? [{ type: 'llm_judge', passed: false, detail: classification.details }]
+      : [],
+    runs: classification.runs,
+    runDetails: allDetails,
+  };
+}
+
+module.exports = {
+  JUDGE_SYSTEM_PROMPT,
+  buildJudgePrompt,
+  parseJudgeResponse,
+  runJudge,
+  runJudgeScenario,
+};

package/lib/eval-runner.js

@@ -8,6 +8,7 @@
  * Pure helper with injected I/O (readFile, listDir).
  */
 const { runCheck } = require('./eval-graders');
+const { wrapDeterministic } = require('./eval-three-valued');
 
 /**
  * Load eval scenarios from the evals directory.
@@ -115,7 +116,7 @@ function runAllScenarios(scenarios, agentPrompts, opts = {}) {
     const promptText = agentPrompts[agentName];
 
     if (!promptText && !scenario.loadError) {
-      results.push({
+      results.push(wrapDeterministic({
         id: scenario.id,
         agent: agentName,
         name: scenario.name || scenario.id,
@@ -123,12 +124,12 @@ function runAllScenarios(scenarios, agentPrompts, opts = {}) {
         checks: 0,
         checks_passed: 0,
         failures: [{ type: 'missing_prompt', passed: false, detail: `agent "${agentName}" not found in AGENT_PROMPTS` }],
-      });
+      }));
       continue;
     }
 
     const result = runScenario(scenario, promptText || '');
-    results.push(result);
+    results.push(wrapDeterministic(result));
 
     if (opts.failFast && result.result !== 'pass') break;
   }
@@ -163,8 +164,10 @@ function formatResults(results, format = 'pretty') {
   for (const [agent, scenarios] of Object.entries(byAgent)) {
     lines.push(`${agent} (${scenarios.length} scenarios)`);
     for (const s of scenarios) {
-      const icon = s.result === 'pass' ? 'PASS' : s.result === 'fail' ? 'FAIL' : 'ERR ';
-      lines.push(`  [${icon}] ${s.id}: ${s.name} (${s.checks_passed}/${s.checks} checks)`);
+      const verdict = s.verdict || s.result;
+      const icon = verdict === 'pass' ? 'PASS' : verdict === 'fail' ? 'FAIL' : verdict === 'inconclusive' ? '????' : 'ERR ';
+      const conf = s.confidence != null && !s.deterministic ? ` ${s.confidence}%` : '';
+      lines.push(`  [${icon}] ${s.id}: ${s.name} (${s.checks_passed}/${s.checks} checks${conf})`);
       for (const f of s.failures) {
         lines.push(`         x ${f.type}: ${f.detail}`);
       }

package/lib/eval-three-valued.js

@@ -0,0 +1,158 @@
+'use strict';
+/**
+ * eval-three-valued.js — HC-019j three-valued test outcomes.
+ *
+ * Replaces binary pass/fail with Pass/Fail/Inconclusive for
+ * non-deterministic evaluations (LLM-as-judge, HC-019i).
+ *
+ * For deterministic checks (current graders), results are always
+ * definitive — Pass or Fail, never Inconclusive.
+ *
+ * For non-deterministic checks (future LLM-judge), the same eval
+ * is run N times and outcomes are classified statistically:
+ *   - Pass: >= passThreshold of runs passed (default 80%)
+ *   - Fail: >= failThreshold of runs failed (default 80%)
+ *   - Inconclusive: neither threshold met (needs more runs or investigation)
+ *
+ * Based on AgentAssay (ICLR 2026) three-valued probabilistic outcomes.
+ */
+
+/**
+ * Classify a set of run results into Pass/Fail/Inconclusive.
+ *
+ * @param {boolean[]} outcomes — array of pass/fail booleans from multiple runs
+ * @param {object} [opts]
+ * @param {number} [opts.passThreshold=0.8] — fraction of passes needed for Pass
+ * @param {number} [opts.failThreshold=0.8] — fraction of fails needed for Fail
+ * @param {number} [opts.minRuns=1] — minimum runs before classifying
+ * @returns {{ verdict: 'pass'|'fail'|'inconclusive', confidence: number, runs, passed, failed, details }}
+ */
+function classify(outcomes, opts = {}) {
+  const { passThreshold = 0.8, failThreshold = 0.8, minRuns = 1 } = opts;
+
+  if (!outcomes || outcomes.length === 0) {
+    return { verdict: 'inconclusive', confidence: 0, runs: 0, passed: 0, failed: 0, details: 'no runs' };
+  }
+
+  const runs = outcomes.length;
+  const passed = outcomes.filter(o => o === true).length;
+  const failed = runs - passed;
+  const passRate = passed / runs;
+  const failRate = failed / runs;
+
+  if (runs < minRuns) {
+    return {
+      verdict: 'inconclusive',
+      confidence: Math.round(passRate * 100),
+      runs, passed, failed,
+      details: `insufficient runs (${runs}/${minRuns})`,
+    };
+  }
+
+  // Pass-priority: if both thresholds could be met (e.g., 1 run),
+  // pass wins. This is optimistic — we assume the agent is correct
+  // unless proven otherwise with enough evidence.
+  if (passRate >= passThreshold) {
+    return {
+      verdict: 'pass',
+      confidence: Math.round(passRate * 100),
+      runs, passed, failed,
+      details: `${passed}/${runs} passed (${Math.round(passRate * 100)}% >= ${Math.round(passThreshold * 100)}% threshold)`,
+    };
+  }
+
+  if (failRate >= failThreshold) {
+    return {
+      verdict: 'fail',
+      confidence: Math.round(failRate * 100),
+      runs, passed, failed,
+      details: `${failed}/${runs} failed (${Math.round(failRate * 100)}% >= ${Math.round(failThreshold * 100)}% threshold)`,
+    };
+  }
+
+  return {
+    verdict: 'inconclusive',
+    confidence: Math.round(Math.max(passRate, failRate) * 100),
+    runs, passed, failed,
+    details: `neither threshold met: ${Math.round(passRate * 100)}% pass, ${Math.round(failRate * 100)}% fail`,
+  };
+}
+
+/**
+ * Compute Wilson score confidence interval for a pass rate.
+ * Used to determine if more runs would change the verdict.
+ *
+ * @param {number} passed — number of passes
+ * @param {number} total — total runs
+ * @param {number} [z=1.96] — z-score for confidence level (1.96 = 95%)
+ * @returns {{ lower: number, upper: number, center: number }}
+ */
+function wilsonInterval(passed, total, z = 1.96) {
+  if (total === 0) return { lower: 0, upper: 1, center: 0.5 };
+
+  const p = passed / total;
+  const denominator = 1 + z * z / total;
+  const center = (p + z * z / (2 * total)) / denominator;
+  const margin = (z * Math.sqrt((p * (1 - p) + z * z / (4 * total)) / total)) / denominator;
+
+  return {
+    lower: Math.max(0, Math.round((center - margin) * 1000) / 1000),
+    upper: Math.min(1, Math.round((center + margin) * 1000) / 1000),
+    center: Math.round(center * 1000) / 1000,
+  };
+}
+
+/**
+ * Recommend whether more runs would help resolve an inconclusive result.
+ *
+ * @param {object} result — from classify()
+ * @param {object} [opts]
+ * @param {number} [opts.maxRuns=10] — maximum recommended additional runs
+ * @returns {{ recommend: boolean, additionalRuns: number, reason: string }}
+ */
+function recommendMoreRuns(result, opts = {}) {
+  const { maxRuns = 10 } = opts;
+
+  if (result.verdict !== 'inconclusive') {
+    return { recommend: false, additionalRuns: 0, reason: 'verdict is definitive' };
+  }
+
+  if (result.runs === 0) {
+    return { recommend: true, additionalRuns: 3, reason: 'no runs yet' };
+  }
+
+  const interval = wilsonInterval(result.passed, result.runs);
+  const spread = interval.upper - interval.lower;
+
+  // If spread is wide, more runs would help narrow it
+  if (spread > 0.3 && result.runs < maxRuns) {
+    const additional = Math.min(maxRuns - result.runs, Math.ceil(result.runs * 0.5) + 2);
+    return { recommend: true, additionalRuns: additional, reason: `wide confidence interval (${interval.lower}-${interval.upper})` };
+  }
+
+  // If spread is narrow but still inconclusive, the result is genuinely borderline
+  return { recommend: false, additionalRuns: 0, reason: `borderline result (${interval.lower}-${interval.upper}), more runs unlikely to resolve` };
+}
+
+/**
+ * Wrap a deterministic eval result as a three-valued outcome.
+ * Deterministic results are always definitive (never inconclusive).
+ *
+ * @param {object} scenarioResult — from runScenario()
+ * @returns {object} — same shape with verdict + confidence added
+ */
+function wrapDeterministic(scenarioResult) {
+  let verdict;
+  if (scenarioResult.result === 'pass') verdict = 'pass';
+  else if (scenarioResult.result === 'fail') verdict = 'fail';
+  else verdict = 'error'; // error means the eval itself broke, not flaky — distinct from inconclusive
+
+  return {
+    ...scenarioResult,
+    verdict,
+    confidence: verdict === 'error' ? 0 : 100,
+    deterministic: true,
+  };
+}
+
+module.exports = { classify, wilsonInterval, recommendMoreRuns, wrapDeterministic };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hone-ai/cli",
-  "version": "1.6.0",
+  "version": "1.7.1",
   "description": "Hone AI — Enterprise SDLC Pipeline CLI",
   "main": "hone-cli.js",
   "bin": {
@@ -10,7 +10,8 @@
     "bin/",
     "hone-cli.js",
     "lib/",
-    "!lib/*.test.js"
+    "!lib/*.test.js",
+    "schema/"
   ],
   "scripts": {
     "test": "echo \"No tests yet\" && exit 0",

package/schema/metadata.schema.json

@@ -0,0 +1,134 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://hone.ai/schema/metadata.schema.json",
+  "title": "Hone SDLC Pipeline Story Metadata",
+  "description": "Schema for .github/pipeline/<STORY-ID>/metadata.yml. Implements SC-010 §10 (metadata.yml as wire protocol) — fields read by 3+ agents must validate against this schema before agents run, otherwise typos silently degrade to no-op behavior.",
+  "type": "object",
+  "additionalProperties": true,
+  "required": ["story_id", "title", "branch", "base", "steps"],
+  "properties": {
+    "story_id": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Story identifier. Hone-server convention: SC-NNN, H-NNN, RP-NNN, AU-NNN, SR-NNN, HC-NNN; OptionsFlow convention: E-NNN-X. Adopters may use any pattern."
+    },
+    "title": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Human-readable story title."
+    },
+    "issue": {
+      "type": ["integer", "string", "null"],
+      "description": "GitHub issue reference. Modern convention: integer issue number or null. Legacy convention: full GitHub issue URL string. Both accepted."
+    },
+    "branch": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Git branch name. Convention: feat/<STORY-ID>-<slug> | fix/<STORY-ID>-<slug> | chore/..."
+    },
+    "base": {
+      "type": "string",
+      "minLength": 1,
+      "description": "Base branch the story merges to. Convention: develop. Stacked story chains (story B branched off feature/A) may use feat/...-style base names. Adopter override schemas may tighten the enum."
+    },
+    "captured_at": {
+      "type": ["string", "null"],
+      "description": "ISO date when the story was captured (YYYY-MM-DD). May be null."
+    },
+    "type": {
+      "type": "string",
+      "enum": ["feature", "enhancement", "bug", "bug_fix", "chore", "refactor", "docs", "meta-epic", "fix"],
+      "description": "Story type per SC-001 classifier vocabulary. 'bug_fix' is a legacy alias for 'bug'/'fix'; both accepted."
+    },
+    "priority": {
+      "type": "string",
+      "description": "Adopter priority. Optional. No canonical enum — different stories use P0/P1/P2/P3, high/medium/low, M, low-medium, etc. Adopters with strict policies override via --schema."
+    },
+    "phase": {
+      "type": ["string", "number"],
+      "description": "Story lifecycle phase. Modern convention: string ('backlog', 'in-progress', 'blocked', 'done', 'completed'). Legacy convention: numeric phase identifier (e.g. 6.2, 2.1). Both accepted; adopter override schema may tighten."
+    },
+    "story_type": {
+      "type": "string",
+      "description": "Free-form story type label (legacy field; prefer 'type' for new stories)."
+    },
+    "fast_track": {
+      "type": "boolean",
+      "description": "True if story is on the fast-track pipeline (skip steps 0-3 gates). Per SC-001 classifier output."
+    },
+    "hot_fix": {
+      "type": "boolean",
+      "description": "True if story is on the hot-fix pipeline. Per SC-001 classifier output."
+    },
+    "fix_for": {
+      "type": ["string", "null"],
+      "description": "Story ID this fixes (for regression-test policy per H-030 + SC-009 §Guardrail-before-fix). Null when this story is not a fix."
+    },
+    "parent_story": {
+      "type": ["string", "null"],
+      "description": "Parent story ID for sub-stories or follow-ups. Null at top level."
+    },
+    "sibling_pipelines": {
+      "type": ["array", "null"],
+      "items": { "type": "string" },
+      "description": "Story IDs sharing this pipeline. Used for multi-story epics."
+    },
+    "author": {
+      "type": ["string", "null"]
+    },
+    "created": {
+      "type": ["string", "null"]
+    },
+    "base_sha": {
+      "type": ["string", "number", "null"],
+      "description": "Base commit SHA. Permissive type — hex-like SHA strings (92192e7) get parsed as numbers by js-yaml. Adopters who want strict SHA strings can override schema."
+    },
+    "steps": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Pipeline step status table. Wire-protocol object — no extra step IDs allowed.",
+      "properties": {
+        "step_0":  { "$ref": "#/$defs/step" },
+        "step_1":  { "$ref": "#/$defs/step" },
+        "step_2":  { "$ref": "#/$defs/step" },
+        "step_3a": { "$ref": "#/$defs/step" },
+        "step_3b": { "$ref": "#/$defs/step" },
+        "step_4":  { "$ref": "#/$defs/step" },
+        "step_5":  { "$ref": "#/$defs/step" },
+        "step_5b": { "$ref": "#/$defs/step" },
+        "step_5c": { "$ref": "#/$defs/step" }
+      }
+    },
+    "cross_validation": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "Cross-validation findings per H-035 + SC-001..SC-005."
+    },
+    "self_applied_classifier": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "SC-001 classifier output recorded for the story. Tracks the classification decision so future readers can audit the routing."
+    }
+  },
+  "$defs": {
+    "step": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "Per-step status. The wire-protocol fields (status, agent) are validated; adopter and historical extension fields (acceptance_criteria_count, acs_met, automation_rationale, bug_caught_in_step, etc.) are passed through. Set additionalProperties: false in your adopter override schema for stricter enforcement.",
+      "properties": {
+        "status": {
+          "type": "string",
+          "description": "Step status. Hone-server uses 'completed'/'in_progress'/'skipped'; older files may use 'complete'/'in-progress' — both accepted. Adopter override schemas may tighten the enum."
+        },
+        "agent": {
+          "type": "string",
+          "description": "Agent ID (story-groomer, implementation-planner, unit-test-writer, e2e-qa-planner, e2e-test-spec-writer, code-builder, code-reviewer, delivery-architect, etc.)."
+        },
+        "artifact": {
+          "type": ["string", "null"],
+          "description": "Path or filename of the step's artifact (e.g., step-0-grooming.md). May be null if the step is pending."
+        }
+      }
+    }
+  }
+}