@hone-ai/cli 1.17.0 → 1.19.0

package/lib/skill-eval-runner.js

@@ -0,0 +1,667 @@
+'use strict';
+/**
+ * skill-eval-runner.js — HC-010d-followup-1 runtime executor for the
+ * `eval-scenarios.json` artifacts that HC-010d emits next to every
+ * derived `<stack>-developer/SKILL.md` and `<stack>-architect/SKILL.md`.
+ *
+ * Two-system clarification (memory: [Don't Confuse hone eval with hone
+ * skill-eval]):
+ *
+ *   - `hone eval` (HC-019d, shipped): grades AGENT PROMPTS deterministically
+ *     against `evals/<agent>/*.eval.yml`. Zero LLM tokens. Tests prompt
+ *     QUALITY (do the prompt's instructions cover the contract?).
+ *
+ *   - `hone skill-eval` (THIS file, HC-010d-followup-1): grades DERIVED
+ *     SKILL OUTPUTS against `eval-scenarios.json`. Calls an LLM with the
+ *     skill content as system prompt + scenario.input as user, scores the
+ *     response. Tests skill EFFECTIVENESS (do the derived patterns
+ *     actually make the LLM produce correct outputs?).
+ *
+ * They answer different questions and coexist; do not unify.
+ *
+ * Design constraints:
+ *
+ *   1. Pure helper. No I/O outside `loadSkillEvalScenarios` (which only
+ *      reads adopter-supplied paths). The LLM call is injected as
+ *      `callLLM(systemPrompt, userPrompt) => Promise<string>` so the
+ *      executor unit-tests against stubs.
+ *
+ *   2. Never throws on user input. validateScenarios already enforces
+ *      shape; this module's loader and scorer return error envelopes
+ *      `{ ok: false, errors: [...] }` rather than throwing — same
+ *      contract memory: [eval-scenarios.js validator never throws].
+ *
+ *   3. Scoring uses BOTH expected_output_keywords (all must appear,
+ *      case-insensitive) AND expected_output_format (heuristic regex
+ *      for the documented format families). Either gate failing →
+ *      scenario fails.
+ *
+ *   4. Format heuristics are deliberately LOOSE — they detect "shape"
+ *      not correctness. A scenario whose keywords pass but format
+ *      doesn't match is a stronger signal than either alone; both
+ *      gates together cut false-positives from comments-that-mention-
+ *      the-keyword-without-implementing-it (the anti-failure mode the
+ *      HC-010d prompt warned about).
+ *
+ *   5. Zero-or-near-zero-cost path required (memory: [Pipeline LLM
+ *      Cost Reduction]). CLI defaults to gh-models (free GH PAT
+ *      inference); Claude is opt-in via --provider claude.
+ */
+
+// ── Output-format heuristics ─────────────────────────────────────
+//
+// One regex per allowed format. Each captures the structural shape of
+// the format — NOT semantic correctness. The intent is:
+//
+//   - If keywords match BUT format doesn't, the LLM probably wrote
+//     prose mentioning the keywords (anti-pattern HC-010d warns about).
+//   - If format matches but keywords don't, the LLM wrote the right
+//     SHAPE of code but missed the patterns the skill should encode.
+//   - Both must match for pass.
+//
+// Heuristics are intentionally loose — adopters can disagree about
+// indentation, brace style, etc. The format gate's job is to reject
+// "wrong category of output entirely" (e.g., expected apex_method got
+// markdown prose), not to grade code quality.
+//
+// Adding a new format: extend ALLOWED_OUTPUT_FORMATS in
+// `server/src/services/eval-scenarios.js` AND add a heuristic here.
+// Missing heuristic falls through to a generic "non-empty" check so
+// validation doesn't break, but the scenario's format gate becomes
+// "any output passes" — log via the unknownFormat field.
+//
+// Pass-2 review caught that the original heuristics were too loose:
+// `apex_class: /\bclass \w+/i` matched "Python class MyClass:" or
+// natural prose like "My favorite class FooBar in school"; `apex_method:
+// /\b\w+\s+\w+\s*\(/` matched any C-style function declaration across
+// many languages; `sql_query: /SELECT[\s\S]+?FROM/i` matched prose like
+// "Please select these items from the menu". Combined with the
+// substring-includes keyword check, the AND-gate effectively passed
+// markdown prose containing the right vocabulary — the exact failure
+// mode HC-010d set out to prevent.
+//
+// Tightened design:
+//   - Each pattern is anchored to start-of-line via `^...` + the `m`
+//     flag. Code declarations almost always start at a line boundary;
+//     prose almost never does.
+//   - Each pattern requires the structural elements a language demands
+//     for a declaration: access modifiers (Apex/Java/C#), JSDoc tags
+//     (NetSuite), required keyword sequences (`func` for Go, `def`
+//     for Python).
+//   - Patterns that previously used `i` are case-sensitive where the
+//     target syntax is case-sensitive in practice (function/class
+//     keywords are lowercase in JS/TS/Python/Go/Java/Apex/C#; SQL
+//     verbs are uppercase by convention in migrations + queries).
+//
+// Each entry's "rejects" comment lists the adversarial prose case the
+// behavioral suite drives — so a future loosening that re-admits prose
+// fails a named test.
+const OUTPUT_FORMAT_HEURISTICS = Object.freeze({
+  // ── Salesforce ─────────────────────────────────────────────────
+  //
+  // Apex is case-INSENSITIVE at the language level (the runtime
+  // accepts `PUBLIC CLASS` and `public class` equivalently). The
+  // discriminator is the structural shape — line-anchor + access
+  // modifier — not case. Rejects "Python class MyClass:" (no Apex
+  // access modifier at line start), prose mentioning "class FooCtrl"
+  // (no line anchor), and Java/C# class declarations (would still
+  // match the structural shape, but they declare java_class /
+  // csharp_method via their own heuristics).
+  //
+  // Pass-3 review caught that the bare `class \w+` shape still matched
+  // prose like "public class definitions in Apex" — `definitions`
+  // satisfies `\w+`. Fix: require structural elements that follow a
+  // real class declaration — either an `extends`/`implements` clause
+  // OR an opening `{` (with optional whitespace/newline before).
+  //
+  // Pass-4 review caught: bare `class \w+` rejected generic class
+  // declarations like `public class Pair<T, U> { ... }` because the
+  // next char after `\w+` is `<`, not `\s+extends/implements` or
+  // `\s*{`. Apex utility libraries routinely declare generic
+  // wrappers (Pair, Wrapper, Result). Fix: insert optional
+  // `(?:<[\w,\s.<>]+>)?` after the class name to accept type
+  // parameters.
+  apex_class:
+    /^[ \t]*(?:@\w+\s+)*(?:public|global|private|protected)\s+(?:with\s+sharing\s+|without\s+sharing\s+|inherited\s+sharing\s+)?(?:virtual\s+|abstract\s+)?class\s+\w+(?:<[\w,\s.<>]+>)?(?:\s+(?:extends|implements)\s+[\w,\s.<>]+)?\s*\{/im,
+  // Apex method: line-anchor + access modifier + return type + name +
+  // params + opening brace. Pass-3 review caught two false negatives:
+  //   (a) Inner-class methods that inherit access from the enclosing
+  //       class — `class Inner { void doThing(String s) { ... } }`.
+  //       Valid Apex but original regex required an explicit access
+  //       modifier on every method. Pass-3 adds an alternative branch
+  //       that matches "indented method without modifier" — INDENT
+  //       required so top-level prose without modifier doesn't slip
+  //       through.
+  //   (b) Legacy `testMethod` keyword: `public static testMethod void
+  //       testFoo() { ... }`. Pass-3 allows `testMethod` in the
+  //       optional-modifier slot alongside override/virtual.
+  // Pass-4 review caught: return-type token class `[\w<>,\[\]]+`
+  // does NOT include whitespace, so nested generics with internal
+  // spaces (`Map<String, List<Account>>`) are rejected. Apex
+  // SObject-collection idioms (Map<Id, List<Account>>, etc.) are
+  // ubiquitous. Fix: include `\s` in the return-type class
+  // (`[\w<>,\s\[\]]+`) — mirrors java_method's already-correct
+  // class. Applied to BOTH alternation branches.
+  apex_method:
+    /^[ \t]*(?:@\w+\s+)*(?:public|global|private|protected)\s+(?:static\s+)?(?:override\s+|virtual\s+|testMethod\s+)?[\w<>,\s\[\]]+\s+\w+\s*\([^)]*\)\s*\{|^[ \t]{2,}(?:@\w+\s+)*(?:static\s+)?(?:override\s+|virtual\s+|testMethod\s+)?[\w<>,\s\[\]]+\s+\w+\s*\([^)]*\)\s*\{/im,
+  // Apex test: requires @isTest at line-start OR Test.startTest() at
+  // line-start. Pass-3 caught that the second alternative was not
+  // line-anchored, so prose "Use Test.startTest() to invoke test
+  // methods" matched. Both branches now require `^[ \t]*`.
+  apex_test:
+    /^[ \t]*@isTest\b|^[ \t]*Test\.startTest\s*\(/im,
+  // LWC requires BOTH the template tag AND the LightningElement
+  // class — prose mentioning "extends LightningElement" alone is not a
+  // component. Pass-3 caught that the `<template>...</template>`
+  // branch was not line-anchored — inline HTML in prose matched. Both
+  // branches now require `^[ \t]*` (templates almost always start at
+  // line boundaries in real .html files / template literals).
+  lwc_component:
+    /^[ \t]*<template\b[\s\S]+?<\/template>|^[ \t]*(?:export\s+default\s+)?class\s+\w+\s+extends\s+LightningElement\b/m,
+  visualforce_page:
+    /<apex:page\b[^>]*>/i,
+
+  // ── NetSuite ───────────────────────────────────────────────────
+  //
+  // SuiteScript 2.x modules require either an AMD-style define([...])
+  // OR a JSDoc with @NApiVersion/@NScriptType. The original
+  // `\bN/(record|search|...)` was too permissive (any prose mentioning
+  // "N/A" matched).
+  suitescript_module:
+    /^[ \t]*define\s*\(\s*\[\s*['"]N\/|@NApiVersion\b|@NScriptType\b/m,
+  suitescript_restlet:
+    /@NScriptType\s+Restlet\b/,
+  // UserEvent requires the JSDoc tag at module-doc level. beforeSubmit/
+  // afterSubmit alone are also valid in NetSuite contexts but match too
+  // broadly (any JS file with those function names); require the
+  // JSDoc tag.
+  suitescript_userevent:
+    /@NScriptType\s+UserEventScript\b/,
+  // SuiteQL requires line-anchored SELECT (rejects prose "please
+  // select these items from the menu") + FROM + an identifier.
+  suiteql_query:
+    /^[ \t]*SELECT\s+[\s\S]+?\s+FROM\s+\w+/im,
+
+  // ── Node / TS / Java / Python / C# / Go ───────────────────────
+  //
+  // js_function: line-anchored function declaration OR const arrow.
+  // Rejects prose "Use a function to handle the callback" or "event
+  // handler: callback => promise" inside an explanation.
+  js_function:
+    /^[ \t]*(?:async\s+)?function\s+\w+\s*\(|^[ \t]*(?:export\s+)?(?:const|let|var)\s+\w+\s*=\s*(?:async\s+)?\([^)]*\)\s*=>/m,
+  // ts_function: same as js_function but optionally allows type
+  // annotations on params and return type.
+  ts_function:
+    /^[ \t]*(?:export\s+)?(?:async\s+)?function\s+\w+\s*(?:<[^>]+>)?\s*\(|^[ \t]*(?:export\s+)?(?:const|let|var)\s+\w+(?:\s*:\s*[\w<>,\s\[\]|]+)?\s*=\s*(?:async\s+)?\([^)]*\)(?:\s*:\s*[\w<>,\s\[\]|]+)?\s*=>/m,
+  // Java class: line-anchored + access modifier. Pass-3 review caught
+  // that bare `class \w+` matched prose like "public class definitions"
+  // — `definitions` satisfies `\w+`. Fix: require `extends`/`implements`
+  // clause OR an opening `{` so the regex distinguishes a declaration
+  // from a noun phrase.
+  // Pass-4 review caught: same generic-class issue as apex_class —
+  // bare `class \w+` rejected `public class Pair<T, U> { ... }`. Fix:
+  // optional `(?:<[\w,\s.<>]+>)?` after class name.
+  java_class:
+    /^[ \t]*(?:@\w+\s+)*(?:public|private|protected)\s+(?:final\s+|abstract\s+|static\s+)*class\s+\w+(?:<[\w,\s.<>]+>)?(?:\s+(?:extends|implements)\s+[\w,\s.<>]+)?\s*\{/m,
+  // Java method: line-anchored + access modifier + return type.
+  // Pass-3 review caught: original required `\{` ending, rejecting
+  // valid interface methods like `public abstract Foo bar();`. Now
+  // accepts EITHER opening brace (concrete method) OR semicolon
+  // (abstract/interface method).
+  java_method:
+    /^[ \t]*(?:@\w+\s+)*(?:public|private|protected)\s+(?:static\s+|final\s+|synchronized\s+|abstract\s+|default\s+)*[\w<>,\s\[\]]+\s+\w+\s*\([^)]*\)\s*(?:throws\s+[\w,\s]+)?\s*[{;]/m,
+  // Python: already anchored; keep but tighten the trailing colon
+  // requirement so "def foo(x" half-written prose doesn't match.
+  python_function:
+    /^[ \t]*(?:async\s+)?def\s+\w+\s*\([^)]*\)\s*(?:->\s*[\w\[\],\s]+\s*)?:/m,
+  python_class:
+    /^[ \t]*class\s+\w+(?:\s*\([^)]*\))?\s*:/m,
+  // C# method: line-anchored + access modifier.
+  csharp_method:
+    /^[ \t]*(?:\[[\w()=,"' \t]+\]\s*)*(?:public|private|protected|internal)\s+(?:static\s+|async\s+|override\s+|virtual\s+|abstract\s+)*[\w<>,\s\[\]?]+\s+\w+\s*\([^)]*\)\s*\{/m,
+  // Go func: line-anchored, requires `func`. Optional receiver group.
+  go_function:
+    /^[ \t]*func\s+(?:\(\s*\w+\s+\*?\w+\s*\)\s+)?\w+\s*\(/m,
+
+  // ── Data / infra ───────────────────────────────────────────────
+  //
+  // SQL queries: line-anchored verb. Uppercase preferred by
+  // convention, but real-world code uses both; case-insensitive at
+  // line start is acceptable. Rejects prose "please select all from
+  // the available options".
+  sql_query:
+    /^[ \t]*(?:SELECT\s+[\s\S]+?\s+FROM|INSERT\s+INTO|UPDATE\s+\w+\s+SET|DELETE\s+FROM)\b/im,
+  // SOQL: line-anchored SELECT + FROM + Salesforce object (standard
+  // or __c suffix).
+  soql_query:
+    /^[ \t]*SELECT\s+[\s\S]+?\s+FROM\s+\w+(?:__c)?\b/im,
+  // SQL migration: line-anchored DDL.
+  sql_migration:
+    /^[ \t]*(?:CREATE\s+(?:TABLE|INDEX|FUNCTION|TYPE|VIEW|MATERIALIZED\s+VIEW)|ALTER\s+TABLE|DROP\s+(?:TABLE|INDEX))\b/im,
+  // dbt: Jinja2 template tags are distinctive — prose can't easily
+  // contain `{{ config(...)`.
+  dbt_model:
+    /\{\{\s*(?:config|ref|source)\s*\(/,
+  terraform_module:
+    /^[ \t]*(?:resource|module|data)\s+"[\w_-]+"/m,
+  terraform_resource:
+    /^[ \t]*resource\s+"[\w_-]+"\s+"[\w_-]+"\s*\{/m,
+
+  // ── Config + prose ─────────────────────────────────────────────
+  //
+  // config_yaml requires at least 2 lines of key-value pairs to
+  // distinguish from a single inline `key: value` mention. Pass-3
+  // review caught that the original `\n[ \t]+` required the SECOND
+  // key to be INDENTED — but real top-level YAML (Kubernetes
+  // manifests, GitHub workflows) has all keys at column 0. Pass-3
+  // accepts EITHER indented (nested) OR column-0 (top-level) second
+  // keys via `[ \t]*` (greedy zero-or-more) instead of `[ \t]+`
+  // (one-or-more).
+  config_yaml:
+    /^[ \t]*[\w-]+\s*:\s*\S[\s\S]*?\n[ \t]*[\w-]+\s*:/m,
+  // config_json: line-anchored object literal with at least one
+  // quoted key.
+  config_json:
+    /^[ \t]*\{[\s\S]*?"[\w-]+"\s*:/m,
+  // Prose formats need a length floor — single-paragraph responses
+  // don't count as design docs even if they mention the keywords.
+  markdown_explanation:
+    /(?:^|\n)#{1,6}\s+\S[\s\S]{50,}/,
+  design_doc:
+    /(?:^|\n)#{1,3}\s+\S[\s\S]{200,}/,
+});
+
+/**
+ * Run the format heuristic for a given expected_output_format. Returns
+ * { matched: boolean, unknownFormat: boolean }. Unknown formats fall
+ * through to a "non-empty after trim" check + a flag so the executor
+ * can warn the operator that the format gate degraded to "any output."
+ *
+ * @param {string} format — value from scenario.expected_output_format
+ * @param {string} response — LLM output
+ * @returns {{ matched: boolean, unknownFormat: boolean }}
+ */
+function matchFormat(format, response) {
+  if (typeof response !== 'string') return { matched: false, unknownFormat: false };
+  const rx = OUTPUT_FORMAT_HEURISTICS[format];
+  if (!rx) {
+    // Degraded check: any non-empty response. Caller flags via
+    // unknownFormat so the warning surfaces. (This branch shouldn't
+    // hit in practice — validateScenarios rejects unknown formats —
+    // but defending against drift between the validator's allowlist
+    // and this module's heuristic table is cheap.)
+    return { matched: response.trim().length > 0, unknownFormat: true };
+  }
+  return { matched: rx.test(response), unknownFormat: false };
+}
+
+/**
+ * Score a single scenario's LLM response. Pure function; both gates
+ * (keywords + format) must pass.
+ *
+ * Keyword matching is CASE-INSENSITIVE substring (the prompt asks for
+ * "keywords that ONLY co-occur in a real solution," not exact-case
+ * identifiers — LLMs lowercase + paraphrase freely).
+ *
+ * @param {object} scenario — already validated via validateScenarios
+ * @param {string} llmResponse
+ * @returns {{
+ *   passed: boolean,
+ *   keywordHits: string[],
+ *   keywordMisses: string[],
+ *   formatMatched: boolean,
+ *   unknownFormat: boolean,
+ *   failures: string[]
+ * }}
+ */
+function scoreScenario({ scenario, llmResponse }) {
+  const responseLower = (llmResponse || '').toLowerCase();
+  const hits = [];
+  const misses = [];
+  for (const kw of scenario.expected_output_keywords) {
+    if (responseLower.includes(kw.toLowerCase())) hits.push(kw);
+    else misses.push(kw);
+  }
+  const { matched: formatMatched, unknownFormat } = matchFormat(
+    scenario.expected_output_format,
+    llmResponse,
+  );
+
+  const failures = [];
+  if (misses.length > 0) {
+    failures.push(`missing keywords: ${misses.join(', ')}`);
+  }
+  if (!formatMatched) {
+    failures.push(`format heuristic for "${scenario.expected_output_format}" did not match`);
+  }
+  return {
+    passed: failures.length === 0,
+    keywordHits: hits,
+    keywordMisses: misses,
+    formatMatched,
+    unknownFormat,
+    failures,
+  };
+}
+
+/**
+ * Run a single scenario end-to-end: call LLM with skill as system
+ * prompt + scenario.input as user, then score. Errors from the LLM
+ * call are captured into the result (does NOT throw).
+ *
+ * @param {object} opts
+ * @param {string} opts.skillContent — the SKILL.md body
+ * @param {object} opts.scenario     — validated scenario
+ * @param {(sys: string, user: string) => Promise<string>} opts.callLLM
+ * @returns {Promise<object>} — scenario result with verdict + diagnostics
+ */
+async function runSkillEvalScenario({ skillContent, scenario, callLLM }) {
+  const systemPrompt = skillContent;
+  const userPrompt   = scenario.input;
+  let llmResponse, llmError = null;
+  try {
+    llmResponse = await callLLM(systemPrompt, userPrompt);
+  } catch (e) {
+    llmError = e?.message || String(e);
+    llmResponse = '';
+  }
+  // Result envelope — keep shape identical on happy + error paths so
+  // formatResults never has to branch. Pass-2 review caught a missing
+  // `expected_output_format` field that surfaced as a misleading
+  // "format heuristic missing for \"security\"" warning (the format
+  // name is what the operator needs, not the category).
+  const baseEnvelope = {
+    id:                    scenario.id,
+    name:                  scenario.name,
+    category:              scenario.category,
+    expected_output_format: scenario.expected_output_format,
+    // Preserve tags on every code path — per-tag aggregation in
+    // runAllSkillScenarios is the operator's "did this HC-010c rule's
+    // eval succeed?" view. Dropping tags on error would silently
+    // under-count rule coverage exactly when a rule fails in a noisy
+    // way (LLM rate limit, transient network), making the error look
+    // like "rule not exercised."
+    tags:                  scenario.tags || [],
+  };
+  if (llmError) {
+    return {
+      ...baseEnvelope,
+      result:        'error',
+      passed:        false,
+      llmError,
+      keywordHits:   [],
+      keywordMisses: scenario.expected_output_keywords.slice(),
+      formatMatched: false,
+      unknownFormat: false,
+      failures:      [`LLM call failed: ${llmError}`],
+      response:      '',
+    };
+  }
+  const scored = scoreScenario({ scenario, llmResponse });
+  return {
+    ...baseEnvelope,
+    result:        scored.passed ? 'pass' : 'fail',
+    passed:        scored.passed,
+    llmError:      null,
+    keywordHits:   scored.keywordHits,
+    keywordMisses: scored.keywordMisses,
+    formatMatched: scored.formatMatched,
+    unknownFormat: scored.unknownFormat,
+    failures:      scored.failures,
+    response:      llmResponse,
+  };
+}
+
+/**
+ * Run every scenario. Supports fail-fast + a progress callback so the
+ * CLI can stream "running N of M..." messages.
+ *
+ * @param {object} opts
+ * @param {object[]} opts.scenarios
+ * @param {string}   opts.skillContent
+ * @param {Function} opts.callLLM
+ * @param {boolean}  [opts.failFast=false]
+ * @param {Function} [opts.onProgress] — called with (current, total, lastResult)
+ * @returns {Promise<{
+ *   total: number,
+ *   passed: number,
+ *   failed: number,
+ *   errors: number,
+ *   results: object[],
+ *   perCategory: Record<string, { total: number, passed: number, failed: number }>,
+ *   perTag:      Record<string, { total: number, passed: number, failed: number }>
+ * }>}
+ */
+async function runAllSkillScenarios({ scenarios, skillContent, callLLM, failFast = false, onProgress }) {
+  const results = [];
+  for (let i = 0; i < scenarios.length; i++) {
+    const r = await runSkillEvalScenario({
+      skillContent, scenario: scenarios[i], callLLM,
+    });
+    results.push(r);
+    if (typeof onProgress === 'function') {
+      onProgress(i + 1, scenarios.length, r);
+    }
+    if (failFast && r.result !== 'pass') break;
+  }
+
+  const perCategory = {};
+  const perTag      = {};
+  for (const r of results) {
+    perCategory[r.category] = perCategory[r.category]
+      || { total: 0, passed: 0, failed: 0, errors: 0 };
+    perCategory[r.category].total += 1;
+    perCategory[r.category][r.result === 'pass' ? 'passed'
+                          : r.result === 'fail' ? 'failed'
+                          : 'errors'] += 1;
+    for (const tag of (r.tags || [])) {
+      perTag[tag] = perTag[tag] || { total: 0, passed: 0, failed: 0, errors: 0 };
+      perTag[tag].total += 1;
+      perTag[tag][r.result === 'pass' ? 'passed'
+                : r.result === 'fail' ? 'failed'
+                : 'errors'] += 1;
+    }
+  }
+
+  return {
+    total:  results.length,
+    passed: results.filter(r => r.result === 'pass').length,
+    failed: results.filter(r => r.result === 'fail').length,
+    errors: results.filter(r => r.result === 'error').length,
+    results,
+    perCategory,
+    perTag,
+  };
+}
+
+/**
+ * Locate + load a skill's SKILL.md + adjacent eval-scenarios.json,
+ * validating the JSON via the shared validator. Pure function w/r/t
+ * arguments — all filesystem access goes through injected helpers so
+ * tests can drive against in-memory fixtures.
+ *
+ * Path resolution mirrors HC-019y AI-tool path mapping:
+ *   1. `<repoRoot>/.claude/agents/<skill>/SKILL.md`   (HC-019y canonical)
+ *   2. `<repoRoot>/.github/skills/<skill>/SKILL.md`   (legacy)
+ *   3. `<repoRoot>/.github/skills/<skill>.md`         (older legacy)
+ *
+ * eval-scenarios.json must be ADJACENT to the SKILL.md.
+ *
+ * @param {object} opts
+ * @param {string} opts.repoRoot
+ * @param {string} opts.skillName
+ * @param {object} opts.fs — node:fs subset { existsSync, readFileSync }
+ * @param {object} opts.path — node:path
+ * @param {Function} opts.validateScenarios — server/src/services/eval-scenarios.js
+ * @returns {{
+ *   ok: true,
+ *   scenarios: object[],
+ *   skill: string,
+ *   skillContent: string,
+ *   skillPath: string,
+ *   scenariosPath: string,
+ * } | {
+ *   ok: false,
+ *   errors: { path: string, message: string }[],
+ * }}
+ */
+function loadSkillEvalScenarios({ repoRoot, skillName, fs, path, validateScenarios }) {
+  // Search order — first match wins. Each candidate is a (skillPath,
+  // dir) pair; the adjacent JSON lives in dir + 'eval-scenarios.json'.
+  const candidates = [
+    {
+      skill:     path.join(repoRoot, '.claude', 'agents', skillName, 'SKILL.md'),
+      scenarios: path.join(repoRoot, '.claude', 'agents', skillName, 'eval-scenarios.json'),
+    },
+    {
+      skill:     path.join(repoRoot, '.github', 'skills', skillName, 'SKILL.md'),
+      scenarios: path.join(repoRoot, '.github', 'skills', skillName, 'eval-scenarios.json'),
+    },
+    {
+      skill:     path.join(repoRoot, '.github', 'skills', `${skillName}.md`),
+      scenarios: path.join(repoRoot, '.github', 'skills', `${skillName}.eval-scenarios.json`),
+    },
+  ];
+
+  let found = null;
+  for (const c of candidates) {
+    if (fs.existsSync(c.skill)) {
+      found = c;
+      break;
+    }
+  }
+  if (!found) {
+    return {
+      ok: false,
+      errors: [{
+        path: '$',
+        message: `SKILL.md for "${skillName}" not found under .claude/agents/ or .github/skills/. ` +
+                 `Run \`hone derive\` to generate it.`,
+      }],
+    };
+  }
+  if (!fs.existsSync(found.scenarios)) {
+    return {
+      ok: false,
+      errors: [{
+        path: '$',
+        message: `eval-scenarios.json not found adjacent to ${found.skill}. ` +
+                 `Re-run \`hone derive\` to emit it (HC-010d Phase 2d).`,
+      }],
+    };
+  }
+
+  let raw, parsed;
+  try {
+    raw = fs.readFileSync(found.scenarios, 'utf8');
+  } catch (e) {
+    return {
+      ok: false,
+      errors: [{ path: '$', message: `failed to read ${found.scenarios}: ${e.message}` }],
+    };
+  }
+  try {
+    parsed = JSON.parse(raw);
+  } catch (e) {
+    return {
+      ok: false,
+      errors: [{ path: '$', message: `${found.scenarios}: JSON parse failed — ${e.message}` }],
+    };
+  }
+  const v = validateScenarios(parsed);
+  if (!v.ok) return v;
+
+  let skillContent;
+  try {
+    skillContent = fs.readFileSync(found.skill, 'utf8');
+  } catch (e) {
+    return {
+      ok: false,
+      errors: [{ path: '$', message: `failed to read ${found.skill}: ${e.message}` }],
+    };
+  }
+  if (skillContent.trim().length === 0) {
+    return {
+      ok: false,
+      errors: [{ path: '$', message: `${found.skill} is empty — nothing to evaluate` }],
+    };
+  }
+
+  return {
+    ok: true,
+    scenarios:     v.scenarios,
+    skill:         parsed.skill,
+    skillContent,
+    skillPath:     found.skill,
+    scenariosPath: found.scenarios,
+  };
+}
+
+/**
+ * Pretty / JSON renderer. Mirrors `formatResults` in eval-runner.js so
+ * CLI users get consistent output across `hone eval` and `hone skill-eval`.
+ *
+ * @param {object} summary — output of runAllSkillScenarios
+ * @param {string} fmt     — 'pretty' | 'json'
+ * @returns {string}
+ */
+function formatResults(summary, fmt = 'pretty') {
+  if (fmt === 'json') return JSON.stringify(summary, null, 2);
+
+  const lines = [];
+  lines.push('Hone Skill Eval — Scenario Results');
+  lines.push('===================================');
+  lines.push('');
+  for (const r of summary.results) {
+    const icon = r.result === 'pass'  ? '✓'    // ✓
+               : r.result === 'fail'  ? '✗'    // ✗
+               : '!';
+    const label = r.tags && r.tags.length
+      ? `${r.id}  [${r.tags.join(', ')}]`
+      : r.id;
+    lines.push(`${icon} ${label} — ${r.name}`);
+    // Default failures to [] defensively. Pass-2 review caught that
+    // a synthetic summary missing the `failures` field (e.g., from a
+    // dashboard mock or aggregator) crashed the renderer with
+    // "r.failures is not iterable". Public exports should tolerate
+    // partial inputs.
+    if (r.result !== 'pass') {
+      for (const f of (r.failures || [])) lines.push(`    - ${f}`);
+    }
+    if (r.unknownFormat) {
+      // Pass-2 review caught: printed r.category (security /
+      // correctness / etc.) instead of the actual format name. The
+      // operator needs to know WHICH format the validator added that
+      // the executor's heuristic table doesn't cover.
+      const fmtName = r.expected_output_format || '<format name missing from result envelope>';
+      lines.push(`    ⚠ format heuristic missing for "${fmtName}" — gate degraded to non-empty check`);
+    }
+  }
+  lines.push('');
+  lines.push(`Total: ${summary.total}  pass: ${summary.passed}  fail: ${summary.failed}  error: ${summary.errors}`);
+  if (Object.keys(summary.perCategory).length > 0) {
+    lines.push('');
+    lines.push('By category:');
+    for (const [cat, c] of Object.entries(summary.perCategory)) {
+      lines.push(`  ${cat.padEnd(16)}  ${c.passed}/${c.total} pass`);
+    }
+  }
+  if (Object.keys(summary.perTag).length > 0) {
+    lines.push('');
+    lines.push('By tag (HC-010c rule id):');
+    for (const [tag, c] of Object.entries(summary.perTag)) {
+      lines.push(`  ${tag.padEnd(20)}  ${c.passed}/${c.total} pass`);
+    }
+  }
+  return lines.join('\n');
+}
+
+module.exports = {
+  OUTPUT_FORMAT_HEURISTICS,
+  matchFormat,
+  scoreScenario,
+  runSkillEvalScenario,
+  runAllSkillScenarios,
+  loadSkillEvalScenarios,
+  formatResults,
+};