@damiangilgonzalez/harness 1.3.7

package/scripts/drift-check.js

@@ -0,0 +1,354 @@
+#!/usr/bin/env node
+// drift-check.js — do the paths the harness claims exist actually exist?
+//
+// The harness writes knowledge about itself in three places and every one of them can name a
+// file that has since moved. Measured before this existed: 67 of 688 markdown links in the
+// published catalog did not resolve, and two were real wrong-depth bugs shipping to users
+// (`data-cleaning` → duckdb, `solid-js` → vercel, both missing a `../`). The private wiki had a
+// plan marked "implementada" naming three files that never existed. Nothing checked any of it,
+// while the memory instructions already ordered "verify it still exists before recommending it"
+// — a binding rule resting on the model remembering. That is the pattern P2 exists to kill.
+//
+// THE ONE IDEA THAT MAKES THIS VIABLE: **code shows, prose points.** A link inside a fenced
+// block or an inline-code span is being *displayed* as syntax; a link in prose is *offered to
+// follow*. That distinction is already in the markup, so it needs no markers to author and no
+// allowlist to maintain (P3 — the content is the ledger). Applied to the catalog it takes 67
+// unresolved links down to 4 real ones, which is the difference between a gate people act on
+// and a gate they learn to ignore.
+//
+// Two modes, because the precision a check needs scales with what it costs to be wrong (P7):
+//   catalog   blocking. Prose links only, resolved exactly as a markdown reader would (relative
+//             to the document). Precision first: a false positive here fails a build.
+//   knowledge report-only. Prose links PLUS paths in inline code, resolved against several
+//             plausible bases. Recall first: a human triages, nothing blocks, and the wiki's
+//             worst finding lives in backticks — precision-only extraction would miss it.
+//
+// It never writes. The output is a report; adopting it is a human's job.
+import { readdirSync, readFileSync, existsSync, realpathSync } from 'node:fs';
+import { join, dirname, resolve, relative, basename, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { homedir } from 'node:os';
+
+const REPO = join(dirname(fileURLToPath(import.meta.url)), '..');
+
+// ── extraction ────────────────────────────────────────────────────────────────────────────
+// Both strippers keep the line count (and stripInlineCode the column offsets) intact, so a
+// finding can name the line a human has to open.
+
+// Fenced blocks, line-scanned rather than regexed: an UNTERMINATED fence must swallow the rest
+// of the file, which a paired regex silently declines to do — leaving example links in a
+// half-open block looking like prose.
+export function stripFences(text) {
+  const out = [];
+  let fence = null;
+  for (const line of String(text).split('\n')) {
+    const m = line.match(/^([ \t]*)(`{3,}|~{3,})(.*)$/);
+    if (fence) {
+      if (m && m[2][0] === fence.char && m[2].length >= fence.len && m[3].trim() === '') fence = null;
+      out.push('');
+      continue;
+    }
+    if (m) {
+      fence = { char: m[2][0], len: m[2].length };
+      out.push('');
+      continue;
+    }
+    out.push(line);
+  }
+  return out.join('\n');
+}
+
+// Inline code spans, per line so a span can never eat across a newline. An UNBALANCED backtick
+// run is left as prose on purpose: swallowing it would hide a real link, and a false negative
+// is the failure mode this whole check exists to prevent.
+export function stripInlineCode(text) {
+  return String(text)
+    .split('\n')
+    .map((line) => line.replace(/(`+)(?:(?!\1)[^`])*\1/g, (m) => ' '.repeat(m.length)))
+    .join('\n');
+}
+
+const FILE_EXT = /\.(md|markdown|ya?ml|json|jsonc|sh|bash|mjs|cjs|js|ts|txt|toml)$/i;
+const NOT_A_PATH = /^(https?:|mailto:|tel:|#|\/\/)/i;
+const isPathLike = (t) => !NOT_A_PATH.test(t) && (FILE_EXT.test(t) || t.endsWith('/'));
+
+// Inline links only — `[text](target)`. Reference-style links and autolinks are deliberately
+// out: `<path>` carries angle brackets and is classified as a placeholder anyway.
+const LINK_RE = /\[[^\]]*\]\(\s*([^)\s]+?)\s*\)/g;
+
+export function extractProseLinks(text) {
+  const prose = stripInlineCode(stripFences(text));
+  const out = [];
+  prose.split('\n').forEach((line, i) => {
+    for (const m of line.matchAll(LINK_RE)) {
+      const target = m[1].replace(/^<|>$/g, '');
+      if (isPathLike(target)) out.push({ target, line: i + 1, kind: 'link' });
+    }
+  });
+  return out;
+}
+
+// Report mode only: a path asserted inside backticks. Noisier by nature (this is where syntax
+// examples live), which is exactly why it never gates a build.
+const TOP_DIRS = /^(scripts|targets|skills|tests|bin|site|schema|docs|\.github|\.rsc)(\/|$)/;
+export function extractInlinePaths(text) {
+  const noFences = stripFences(text);
+  const out = [];
+  noFences.split('\n').forEach((line, i) => {
+    for (const m of line.matchAll(/(`+)((?:(?!\1)[^`])*)\1/g)) {
+      const t = m[2].trim();
+      if (!t || /\s/.test(t) || NOT_A_PATH.test(t)) continue;
+      if (!/\//.test(t)) continue; // a bare filename is a mention, not a location
+      if (TOP_DIRS.test(t) || FILE_EXT.test(t)) out.push({ target: t, line: i + 1, kind: 'inline' });
+    }
+  });
+  return out;
+}
+
+// ── classification ────────────────────────────────────────────────────────────────────────
+// Every rule is syntactic and narrow. Nothing here guesses from file names or wording: a
+// classifier that reasons about intent is a classifier that learns to stay quiet, and a quiet
+// detector is indistinguishable from no detector.
+export const CLASSES = {
+  // A shape, not a path: `skills/<ID>/`, `references/*.md`, `{slug}.md`.
+  placeholder: (t) => /[<>*{}]/.test(t),
+  // The file exists; the line number moves on its own. Anchors are recognised, not chased.
+  anchored: (t) => /:\d+$/.test(t) || t.includes('#'),
+  // Born at runtime, never in a repo. Note the trailing slash: `.rsc-notes.md` is NOT this.
+  runtime: (t) => t.startsWith('~') || /(^|\/)\.rsc\//.test(t),
+  // A site route, not a file: `/llms.txt`, `/skills/`. Found by running the report on the real
+  // wiki, where six route mentions arrived as "missing files". A genuinely absolute filesystem
+  // path in these documents would start with a home or volume directory, and the P9 test in
+  // tests/drift-check.test.js already forbids shipping one.
+  'web-route': (t) => t.startsWith('/'),
+  // The INSTALLED user's project, which the catalog legitimately talks about and does not
+  // contain. Segment-exact, so `rawdata/` and `docsX/` stay real claims.
+  'user-project': (t) => /(^|\/)(docs|raw)\//.test(t),
+};
+
+export function classify(target, { mode = 'catalog' } = {}) {
+  for (const name of ['placeholder', 'anchored', 'runtime', 'web-route']) {
+    if (CLASSES[name](target)) return name;
+  }
+  // In knowledge mode this repo IS the project, so `docs/…` is a real claim about real files.
+  if (mode === 'catalog' && CLASSES['user-project'](target)) return 'user-project';
+  return 'real';
+}
+
+// ── resolution ────────────────────────────────────────────────────────────────────────────
+// Catalog: the document's own directory, and nothing else — that is what a reader following the
+// link does, and being generous here would hide the exact wrong-depth bug this must catch.
+// Knowledge: several plausible bases, because the wiki writes shorthand
+// (`orient/references/x.md` meaning `skills/orient/references/x.md`) and 20 such references
+// would otherwise arrive as false positives.
+export function basesFor(docPath, { mode, root }) {
+  if (mode === 'catalog') return [dirname(docPath)];
+  return [dirname(docPath), root, join(root, 'docs'), join(root, 'docs', 'wiki'), join(root, 'skills')];
+}
+
+// `existsSync` asks the filesystem, and on APFS/HFS+ (macOS) and NTFS (Windows) the filesystem
+// ignores case. So a link to `AUDIT.md` resolves against a file named `audit.md` and this gate goes
+// GREEN on the maintainer's machine while Linux CI — and every installed user — sees a broken link.
+// That is P2 in its most expensive form: the gate fails open exactly where the work is done. It
+// happened for real (the motion-craft port moved references and one link kept its old casing).
+//
+// So existence is checked one path segment at a time against the parent's actual directory listing,
+// which is byte-exact on every filesystem. The walk starts at `root`, which is real by
+// construction; the declared hole is a target that resolves OUTSIDE the repo, where there is no
+// listing we can trust — the repo's own path may sit under a differently-cased volume or behind a
+// symlink, and a false positive in a blocking gate costs more than a named gap.
+// Spec: docs/wiki/sdd/specs/drift-check-case.md
+function existsExact(absTarget, root, cache) {
+  const rel = relative(root, absTarget);
+  // Outside the repo (or the root itself): keep the old behaviour, and say so rather than guess.
+  if (rel === '' || rel.startsWith('..') || rel.startsWith('/')) return existsSync(absTarget);
+
+  let dir = root;
+  // `relative()` returns platform-native separators (`\` on Windows, `/` elsewhere); splitting
+  // on a hardcoded '/' left every Windows path as a single un-matched segment, so no link ever
+  // resolved there. Split on `sep` instead — it is exactly the separator `relative()` used.
+  for (const segment of rel.split(sep)) {
+    let entries = cache.get(dir);
+    if (entries === undefined) {
+      // An unreadable directory is not evidence of drift; fall back rather than accuse.
+      try { entries = new Set(readdirSync(dir)); } catch { return existsSync(absTarget); }
+      cache.set(dir, entries);
+    }
+    if (!entries.has(segment)) return false;
+    dir = join(dir, segment);
+  }
+  return true;
+}
+
+const resolves = (target, bases, root, cache) =>
+  bases.some((b) => existsExact(resolve(b, target), root, cache));
+
+// Knowledge mode only. The authoring standards write per-skill paths generically — "every skill
+// carries `evals/cases.yaml`" — which is a claim about a shape shared by 258 directories, not a
+// location. Resolving it against any one skill is the honest reading; treating it as drift
+// produced five findings that were never wrong.
+function resolvesInAnySkill(target, root, cache) {
+  if (target.startsWith('.') || target.startsWith('/')) return false;
+  const skillsDir = join(root, 'skills');
+  if (!existsSync(skillsDir)) return false;
+  for (const e of readdirSync(skillsDir, { withFileTypes: true })) {
+    if (e.isDirectory() && existsExact(join(skillsDir, e.name, target), root, cache)) return true;
+  }
+  return false;
+}
+
+// ── walking ───────────────────────────────────────────────────────────────────────────────
+function markdownFiles(dir, out = []) {
+  for (const e of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+    if (e.name === 'node_modules' || e.name.startsWith('.git')) continue;
+    const p = join(dir, e.name);
+    if (e.isDirectory()) markdownFiles(p, out);
+    else if (/\.(md|markdown)$/i.test(e.name)) out.push(p);
+  }
+  return out;
+}
+
+// Claude Code's auto-memory for a project lives under a slug of its absolute path. Derived from
+// HOME and the repo path — never a hardcoded user, because this file ships in a public package.
+export function memoryDir(root, home = homedir()) {
+  return join(home, '.claude', 'projects', String(root).replace(/\//g, '-'), 'memory');
+}
+
+// Scan one tree. Returns findings plus the number of claims actually examined — a check that
+// silently stops looking would otherwise report "all clear" forever (see the count assertion in
+// tests/drift-check.test.js).
+export function scanTree(dir, { mode, root = REPO, label = null } = {}) {
+  const findings = [];
+  let claims = 0;
+  let skipped = 0;
+  // Directory listings, memoized per scan rather than per module: 824 claims share a few hundred
+  // directories, and a cache that outlived the scan would answer from a stale listing.
+  const cache = new Map();
+  for (const doc of markdownFiles(dir)) {
+    const text = readFileSync(doc, 'utf8');
+    const raw = mode === 'catalog'
+      ? extractProseLinks(text)
+      : [...extractProseLinks(text), ...extractInlinePaths(text)];
+    const bases = basesFor(doc, { mode, root });
+    for (const c of raw) {
+      const cls = classify(c.target, { mode });
+      if (cls !== 'real') {
+        skipped++;
+        continue;
+      }
+      claims++;
+      if (mode === 'knowledge' && resolvesInAnySkill(c.target, root, cache)) continue;
+      if (!resolves(c.target, bases, root, cache)) {
+        findings.push({
+          doc: relative(root, doc),
+          line: c.line,
+          target: c.target,
+          kind: c.kind,
+          tree: label || basename(dir),
+        });
+      }
+    }
+  }
+  return { findings, claims, skipped };
+}
+
+// ── the two modes ─────────────────────────────────────────────────────────────────────────
+export class MissingRoot extends Error {}
+
+// Blocking. A broken link in the catalog is broken product: the user follows a pointer to
+// nowhere and cannot tell that from having misunderstood the skill.
+export function checkCatalog({ root = REPO } = {}) {
+  const dir = join(root, 'skills');
+  if (!existsSync(dir)) throw new MissingRoot(`no catalog to check: ${relative(root, dir) || dir} does not exist`);
+  return scanTree(dir, { mode: 'catalog', root, label: 'skills' });
+}
+
+// Report-only. Inputs are the harness's own artifacts — written by a trusted agent, and for the
+// wiki already reviewed. Session transcripts are NOT an input here and must not become one: they
+// carry web pages, third-party READMEs and tool output, and anything derived from them would be
+// loaded into every later session.
+export function checkKnowledge({ root = REPO, home = homedir() } = {}) {
+  const trees = [
+    { dir: join(root, 'docs', 'wiki'), label: 'wiki' },
+    { dir: memoryDir(root, home), label: 'memory' },
+  ].filter((t) => existsSync(t.dir));
+  if (!trees.length) throw new MissingRoot('no knowledge tree found (docs/wiki and auto-memory are both absent)');
+  const all = { findings: [], claims: 0, skipped: 0, trees: trees.map((t) => t.label) };
+  for (const t of trees) {
+    const r = scanTree(t.dir, { mode: 'knowledge', root, label: t.label });
+    all.findings.push(...r.findings);
+    all.claims += r.claims;
+    all.skipped += r.skipped;
+  }
+  return all;
+}
+
+// ── CLI ───────────────────────────────────────────────────────────────────────────────────
+// Every denial carries its own exit (P6). The second half of the recovery matters as much as the
+// first: an example link is fixed by putting it in code, and that is not a loophole — it is the
+// rule this check is built on, stated where someone hitting the wall will read it.
+const RECOVER = [
+  'Recover: fix the link so it resolves from the document that contains it,',
+  '  or — if it is showing what a link looks like rather than pointing at a file —',
+  '  wrap it in backticks or a fenced block. Code shows; prose points.',
+].join('\n');
+
+const report = (f) => `  ${f.doc}:${f.line}  →  ${f.target}${f.kind === 'inline' ? '  [inline]' : ''}`;
+
+function main(argv) {
+  const wantKnowledge = argv.includes('--knowledge') || argv.includes('--all');
+  const wantCatalog = !argv.includes('--knowledge') || argv.includes('--all');
+  let failed = false;
+
+  if (wantCatalog) {
+    let r;
+    try {
+      r = checkCatalog({});
+    } catch (e) {
+      // Distinct exit and message: "could not run" must never be reported as "nothing wrong".
+      console.error(`drift-check: ${e.message}`);
+      console.error('Recover: run this from the repo root, where skills/ lives.');
+      process.exit(2);
+    }
+    console.log(`drift-check catalog: ${r.claims} prose link claim(s) checked, ${r.skipped} classified as non-claims`);
+    if (r.findings.length) {
+      console.log(`\nFAIL: ${r.findings.length} link(s) do not resolve:\n${r.findings.map(report).join('\n')}\n`);
+      console.log(RECOVER);
+      failed = true;
+    } else {
+      console.log('RESULT: PASS — every prose link in the catalog resolves');
+    }
+  }
+
+  if (wantKnowledge) {
+    try {
+      const k = checkKnowledge({});
+      console.log(`\ndrift-check knowledge (${k.trees.join(', ')}): ${k.claims} claim(s) checked, ${k.skipped} classified as non-claims`);
+      if (k.findings.length) {
+        console.log(`${k.findings.length} claim(s) do not resolve — advisory, nothing is blocked:\n${k.findings.map(report).join('\n')}`);
+        console.log('\nEach is either drift (fix the document) or a deliberate mention of something absent (leave it).');
+      } else {
+        console.log('no drift found');
+      }
+    } catch (e) {
+      // Advisory tree missing is normal (docs is gitignored; a fresh install has no wiki).
+      console.log(`\ndrift-check knowledge: skipped — ${e.message}`);
+    }
+  }
+
+  process.exit(failed ? 1 : 0);
+}
+
+// Compared through realpath, not by string: on macOS the temp dir is a symlink, so argv[1] and
+// import.meta.url disagree for any script under it and the common
+// `import.meta.url === \`file://${process.argv[1]}\`` guard silently declines to run —
+// which looks exactly like a clean pass.
+function invokedDirectly() {
+  try {
+    return !!process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+}
+
+if (invokedDirectly()) main(process.argv.slice(2));

package/scripts/eval-lint.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# eval-lint.sh — validate every skills/*/evals/cases.yaml against the eval minimums.
+#
+# Minimums per skill:
+#   >= 5 should_trigger entries
+#   >= 4 should_not_trigger entries
+#   >= 1 capability scenario
+#   every should_not_trigger route_to resolves (see below)
+#
+# route_to is the one that used to be decorative: the rubric listed "route_to names a
+# real skill" as a deterministic gate and nothing checked it, so evals accumulated
+# routes to skills that were never authored. Each value must now be one of:
+#   - a real catalog skill id (a directory under skills/ with a SKILL.md)
+#   - "none" — deliberately nothing to route to
+#   - "external:<name>" — a skill outside this catalog, on purpose. Some skills
+#     legitimately defer to tools that ship elsewhere; the prefix makes that a
+#     visible, greppable claim instead of a typo indistinguishable from drift.
+#
+# Uses python3 for real YAML parsing when available. If python3 is missing,
+# falls back to a basic structural grep check and SKIPs the deep validation
+# with a warning (still enforces presence of evals/cases.yaml and the keys).
+#
+# bash 3.2-safe: no mapfile, no unguarded ${arr[@]} under set -u.
+
+# Resolve repo root (parent of scripts/).
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd)
+SKILLS_DIR="$REPO_ROOT/skills"
+
+MIN_TRIGGER=5
+MIN_NOT_TRIGGER=4
+MIN_CAPABILITY=1
+
+HAVE_PY=0
+if command -v python3 >/dev/null 2>&1; then
+  HAVE_PY=1
+fi
+
+fail_count=0
+skill_count=0
+warned_skip=0
+
+# Validate one cases.yaml. Echos "PASS"/"FAIL: reason". Returns 0 on pass, 1 on fail.
+validate_py() {
+  cy="$1"
+  CASES_YAML="$cy" MIN_TRIGGER="$MIN_TRIGGER" MIN_NOT_TRIGGER="$MIN_NOT_TRIGGER" \
+  MIN_CAPABILITY="$MIN_CAPABILITY" VALID_IDS="$VALID_IDS" python3 - <<'PYEOF'
+import os, sys
+try:
+    import yaml
+except Exception as e:
+    sys.stderr.write("NOYAML\n")
+    sys.exit(3)
+
+path = os.environ["CASES_YAML"]
+mt = int(os.environ["MIN_TRIGGER"])
+mn = int(os.environ["MIN_NOT_TRIGGER"])
+mc = int(os.environ["MIN_CAPABILITY"])
+
+try:
+    with open(path) as f:
+        data = yaml.safe_load(f)
+except Exception as e:
+    print("FAIL: YAML parse error: %s" % e)
+    sys.exit(1)
+
+if not isinstance(data, dict):
+    print("FAIL: top-level YAML is not a mapping")
+    sys.exit(1)
+
+def count(key):
+    v = data.get(key)
+    if v is None:
+        return 0
+    if not isinstance(v, list):
+        return -1  # present but wrong type
+    return len(v)
+
+st = count("should_trigger")
+sn = count("should_not_trigger")
+cap = count("capability")
+
+# --- route_to resolution -------------------------------------------------------
+# Accepts a real catalog id, "none", or "external:<name>". A value may be a list,
+# or a string holding several targets separated by comma / | / " or ".
+valid = set(filter(None, os.environ.get("VALID_IDS", "").split(",")))
+NOTHING = {"none", "n/a", "-", ""}
+bad_routes = []
+
+def targets(value):
+    items = value if isinstance(value, list) else [value]
+    out = []
+    for item in items:
+        for part in str(item).replace("|", ",").replace(" or ", ",").split(","):
+            out.append(part.strip().strip("`"))
+    return out
+
+for case in (data.get("should_not_trigger") or []):
+    if not isinstance(case, dict) or case.get("route_to") is None:
+        continue
+    for t in targets(case["route_to"]):
+        low = t.lower()
+        if low in NOTHING:
+            continue
+        if low.startswith("external:"):
+            if not low[len("external:"):].strip():
+                bad_routes.append('"external:" with no name')
+            continue
+        if valid and t not in valid:
+            bad_routes.append(t)
+
+reasons = []
+if bad_routes:
+    uniq = sorted(set(bad_routes))
+    reasons.append("route_to does not resolve: %s (use a real skill id, \"none\", or \"external:<name>\")" % ", ".join(uniq))
+if st < 0:
+    reasons.append("should_trigger is not a list")
+elif st < mt:
+    reasons.append("should_trigger=%d (<%d)" % (st, mt))
+if sn < 0:
+    reasons.append("should_not_trigger is not a list")
+elif sn < mn:
+    reasons.append("should_not_trigger=%d (<%d)" % (sn, mn))
+if cap < 0:
+    reasons.append("capability is not a list")
+elif cap < mc:
+    reasons.append("capability=%d (<%d)" % (cap, mc))
+
+if reasons:
+    print("FAIL: " + "; ".join(reasons))
+    sys.exit(1)
+
+print("PASS (should_trigger=%d, should_not_trigger=%d, capability=%d)" % (st, sn, cap))
+sys.exit(0)
+PYEOF
+}
+
+# Fallback: basic structural grep check. Counts top-level list items as lines
+# beginning with two-space indent + "- " that follow each section header.
+# This is intentionally conservative; deep validation is SKIPPED.
+validate_grep() {
+  cy="$1"
+  for key in should_trigger should_not_trigger capability; do
+    if ! grep -Eq "^${key}:" "$cy"; then
+      echo "FAIL: missing top-level key '${key}'"
+      return 1
+    fi
+  done
+  echo "SKIP: python3/yaml unavailable — structural keys present, deep validation skipped"
+  return 0
+}
+
+echo "eval-lint: scanning $SKILLS_DIR"
+echo "minimums: should_trigger>=$MIN_TRIGGER should_not_trigger>=$MIN_NOT_TRIGGER capability>=$MIN_CAPABILITY"
+echo "route_to must resolve: a catalog skill id, \"none\", or \"external:<name>\""
+echo
+
+# The set of real skill ids, built once from disk — the same surface the router sees.
+VALID_IDS=""
+for sp in "$SKILLS_DIR"/*; do
+  [ -f "$sp/SKILL.md" ] || continue
+  VALID_IDS="$VALID_IDS,$(basename "$sp")"
+done
+VALID_IDS="${VALID_IDS#,}"
+
+# Iterate skills deterministically; bash 3.2-safe (no mapfile).
+for skill_path in "$SKILLS_DIR"/*; do
+  [ -d "$skill_path" ] || continue
+  [ -f "$skill_path/SKILL.md" ] || continue
+  skill=$(basename "$skill_path")
+  skill_count=$((skill_count + 1))
+
+  evals_dir="$skill_path/evals"
+  cases_yaml="$evals_dir/cases.yaml"
+
+  if [ ! -d "$evals_dir" ]; then
+    printf '%-22s FAIL: missing evals/ directory\n' "$skill"
+    fail_count=$((fail_count + 1))
+    continue
+  fi
+  if [ ! -f "$cases_yaml" ]; then
+    printf '%-22s FAIL: missing evals/cases.yaml\n' "$skill"
+    fail_count=$((fail_count + 1))
+    continue
+  fi
+
+  if [ "$HAVE_PY" -eq 1 ]; then
+    # Capture output and exit status without tripping set -e.
+    set +e
+    result=$(validate_py "$cases_yaml" 2>/tmp/eval-lint.$$.err)
+    rc=$?
+    set -e
+    if [ "$rc" -eq 3 ] || grep -q NOYAML /tmp/eval-lint.$$.err 2>/dev/null; then
+      # python3 present but pyyaml not installed -> fallback.
+      set +e
+      result=$(validate_grep "$cases_yaml")
+      rc=$?
+      set -e
+      warned_skip=1
+    fi
+    rm -f /tmp/eval-lint.$$.err
+  else
+    set +e
+    result=$(validate_grep "$cases_yaml")
+    rc=$?
+    set -e
+    warned_skip=1
+  fi
+
+  printf '%-22s %s\n' "$skill" "$result"
+  if [ "$rc" -ne 0 ]; then
+    fail_count=$((fail_count + 1))
+  fi
+done
+
+echo
+echo "scanned $skill_count skill(s); $fail_count failure(s)"
+if [ "$warned_skip" -eq 1 ]; then
+  echo "WARNING: deep YAML validation was skipped for one or more skills (python3 or PyYAML unavailable)."
+fi
+
+if [ "$fail_count" -gt 0 ]; then
+  echo "RESULT: FAIL"
+  exit 1
+fi
+
+echo "RESULT: PASS"
+exit 0