harness-everything 0.2.2 → 0.4.0

package/ci/consistency-check.js

@@ -0,0 +1,393 @@
+#!/usr/bin/env node
+/**
+ * Harness Consistency Check
+ * Validates that the docs surface, skill catalog, and distribution manifests
+ * tell the same story. A stale manifest or a dead README link is a router
+ * that lies — this script keeps them honest.
+ *
+ * Checks:
+ *  1. Every skill dir's SKILL.md frontmatter name matches its directory.
+ *  2. Every SKILL.md has a description and both trigger sections.
+ *  3. .claude-plugin/plugin.json + marketplace.json list exactly the on-disk skills.
+ *  4. Version strings agree across package.json / plugin.json / marketplace.json.
+ *  5. Every skill has a routing eval (evals/<skill>/eval.yaml).
+ *  6. Local links in README.md and docs markdown files resolve to real files.
+ */
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+const { checkDisclosure } = require('./disclosure-check');
+
+const ROOT = path.resolve(__dirname, '..');
+let failures = 0;
+function check(name, condition, detail) {
+  if (condition) {
+    console.log(`✅ ${name}`);
+  } else {
+    console.error(`❌ ${name}`);
+    if (detail) console.error(`   ${detail}`);
+    failures++;
+  }
+}
+
+// --- Collect skills from disk -------------------------------------------
+const SKIP_DIRS = new Set([
+  '.git', '.github', '.claude-plugin', 'bin', 'docs', 'evals',
+  'behavioral-evals', 'benchmarks', 'hooks', 'node_modules', 'references', 'scripts'
+]);
+function discoverSkills() {
+  const skills = [];
+  for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
+    if (!entry.isDirectory() || SKIP_DIRS.has(entry.name) || entry.name.startsWith('.')) continue;
+    const skillMd = path.join(ROOT, entry.name, 'SKILL.md');
+    if (!fs.existsSync(skillMd)) continue;
+    const raw = fs.readFileSync(skillMd, 'utf8');
+    const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+    const fm = fmMatch ? fmMatch[1] : '';
+    const name = (fm.match(/^name:\s*(.+)$/m) || [])[1];
+    const desc = (fm.match(/^description:\s*(.+)$/m) || [])[1];
+    skills.push({
+      dir: entry.name,
+      name: name ? name.trim().replace(/["']/g, '') : null,
+      description: desc ? desc.trim() : null,
+      body: raw,
+      path: skillMd,
+    });
+  }
+  return skills.sort((a, b) => a.dir.localeCompare(b.dir));
+}
+const skills = discoverSkills();
+check(`Found ${skills.length} skills on disk`, skills.length >= 20, `Got ${skills.length}`);
+
+// --- 1+2. Frontmatter & trigger sections --------------------------------
+for (const s of skills) {
+  check(
+    `${s.dir}: frontmatter name matches directory`,
+    s.name === s.dir,
+    `frontmatter "${s.name}" != dir "${s.dir}"`
+  );
+  check(
+    `${s.dir}: has description in frontmatter`,
+    !!s.description && s.description.length > 10,
+    s.description ? 'description too short' : 'missing description'
+  );
+  check(`${s.dir}: has "## USE FOR:" section`, /^## USE FOR:/m.test(s.body), 'missing section');
+  check(
+    `${s.dir}: has "## DO NOT USE FOR:" section`,
+    /^## DO NOT USE FOR:/m.test(s.body),
+    'missing section'
+  );
+}
+
+// --- 1b. YAML frontmatter syntax validation (parser parity) ---------------
+// Catches unquoted colons, unclosed quotes, and broken multi-line strings
+// that regex extraction misses but waza's real YAML parser catches in CI.
+function validateFrontmatterSyntax(raw, skillDir) {
+  const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (!fmMatch) return check(`${skillDir}: has valid YAML frontmatter delimiters`, false, 'missing --- delimiters');
+  const fm = fmMatch[1];
+  
+  // Parser parity: use js-yaml to validate the frontmatter
+  try {
+    yaml.load(fm);
+  } catch (e) {
+    return check(`${skillDir}: frontmatter YAML syntax valid`, false, e.message);
+  }
+  
+  // Additional regex-based checks for common issues
+  const lines = fm.split('\n');
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!line.trim() || line.trim().startsWith('#')) continue;
+    // Check for unquoted colons in values: "key: value with : inside"
+    const kvMatch = line.match(/^(\s*)(\w[\w-]*):\s*(.*)/);
+    if (kvMatch && kvMatch[3]) {
+      const val = kvMatch[3];
+      // Value contains colon not inside quotes → likely unquoted YAML
+      if (val.includes(':') && !/^["']/.test(val) && !/["']$/.test(val)) {
+        const inQuotes = /^["'].*:.*["']$/.test(val);
+        if (!inQuotes) {
+          check(
+            `${skillDir}: frontmatter line ${i + 1} has unquoted colon in value`,
+            false,
+            `"${line.trim()}" — wrap value in quotes to avoid YAML parse errors`
+          );
+          return;
+        }
+      }
+    }
+    // Check for unclosed single quotes
+    const singleQuoteCount = (line.match(/(?<!\\)'/g) || []).length;
+    if (singleQuoteCount % 2 !== 0) {
+      check(`${skillDir}: frontmatter line ${i + 1} has unclosed single quote`, false, `"${line.trim()}"`);
+      return;
+    }
+    // Check for unclosed double quotes
+    const doubleQuoteCount = (line.match(/(?<!\\)"/g) || []).length;
+    if (doubleQuoteCount % 2 !== 0) {
+      check(`${skillDir}: frontmatter line ${i + 1} has unclosed double quote`, false, `"${line.trim()}"`);
+      return;
+    }
+  }
+  check(`${skillDir}: frontmatter YAML syntax valid`, true);
+}
+for (const s of skills) {
+  validateFrontmatterSyntax(s.body, s.dir);
+}
+
+// --- 2b. Progressive-disclosure surface -------------------------------
+// Keep routed contracts, plain-language workflow docs, and lazy-loaded detail
+// paired. The standalone module is also exercised by focused mechanism tests.
+const disclosure = checkDisclosure(ROOT);
+for (const failure of disclosure.failures) {
+  check(`disclosure: ${failure.check}`, false, failure.detail);
+}
+console.log(`Checked disclosure for ${disclosure.skills.length} skill(s) and ${disclosure.workflows.length} workflow doc(s).`);
+
+// --- 3+4. Distribution manifests -----------------------------------------
+const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
+const pluginJsonPath = path.join(ROOT, '.claude-plugin', 'plugin.json');
+const marketplaceJsonPath = path.join(ROOT, '.claude-plugin', 'marketplace.json');
+check('.claude-plugin/plugin.json exists', fs.existsSync(pluginJsonPath));
+check('.claude-plugin/marketplace.json exists', fs.existsSync(marketplaceJsonPath));
+
+if (fs.existsSync(pluginJsonPath)) {
+  const plugin = JSON.parse(fs.readFileSync(pluginJsonPath, 'utf8'));
+  const listed = new Set((plugin.skills || []).map((p) => p.replace(/^\.\//, '').replace(/\/$/, '')));
+  const onDisk = new Set(skills.map((s) => s.dir));
+  const missing = [...onDisk].filter((d) => !listed.has(d));
+  const extra = [...listed].filter((d) => !onDisk.has(d));
+  check(
+    'plugin.json lists every on-disk skill',
+    missing.length === 0,
+    `missing: ${missing.join(', ')}`
+  );
+  check(
+    'plugin.json lists no phantom skills',
+    extra.length === 0,
+    `extra: ${extra.join(', ')}`
+  );
+  check('plugin.json hooks file exists', !plugin.hooks || fs.existsSync(path.join(ROOT, plugin.hooks)), plugin.hooks);
+  if (plugin.agents) {
+    const agentEntries = Array.isArray(plugin.agents) ? plugin.agents : [plugin.agents];
+    for (const rel of agentEntries) {
+      check(`plugin.json agents entry is a .md file: ${rel}`, rel.endsWith('.md'), rel);
+      check(`plugin.json agents path exists: ${rel}`, fs.existsSync(path.join(ROOT, rel)), rel);
+    }
+  }
+  for (const rel of plugin.skills || []) {
+    const p = path.join(ROOT, rel);
+    check(`plugin.json skill path resolves: ${rel}`, fs.existsSync(path.join(p, 'SKILL.md')));
+  }
+  check(
+    'version agrees: package.json vs plugin.json',
+    pkg.version === plugin.version,
+    `"${pkg.version}" vs "${plugin.version}"`
+  );
+  if (fs.existsSync(marketplaceJsonPath)) {
+    const market = JSON.parse(fs.readFileSync(marketplaceJsonPath, 'utf8'));
+    const entry = (market.plugins || []).find((p) => p.name === plugin.name);
+    check('marketplace.json references this plugin', !!entry);
+    if (entry && entry.version) {
+      check(
+        'version agrees: package.json vs marketplace.json',
+        pkg.version === entry.version,
+        `"${pkg.version}" vs "${entry.version}"`
+      );
+    }
+  }
+}
+
+// --- 5. Routing eval coverage --------------------------------------------
+const evalsDir = path.join(ROOT, 'evals');
+for (const s of skills) {
+  check(
+    `${s.dir}: has routing eval (evals/${s.dir}/eval.yaml)`,
+    fs.existsSync(path.join(evalsDir, s.dir, 'eval.yaml')),
+    'no trigger/routing eval for this skill'
+  );
+}
+
+// --- 5b. Skill version ceiling --------------------------------------------
+// Skill frontmatter versions move in lockstep with releases; no skill may
+// exceed the package version's numeric base (prerelease suffixes ignored).
+function verBase(v) {
+  const m = String(v).match(/^(\d+)\.(\d+)\.(\d+)/);
+  return m ? m.slice(1).map(Number) : null;
+}
+function cmpVer(a, b) {
+  for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] - b[i];
+  return 0;
+}
+const pkgBase = verBase(pkg.version);
+check(
+  `package.json version parses (${pkg.version})`,
+  !!pkgBase
+);
+for (const s of skills) {
+  const raw = fs.readFileSync(s.path, 'utf8');
+  const fm = (raw.match(/^---\r?\n([\s\S]*?)\r?\n---/) || [])[1] || '';
+  const vRaw = ((fm.match(/^  version:\s*(.+)$/m) || [])[1] || '').trim();
+  const v = verBase(vRaw);
+  check(
+    `${s.dir}: skill version ${vRaw || '(missing)'} <= package base ${pkgBase ? pkgBase.join('.') : '?'}`,
+    !!v && !!pkgBase && cmpVer(v, pkgBase) <= 0,
+    v ? 'skill version exceeds package version' : 'missing or unparseable metadata.version'
+  );
+}
+
+// --- 5c. Nested sub-skill versions -----------------------------------------
+// Sub-skills nested inside a parent skill dir (<skill>/<sub>/SKILL.md) are
+// invoked by the parent, not routed independently; they inherit the parent
+// skill's version so the lockstep policy stays enforceable.
+const nestedSubSkills = [];
+for (const s of skills) {
+  (function walk(dir) {
+    let entries;
+    try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const e of entries) {
+      if (!e.isDirectory() || e.name.startsWith('.') || e.name === 'node_modules') continue;
+      const sub = path.join(dir, e.name);
+      if (fs.existsSync(path.join(sub, 'SKILL.md'))) nestedSubSkills.push(sub);
+      else walk(sub);
+    }
+  })(path.join(ROOT, s.dir));
+}
+for (const sub of nestedSubSkills) {
+  const rel = path.relative(ROOT, path.join(sub, 'SKILL.md'));
+  const parentDir = path.dirname(sub);
+  const parentRaw = fs.readFileSync(path.join(parentDir, 'SKILL.md'), 'utf8');
+  const parentFm = (parentRaw.match(/^---\r?\n([\s\S]*?)\r?\n---/) || [])[1] || '';
+  const parentV = ((parentFm.match(/^  version:\s*(.+)$/m) || [])[1] || '').trim();
+  const raw = fs.readFileSync(path.join(sub, 'SKILL.md'), 'utf8');
+  const fm = (raw.match(/^---\r?\n([\s\S]*?)\r?\n---/) || [])[1] || '';
+  const v = ((fm.match(/^  version:\s*(.+)$/m) || [])[1] || '').trim();
+  check(
+    `${rel}: nested sub-skill inherits parent version (${parentV})`,
+    !!v && v === parentV,
+    `"${v}" != parent "${parentV}"`
+  );
+}
+
+// --- 5d. Word-count budget (token proxy) ------------------------------------
+// Skill frontmatter descriptions are routing surfaces; long descriptions
+// bloat token budgets. A simple word-count proxy flags files likely to
+// exceed the waza 500-token limit (~1.55 tokens/word → 330 words ≈ 507 tokens).
+const WORD_BUDGET = 330;
+for (const s of skills) {
+  const raw = fs.readFileSync(s.path, 'utf8');
+  const body = raw.replace(/^---\r?\n[\s\S]*?\r?\n---/, '').trim();
+  const wc = body.split(/\s+/).filter(Boolean).length;
+  check(
+    `${s.dir}: SKILL.md word count ${wc} <= ${WORD_BUDGET}`,
+    wc <= WORD_BUDGET,
+    `body is ${wc} words (~${Math.round(wc * 1.55)} tokens); trim to under ${WORD_BUDGET} words`
+  );
+}
+
+// --- 6. Dead-link audit over README.md and docs/** -----------------------
+function extractLocalLinks(file) {
+  const text = fs.readFileSync(file, 'utf8');
+  const links = [];
+  const re = /\[[^\]]*\]\(([^)\s]+)\)/g;
+  let m;
+  while ((m = re.exec(text)) !== null) {
+    let href = m[1].split('#')[0];
+    if (!href || /^(https?:|mailto:|npx)/.test(href)) continue;
+    links.push(href);
+  }
+  return links;
+}
+const docFiles = [path.join(ROOT, 'README.md')];
+(function walk(dir) {
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) walk(p);
+    else if (e.name.endsWith('.md')) docFiles.push(p);
+  }
+})(path.join(ROOT, 'docs'));
+
+let linkCount = 0;
+for (const file of docFiles) {
+  for (const href of extractLocalLinks(file)) {
+    linkCount++;
+    const target = path.resolve(path.dirname(file), decodeURIComponent(href));
+    check(
+      `link resolves: ${path.relative(ROOT, file)} -> ${href}`,
+      fs.existsSync(target),
+      `target not found on disk`
+    );
+  }
+}
+console.log(`\nChecked ${linkCount} local doc links.`);
+
+// --- 7. Token-budget gate (word-count proxy, hard limit 500) ----------------
+// Exact tokenization is waza's job; this is the local early-warning gate.
+// Includes warning threshold at 80% (400 words) for early detection.
+const TOKEN_HARD_LIMIT = 500;
+const TOKEN_WARNING_THRESHOLD = 400; // 80% of hard limit
+for (const s of skills) {
+  const raw = fs.readFileSync(s.path, 'utf8');
+  const withoutFm = raw.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, '');
+  const wordCount = withoutFm.split(/\s+/).filter(Boolean).length;
+  
+  // Early warning at WORD_BUDGET threshold (non-blocking)
+  if (wordCount > WORD_BUDGET && wordCount <= TOKEN_WARNING_THRESHOLD) {
+    console.log(`⚠️ ${s.dir}: word count ${wordCount} exceeds WORD_BUDGET (${WORD_BUDGET})`);
+  }
+  
+  // Warning at 80% threshold (non-blocking)
+  if (wordCount > TOKEN_WARNING_THRESHOLD && wordCount <= TOKEN_HARD_LIMIT) {
+    console.log(`⚠️ ${s.dir}: SKILL.md word count ${wordCount} approaching limit (${TOKEN_HARD_LIMIT - wordCount} words remaining)`);
+  }
+  check(
+    `${s.dir}: SKILL.md word count ${wordCount} <= ${TOKEN_HARD_LIMIT}`,
+    wordCount <= TOKEN_HARD_LIMIT,
+    wordCount > TOKEN_HARD_LIMIT
+      ? `exceeds hard limit by ${wordCount - TOKEN_HARD_LIMIT} words`
+      : `${TOKEN_HARD_LIMIT - wordCount} words of headroom`
+  );
+}
+
+// 6. Changelog headings vs git tags.
+// release.yml triggers on tag push, so a version with a changelog entry and no
+// tag was never gated and never published - 0.3.4-beta and 0.3.5-beta both sat
+// in that state (issue #39). A version deliberately left unreleased must say so
+// in its heading rather than being silently absent from git.
+const { execFileSync } = require('child_process');
+let knownTags = null;
+try {
+  knownTags = new Set(
+    execFileSync('git', ['tag', '--list'], { cwd: ROOT, encoding: 'utf8' })
+      .split(/\r?\n/)
+      .filter(Boolean)
+  );
+} catch (err) {
+  knownTags = null;
+}
+
+if (knownTags === null || knownTags.size === 0) {
+  console.log('WARN changelog/tag check skipped: no git tags available (shallow clone? use fetch-depth: 0)');
+} else {
+  const changelog = fs.readFileSync(path.join(ROOT, 'CHANGELOG.md'), 'utf8');
+  const headings = [...changelog.matchAll(/^## \[([^\]]+)\](.*)$/gm)];
+  check('CHANGELOG.md exposes version headings', headings.length > 0, 'no "## [version]" headings found');
+  for (const [, version, rest] of headings) {
+    if (/^unreleased$/i.test(version.trim())) continue;
+    if (/unreleased/i.test(rest)) {
+      console.log(`OK CHANGELOG ${version}: declared unreleased, no tag required`);
+      continue;
+    }
+    check(
+      `CHANGELOG ${version}: released heading has tag v${version}`,
+      knownTags.has(`v${version}`),
+      `no tag v${version}; tag the release or mark the heading unreleased`
+    );
+  }
+}
+
+if (failures > 0) {
+  console.error(`\n❌ CONSISTENCY CHECK FAILED: ${failures} problem(s).`);
+  process.exit(1);
+}
+console.log('\n🎉 ALL CONSISTENCY CHECKS PASSED.');

package/ci/description-collision.js

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/**
+ * Skill Description Collision Detector
+ *
+ * Routing skills are selected by their `description:` frontmatter. When two
+ * descriptions overlap too much, the router effectively flips a coin between
+ * them. This script tokenizes every SKILL.md description, computes pairwise
+ * Jaccard similarity over stemmed tokens, and fails on collisions.
+ *
+ * Thresholds:
+ *   >= 0.75 similarity -> ERROR (collision; rewrite one description)
+ *   >= 0.50 similarity -> WARNING (review recommended)
+ *
+ * Usage: node ci/description-collision.js
+ */
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+const ROOT = path.resolve(__dirname, '..');
+const ERROR_THRESHOLD = 0.75;
+const WARN_THRESHOLD = 0.5;
+
+// Words so generic they carry no routing signal — dropping them keeps the
+// metric honest (otherwise "the", "a", "skill" dominate every pair).
+const STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'for', 'to', 'of', 'in', 'on', 'with',
+  'this', 'that', 'it', 'its', 'is', 'are', 'be', 'when', 'before', 'after',
+  'use', 'used', 'using', 'must', 'should', 'new', 'your', 'you', 'from',
+  'into', 'any', 'all', 'not', 'no', 'so', 'by', 'at', 'as',
+]);
+
+function stem(word) {
+  // Deliberately crude suffix stripper — consistent crudeness is what matters.
+  return word
+    .replace(/(?:ing|ies|ied|edly|ements?)$/, '')
+    .replace(/(?:s|es)$/, '');
+}
+
+function tokenize(description) {
+  const cleaned = description.replace(/^["']|["'],?$/g, '').toLowerCase();
+  const raw = cleaned.match(/[a-z][a-z0-9-]+/g) || [];
+  const tokens = new Set();
+  for (const w of raw) {
+    if (STOPWORDS.has(w)) continue;
+    tokens.add(stem(w));
+  }
+  return tokens;
+}
+
+function jaccard(a, b) {
+  let inter = 0;
+  for (const t of a) if (b.has(t)) inter++;
+  return inter / (a.size + b.size - inter);
+}
+
+function discoverSkills() {
+  const out = [];
+  const parseErrors = [];
+  for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
+    if (!entry.isDirectory() || entry.name.startsWith('.')) continue;
+    const skillMd = path.join(ROOT, entry.name, 'SKILL.md');
+    if (!fs.existsSync(skillMd)) continue;
+    const raw = fs.readFileSync(skillMd, 'utf8');
+    const fmMatch = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+    if (!fmMatch) {
+      parseErrors.push(`${entry.name}: missing YAML frontmatter delimiters`);
+      continue;
+    }
+    const fm = fmMatch[1];
+    // Parser parity: use js-yaml to parse frontmatter
+    let desc = '';
+    try {
+      const parsed = yaml.load(fm);
+      desc = (parsed.description || '').trim();
+    } catch (error) {
+      parseErrors.push(`${entry.name}: invalid YAML frontmatter (${error.message})`);
+      continue;
+    }
+    out.push({ name: entry.name, desc });
+  }
+  return { skills: out, parseErrors };
+}
+
+const { skills, parseErrors } = discoverSkills();
+if (parseErrors.length > 0) {
+  console.error('\n❌ DESCRIPTION COLLISION CHECK FAILED: malformed skill frontmatter.');
+  for (const error of parseErrors) console.error(`  ${error}`);
+  process.exit(1);
+}
+if (skills.length < 2) {
+  console.error('Not enough skills found to compare.');
+  process.exit(1);
+}
+
+let errors = 0;
+console.log(`Comparing ${skills.length} skill descriptions...\n`);
+for (let i = 0; i < skills.length; i++) {
+  for (let j = i + 1; j < skills.length; j++) {
+    const sim = jaccard(tokenize(skills[i].desc), tokenize(skills[j].desc));
+    if (sim >= WARN_THRESHOLD) {
+      const label = sim >= ERROR_THRESHOLD ? '❌ COLLISION' : '⚠️  WARNING ';
+      console.log(
+        `${label} ${skills[i].name} <-> ${skills[j].name}: ${sim.toFixed(2)}`
+      );
+      if (sim >= ERROR_THRESHOLD) errors++;
+    }
+  }
+}
+
+if (errors > 0) {
+  console.error(
+    `\n❌ ${errors} description collision(s) at >= ${ERROR_THRESHOLD}. ` +
+      `Rewrite the affected descriptions so the router can tell them apart.`
+  );
+  process.exit(1);
+}
+console.log(`\n🎉 NO DESCRIPTION COLLISIONS (threshold ${ERROR_THRESHOLD}).`);