harness-everything 0.2.2 → 0.3.6

package/{eval-framework → ci}/runner.js

@@ -1,56 +1,56 @@
-#!/usr/bin/env node
-const fs = require('fs');
-const path = require('path');
-const { execSync } = require('child_process');
-
-const casesDir = path.join(__dirname, 'cases');
-const files = fs.readdirSync(casesDir).filter(f => f.endsWith('.json'));
-
-console.log("=== Harness Tier Verification Framework ===");
-console.log("This framework simulates routing prompts to verify trigger rates.\n");
-
-let passed = 0;
-let failed = 0;
-
-files.forEach(file => {
-  const data = JSON.parse(fs.readFileSync(path.join(casesDir, file), 'utf8'));
-  console.log(`[Testing] ${data.name}`);
-  console.log(`  Prompt: "${data.prompt}"`);
-  console.log(`  Expected Tier: ${data.expected_tier}`);
-  
-  try {
-    const tierRouterPath = path.join(__dirname, '..', 'harness-everything', 'scripts', 'tier-router.js');
-    // Claude Code feeds UserPromptSubmit hooks JSON on stdin (a "prompt"
-    // field), not a CLI argument - match that here so this test actually
-    // exercises the real interface instead of a path tier-router.js never
-    // sees in production.
-    const output = execSync(`node "${tierRouterPath}"`, {
-      encoding: 'utf8',
-      input: JSON.stringify({ prompt: data.prompt }),
-      env: { ...process.env, HARNESS_EVAL: 'true' }
-    });
-    
-    // Parse the output to find the recommended tier (accept the old
-    // "REQUIRED TIER" label too so the runner works across versions)
-    const tierMatch = output.match(/(?:RECOMMENDED|REQUIRED) TIER: (Tier \d+)/i) || output.match(/(Tier \d+)/i);
-    const actualTier = tierMatch ? tierMatch[1] : "Unknown";
-    
-    if (actualTier.includes(data.expected_tier)) {
-      console.log(`  -> Result: PASS (Got ${actualTier})\n`);
-      passed++;
-    } else {
-      console.log(`  -> Result: FAIL (Expected ${data.expected_tier}, Got ${actualTier})\n`);
-      console.log(`  Output was:\n${output}`);
-      failed++;
-    }
-  } catch (err) {
-    console.log(`  -> Result: ERROR executing router`);
-    failed++;
-  }
-});
-
-console.log(`--- Summary: ${passed} passed, ${failed} failed ---`);
-if (failed > 0) {
-  process.exit(1);
-}
-
+#!/usr/bin/env node
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+
+const casesDir = path.join(__dirname, 'cases');
+const files = fs.readdirSync(casesDir).filter(f => f.endsWith('.json'));
+
+console.log("=== Harness Tier Verification Framework ===");
+console.log("This framework simulates routing prompts to verify trigger rates.\n");
+
+let passed = 0;
+let failed = 0;
+
+files.forEach(file => {
+  const data = JSON.parse(fs.readFileSync(path.join(casesDir, file), 'utf8'));
+  console.log(`[Testing] ${data.name}`);
+  console.log(`  Prompt: "${data.prompt}"`);
+  console.log(`  Expected Tier: ${data.expected_tier}`);
+  
+  try {
+    const tierRouterPath = path.join(__dirname, '..', 'harness-everything', 'scripts', 'tier-router.js');
+    // Claude Code feeds UserPromptSubmit hooks JSON on stdin (a "prompt"
+    // field), not a CLI argument - match that here so this test actually
+    // exercises the real interface instead of a path tier-router.js never
+    // sees in production.
+    const output = execSync(`node "${tierRouterPath}"`, {
+      encoding: 'utf8',
+      input: JSON.stringify({ prompt: data.prompt }),
+      env: { ...process.env, HARNESS_EVAL: 'true' }
+    });
+    
+    // Parse the output to find the recommended tier (accept the old
+    // "REQUIRED TIER" label too so the runner works across versions)
+    const tierMatch = output.match(/(?:RECOMMENDED|REQUIRED) TIER: (Tier \d+)/i) || output.match(/(Tier \d+)/i);
+    const actualTier = tierMatch ? tierMatch[1] : "Unknown";
+    
+    if (actualTier.includes(data.expected_tier)) {
+      console.log(`  -> Result: PASS (Got ${actualTier})\n`);
+      passed++;
+    } else {
+      console.log(`  -> Result: FAIL (Expected ${data.expected_tier}, Got ${actualTier})\n`);
+      console.log(`  Output was:\n${output}`);
+      failed++;
+    }
+  } catch (err) {
+    console.log(`  -> Result: ERROR executing router`);
+    failed++;
+  }
+});
+
+console.log(`--- Summary: ${passed} passed, ${failed} failed ---`);
+if (failed > 0) {
+  process.exit(1);
+}
+

package/ci/skill-routing-check.js

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+/**
+ * Deterministic route coverage gate for every skill's positive eval prompts.
+ * This complements waza: it executes the real local tier-router and does not
+ * require a network-installed evaluator.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const yaml = require('js-yaml');
+
+const ROOT = path.resolve(__dirname, '..');
+const ROUTER = path.join(ROOT, 'harness-everything', 'scripts', 'tier-router.js');
+const EVALS = path.join(ROOT, 'evals');
+
+function walk(dir) {
+  if (!fs.existsSync(dir)) return [];
+  const files = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) files.push(...walk(full));
+    else if (/^positive.*\.ya?ml$/i.test(entry.name)) files.push(full);
+  }
+  return files;
+}
+
+function expectedMarker(skill) {
+  if (skill === 'harness-everything') return 'RECOMMENDED TIER';
+  if (skill === 'todo-driven-workflow') return 'BASE EXECUTION LOOP';
+  return `${skill}/SKILL.md`;
+}
+
+const skillDirs = fs.readdirSync(ROOT, { withFileTypes: true })
+  .filter(entry => entry.isDirectory() && fs.existsSync(path.join(ROOT, entry.name, 'SKILL.md')))
+  .map(entry => entry.name)
+  .sort();
+
+let failures = 0;
+let cases = 0;
+for (const skill of skillDirs) {
+  const skillText = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf8');
+  const frontmatter = skillText.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  let skillDescription;
+  try {
+    skillDescription = yaml.load(frontmatter[1]).description;
+  } catch (error) {
+    console.error(`FAIL ${skill}/SKILL.md: invalid frontmatter (${error.message})`);
+    failures++;
+    continue;
+  }
+  const taskFiles = walk(path.join(EVALS, skill, 'tasks'));
+  if (taskFiles.length === 0) {
+    console.error(`FAIL ${skill}: no positive task files`);
+    failures++;
+    continue;
+  }
+
+  for (const taskFile of taskFiles) {
+    cases++;
+    let task;
+    try {
+      task = yaml.load(fs.readFileSync(taskFile, 'utf8'));
+    } catch (error) {
+      console.error(`FAIL ${path.relative(ROOT, taskFile)}: invalid YAML (${error.message})`);
+      failures++;
+      continue;
+    }
+
+    const prompt = task && task.inputs && task.inputs.prompt;
+    if (typeof prompt !== 'string' || !prompt.trim()) {
+      console.error(`FAIL ${path.relative(ROOT, taskFile)}: missing inputs.prompt`);
+      failures++;
+      continue;
+    }
+    if (task.description !== skillDescription) {
+      console.error(`FAIL ${path.relative(ROOT, taskFile)}: description does not exactly match SKILL.md frontmatter`);
+      failures++;
+      continue;
+    }
+
+    const result = spawnSync(process.execPath, [ROUTER, prompt], {
+      cwd: ROOT,
+      encoding: 'utf8',
+    });
+    const output = `${result.stdout || ''}\n${result.stderr || ''}`;
+    const marker = expectedMarker(skill);
+    if (result.status !== 0 || !output.includes(marker)) {
+      console.error(`FAIL ${path.relative(ROOT, taskFile)}: expected "${marker}"`);
+      failures++;
+    } else {
+      console.log(`PASS ${skill}/${path.basename(taskFile)}`);
+    }
+  }
+}
+
+console.log(`\nRoute coverage: ${cases - failures}/${cases} positive cases passed across ${skillDirs.length} skills.`);
+process.exit(failures ? 1 : 0);

package/ci/test-helper.js

@@ -0,0 +1,104 @@
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+process.env.CLAUDE = '1'; // Force Claude Code state path
+
+const { getWorkspaceRoot, getSessionDir } = require('../hooks/scripts/lib/harness-state');
+
+const root = getWorkspaceRoot();
+const hooksDir = path.join(root, 'hooks', 'scripts');
+const SESSION_ID = '__mechanism_test__';
+const sessionDir = getSessionDir(root, SESSION_ID);
+
+// Create session directory
+fs.mkdirSync(sessionDir, { recursive: true });
+
+const tempFiles = [];
+const tempDirs = [];
+const state = { failures: 0 };
+
+function check(name, condition, detail) {
+  if (condition) {
+    console.log(`✅ ${name}`);
+  } else {
+    console.error(`❌ ${name}`);
+    if (detail) console.error(`   ${detail}`);
+    state.failures++;
+  }
+}
+
+function runHook(scriptName, payload, extraEnv) {
+  const scriptPath = path.join(hooksDir, scriptName);
+  try {
+    const stdout = execSync(`node "${scriptPath}"`, {
+      encoding: 'utf8',
+      input: payload === null ? '' : JSON.stringify(payload),
+      env: { ...process.env, ...extraEnv },
+    });
+    return { code: 0, stdout, stderr: '' };
+  } catch (err) {
+    return { code: err.status, stdout: err.stdout || '', stderr: err.stderr || '' };
+  }
+}
+
+function writeState(file, obj) {
+  fs.writeFileSync(path.join(sessionDir, file), JSON.stringify(obj, null, 2), 'utf8');
+}
+function readState(file) {
+  return JSON.parse(fs.readFileSync(path.join(sessionDir, file), 'utf8'));
+}
+function stateExists(file) {
+  return fs.existsSync(path.join(sessionDir, file));
+}
+
+function tempFile(name) {
+  const p = path.join(root, name);
+  tempFiles.push(p);
+  return p;
+}
+
+function tempDir(name) {
+  const p = path.join(root, name);
+  tempDirs.push(p);
+  return p;
+}
+
+function finish() {
+  cleanup();
+  if (state.failures > 0) {
+    process.exit(1);
+  } else {
+    process.exit(0);
+  }
+}
+
+function cleanup() {
+  for (const f of tempFiles) {
+    try { if (fs.existsSync(f)) fs.unlinkSync(f); } catch (e) { /* best-effort */ }
+  }
+  for (const d of tempDirs) {
+    try { if (fs.existsSync(d)) fs.rmSync(d, { recursive: true, force: true }); } catch (e) { /* best-effort */ }
+  }
+  try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch (e) { /* best-effort */ }
+}
+
+process.on('exit', cleanup);
+process.on('SIGINT', () => { cleanup(); process.exit(1); });
+process.on('SIGTERM', () => { cleanup(); process.exit(1); });
+
+module.exports = {
+  root,
+  hooksDir,
+  SESSION_ID,
+  sessionDir,
+  check,
+  runHook,
+  writeState,
+  readState,
+  stateExists,
+  tempFile,
+  tempDir,
+  finish,
+  state
+};

package/ci/test-negative-controls.js

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+/**
+ * Negative Control Tests
+ * 
+ * Tests that quality gates correctly FAIL on known-bad fixtures.
+ * Each gate must have a corresponding negative control that asserts detection works.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const ROOT = path.resolve(__dirname, '..');
+const FIXTURES_DIR = path.join(ROOT, 'eval-framework', 'fixtures');
+
+function runCheck(script, args = []) {
+  const result = spawnSync('node', [path.join(ROOT, 'ci', script), ...args], {
+    cwd: ROOT,
+    encoding: 'utf8'
+  });
+  return result;
+}
+
+function copyFixture(fixtureName, targetDir) {
+  const src = path.join(FIXTURES_DIR, fixtureName);
+  const dest = path.join(ROOT, targetDir);
+  
+  if (fs.existsSync(dest)) {
+    fs.rmSync(dest, { recursive: true, force: true });
+  }
+  fs.cpSync(src, dest, { recursive: true });
+}
+
+function removeFixture(targetDir) {
+  const dest = path.join(ROOT, targetDir);
+  if (fs.existsSync(dest)) {
+    fs.rmSync(dest, { recursive: true, force: true });
+  }
+}
+
+async function testNegativeControls() {
+  console.log('=== Negative Control Tests ===\n');
+  
+  let passed = 0;
+  let total = 0;
+
+  // Test 1: consistency-check fails on bad-frontmatter-colon
+  console.log('Test 1: consistency-check fails on unquoted YAML colon...');
+  total++;
+  copyFixture('bad-frontmatter-colon', 'bad-frontmatter-colon');
+  const result1 = runCheck('consistency-check.js');
+  removeFixture('bad-frontmatter-colon');
+  
+  if (result1.status !== 0 && result1.stderr.includes('unquoted colon')) {
+    console.log('  ✅ PASS - consistency-check catches unquoted colon');
+    passed++;
+  } else {
+    console.log('  ❌ FAIL - consistency-check did not catch unquoted colon');
+    console.log(`     Exit code: ${result1.status}`);
+    console.log(`     Stderr: ${result1.stderr}`);
+  }
+
+  // Test 2: consistency-check fails on missing delimiter
+  console.log('\nTest 2: consistency-check fails on missing delimiter...');
+  total++;
+  copyFixture('missing-delimiter', 'missing-delimiter');
+  const result2 = runCheck('consistency-check.js');
+  removeFixture('missing-delimiter');
+  
+  if (result2.status !== 0 && (result2.stderr.includes('delimiter') || result2.stderr.includes('YAML'))) {
+    console.log('  ✅ PASS - consistency-check catches missing delimiter');
+    passed++;
+  } else {
+    console.log('  ❌ FAIL - consistency-check did not catch missing delimiter');
+    console.log(`     Exit code: ${result2.status}`);
+    console.log(`     Stderr: ${result2.stderr}`);
+  }
+
+  // Test 3: description-collision fails on duplicate descriptions
+  console.log('\nTest 3: description-collision fails on duplicate descriptions...');
+  total++;
+  copyFixture('duplicate-desc-skill-a', 'duplicate-desc-skill-a');
+  copyFixture('duplicate-desc-skill-b', 'duplicate-desc-skill-b');
+  const result3 = runCheck('description-collision.js');
+  removeFixture('duplicate-desc-skill-a');
+  removeFixture('duplicate-desc-skill-b');
+  
+  if (result3.status !== 0 && (result3.stdout.includes('COLLISION') || result3.stderr.includes('COLLISION'))) {
+    console.log('  ✅ PASS - description-collision catches duplicate descriptions');
+    passed++;
+  } else {
+    console.log('  ❌ FAIL - description-collision did not catch duplicates');
+    console.log(`     Exit code: ${result3.status}`);
+    console.log(`     Stderr: ${result3.stderr}`);
+    console.log(`     Stdout: ${result3.stdout}`);
+  }
+
+  // Test 4: consistency-check fails on over-budget skill (token budget gate)
+  console.log('\nTest 4: consistency-check fails on over-budget skill...');
+  total++;
+  copyFixture('over-budget-skill', 'over-budget-skill');
+  const result4 = runCheck('consistency-check.js');
+  removeFixture('over-budget-skill');
+  
+  if (result4.status !== 0 && result4.stderr.includes('exceeds hard limit')) {
+    console.log('  ✅ PASS - consistency-check catches over-budget skill');
+    passed++;
+  } else {
+    console.log('  ❌ FAIL - consistency-check did not catch over-budget skill');
+    console.log(`     Exit code: ${result4.status}`);
+    console.log(`     Stderr: ${result4.stderr}`);
+    console.log(`     Stdout: ${result4.stdout}`);
+  }
+
+  // Test 5: Verify normal repo still passes
+  console.log('\nTest 5: Normal repo passes all checks...');
+  total++;
+  const result5a = runCheck('consistency-check.js');
+  const result5b = runCheck('description-collision.js');
+  
+  if (result5a.status === 0 && result5b.status === 0) {
+    console.log('  ✅ PASS - Normal repo passes all checks');
+    passed++;
+  } else {
+    console.log('  ❌ FAIL - Normal repo fails checks');
+    console.log(`     consistency-check: ${result5a.status}`);
+    console.log(`     description-collision: ${result5b.status}`);
+  }
+
+  // Summary
+  console.log(`\n=== SUMMARY ===`);
+  console.log(`Tests: ${total}`);
+  console.log(`Passed: ${passed}`);
+  console.log(`Failed: ${total - passed}`);
+  
+  if (passed === total) {
+    console.log('\n🎉 ALL NEGATIVE CONTROLS PASSED');
+    process.exit(0);
+  } else {
+    console.log('\n❌ SOME NEGATIVE CONTROLS FAILED');
+    process.exit(1);
+  }
+}
+
+testNegativeControls().catch(err => {
+  console.error('Test error:', err);
+  process.exit(1);
+});

package/commitlint.config.js

@@ -0,0 +1,25 @@
+module.exports = {
+  extends: ['@commitlint/config-conventional'],
+  rules: {
+    'type-enum': [
+      2,
+      'always',
+      [
+        'feat',
+        'fix',
+        'docs',
+        'style',
+        'refactor',
+        'perf',
+        'test',
+        'chore',
+        'revert',
+        'ci',
+        'build'
+      ]
+    ],
+    'subject-case': [2, 'always', 'sentence-case'],
+    'subject-empty': [2, 'never'],
+    'header-max-length': [2, 'always', 100]
+  }
+};