harness-everything 0.3.6 → 0.4.1

package/ci/router-phase2-safety.test.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const ROOT = path.resolve(__dirname, '..');
+const tierRouter = path.join(ROOT, 'harness-everything', 'scripts', 'tier-router.js');
+const { validateRouterContract } = require(path.join(ROOT, 'harness-everything', 'scripts', 'router-contract.js'));
+let failed = 0;
+
+function check(condition, message) {
+  if (condition) console.log(`  PASS ${message}`);
+  else {
+    console.error(`  FAIL ${message}`);
+    failed++;
+  }
+}
+
+function route(prompt) {
+  const contractPath = path.join(os.tmpdir(), `harness-router-phase2-safety-${process.pid}-${Math.random().toString(16).slice(2)}.json`);
+  const result = spawnSync(process.execPath, [tierRouter, prompt], {
+    cwd: ROOT,
+    encoding: 'utf8',
+    env: { ...process.env, HARNESS_ROUTER_CONTRACT_PATH: contractPath },
+  });
+  let contract = null;
+  if (fs.existsSync(contractPath)) {
+    contract = JSON.parse(fs.readFileSync(contractPath, 'utf8'));
+    fs.rmSync(contractPath, { force: true });
+  }
+  return { result, contract };
+}
+
+console.log('=== Router Phase 2 Explicit-Parallel Safety ===');
+
+const unproven = route('Use fable-parallel to audit the entire repository architecture.');
+check(unproven.result.status === 0, 'unproven explicit parallel request does not crash routing');
+check(Boolean(unproven.contract), 'unproven explicit parallel request emits a contract');
+if (unproven.contract) {
+  const validation = validateRouterContract(unproven.contract);
+  check(validation.valid, `serialized explicit request validates (${validation.errors.join('; ') || 'no errors'})`);
+  check(unproven.contract.taskShape.explicitRequest.strategy === 'fable-parallel', 'original explicit fable-parallel request remains recorded');
+  check(unproven.contract.workflowPlan.strategy === 'fable-staged', 'unproven parallel request serializes to fable-staged');
+  check(unproven.contract.workflowPlan.fallback.disposition === 'reduced', 'unproven parallel request exposes reduced fallback');
+  check(unproven.contract.workflowPlan.fallback.mode === 'serialized', 'unproven parallel request exposes serialized fallback mode');
+  check(unproven.contract.workflowPlan.fallback.reasonCodes.includes('parallel-preconditions-unproven'), 'unproven parallel request records precondition reason');
+}
+
+const overlap = route('Use fable-parallel with independent agents editing the same file across the entire repository.');
+check(overlap.result.status === 0, 'overlapping explicit parallel request does not crash routing');
+check(Boolean(overlap.contract), 'overlapping explicit parallel request emits a contract');
+if (overlap.contract) {
+  check(overlap.contract.taskShape.writeSetOverlap === 'overlap', 'overlap evidence is machine-visible');
+  check(overlap.contract.workflowPlan.strategy === 'fable-staged', 'explicit parallel overlap serializes to staged');
+  check(overlap.contract.workflowPlan.fallback.reasonCodes.includes('parallel-write-set-overlap'), 'overlap serialization has a deterministic reason');
+}
+
+const safe = route('Use fable-parallel for independent read-only security and architecture audits across the entire repository.');
+check(safe.result.status === 0, 'validated explicit parallel request routes successfully');
+check(Boolean(safe.contract), 'validated explicit parallel request emits a contract');
+if (safe.contract) {
+  const validation = validateRouterContract(safe.contract);
+  check(validation.valid, `validated explicit parallel contract passes (${validation.errors.join('; ') || 'no errors'})`);
+  check(safe.contract.workflowPlan.strategy === 'fable-parallel', 'validated explicit parallel request keeps fable-parallel');
+  check(safe.contract.workflowPlan.parallelism.allowed === true, 'validated explicit parallel request enables parallelism');
+  check(safe.contract.workflowPlan.fallback.disposition === 'none', 'validated explicit parallel request needs no fallback');
+}
+
+const contradiction = route('Use fable-parallel to audit the entire repository, but do not use parallel execution.');
+check(contradiction.result.status === 0, 'contradictory explicit request remains representable');
+check(Boolean(contradiction.contract), 'contradictory explicit request emits a contract');
+if (contradiction.contract) {
+  const validation = validateRouterContract(contradiction.contract);
+  check(validation.valid, `blocked contradiction is still a valid audit contract (${validation.errors.join('; ') || 'no errors'})`);
+  check(contradiction.contract.workflowPlan.fallback.disposition === 'blocked', 'explicit strategy/prohibition conflict is blocked');
+  check(contradiction.contract.workflowPlan.fallback.reasonCodes.includes('explicit-request-conflicts-with-prohibition'), 'explicit contradiction records its reason');
+}
+
+console.log(`\n${failed === 0 ? 'PASS' : 'FAIL'}: router phase 2 explicit-parallel safety (${failed} failure${failed === 1 ? '' : 's'})`);
+process.exit(failed === 0 ? 0 : 1);

package/ci/router-workflow-plan.test.js

@@ -0,0 +1,264 @@
+#!/usr/bin/env node
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const ROOT = path.resolve(__dirname, '..');
+const tierRouter = path.join(ROOT, 'harness-everything', 'scripts', 'tier-router.js');
+const kernelRouter = path.join(ROOT, 'harness-everything', 'scripts', 'kernel-router.js');
+const {
+  ITERATIVE_MAX_ITERATIONS,
+  validateRouterContract,
+} = require(path.join(ROOT, 'harness-everything', 'scripts', 'router-contract.js'));
+let failed = 0;
+
+function check(condition, message) {
+  if (condition) {
+    console.log(`  PASS ${message}`);
+  } else {
+    console.error(`  FAIL ${message}`);
+    failed++;
+  }
+}
+
+function runTier(prompt, options = {}) {
+  const contractPath = path.join(os.tmpdir(), `harness-router-test-${process.pid}-${Math.random().toString(16).slice(2)}.json`);
+  const context = options.context || null;
+  const args = context ? [tierRouter] : [tierRouter, prompt];
+  const result = spawnSync(process.execPath, args, {
+    cwd: ROOT,
+    encoding: 'utf8',
+    input: context ? JSON.stringify({ ...context, prompt }) : undefined,
+    env: {
+      ...process.env,
+      ...(options.env || {}),
+      HARNESS_ROUTER_CONTRACT_PATH: contractPath,
+    },
+  });
+
+  let contract = null;
+  if (fs.existsSync(contractPath)) {
+    contract = JSON.parse(fs.readFileSync(contractPath, 'utf8'));
+    fs.rmSync(contractPath, { force: true });
+  }
+  return { result, contract };
+}
+
+function validContract(run, label) {
+  check(run.result.status === 0, `${label}: tier router exits successfully`);
+  check(Boolean(run.contract), `${label}: structured contract exists`);
+  if (!run.contract) return false;
+  const validation = validateRouterContract(run.contract);
+  check(validation.valid, `${label}: contract validates (${validation.errors.join('; ') || 'no errors'})`);
+  return validation.valid;
+}
+
+console.log('=== Router Workflow Plan Contract — Phase 2 ===');
+
+const direct = runTier('Update one README typo');
+if (validContract(direct, 'direct-single')) {
+  const plan = direct.contract.workflowPlan;
+  check(plan.tier === 'tier1', 'one-file trivial task remains tier1');
+  check(plan.strategy === 'direct-single', 'one-file trivial task selects direct-single');
+  check(plan.strategySelection === 'selected', 'direct-single is a selected strategy');
+  check(plan.workspace.required === false, 'direct-single does not require a workspace');
+  check(plan.parallelism.allowed === false, 'direct-single does not parallelize');
+}
+
+const iterative = runTier('Fix this checkout bug and add a regression test.');
+if (validContract(iterative, 'iterative-single')) {
+  const plan = iterative.contract.workflowPlan;
+  check(plan.tier === 'tier2', 'ordinary test-first bug fix is tier2');
+  check(plan.strategy === 'iterative-single', 'ordinary bug fix selects iterative-single');
+  check(plan.limits.maxIterations === ITERATIVE_MAX_ITERATIONS, 'iterative-single has an explicit loop budget');
+  check(plan.requiredInvariants.includes('objective-verification'), 'iterative-single carries objective verification invariant');
+  check(plan.requiredInvariants.includes('loop-budget'), 'iterative-single carries loop-budget invariant');
+  check(plan.suggestedSkills.includes('tdd'), 'TDD stays advisory rather than an invariant');
+  check(!plan.requiredInvariants.includes('tdd'), 'suggested skills are separated from mandatory invariants');
+}
+
+const staged = runTier('Refactor the entire authentication architecture in dependent stages.');
+if (validContract(staged, 'fable-staged')) {
+  const plan = staged.contract.workflowPlan;
+  check(plan.tier === 'tier3', 'multi-stage architecture work is tier3');
+  check(plan.strategy === 'fable-staged', 'dependent multi-stage work selects fable-staged');
+  check(plan.parallelism.allowed === false, 'dependent stages remain sequential');
+  check(plan.verification.mode === 'cold-verifier', 'fable-staged requires cold verification');
+}
+
+const parallelPrompt = 'Audit the entire repository with independent read-only security, architecture, and documentation workstreams.';
+const parallel = runTier(parallelPrompt);
+if (validContract(parallel, 'fable-parallel')) {
+  const shape = parallel.contract.taskShape;
+  const plan = parallel.contract.workflowPlan;
+  check(shape.dependencyGraph === 'independent', 'parallel task records independent dependency graph');
+  check(shape.writeSetOverlap === 'read-only', 'parallel audit records read-only write set');
+  check(plan.strategy === 'fable-parallel', 'independent read-only audits select fable-parallel');
+  check(plan.parallelism.allowed === true, 'fable-parallel exposes allowed=true');
+  check(plan.requiredInvariants.includes('synthesis-barrier'), 'parallel strategy requires synthesis barrier');
+}
+
+const sharedWrite = runTier('Audit the entire repository with independent agents in parallel editing the same file.');
+if (validContract(sharedWrite, 'shared-write serialization')) {
+  check(sharedWrite.contract.taskShape.writeSetOverlap === 'overlap', 'same-file writers are recorded as overlapping');
+  check(sharedWrite.contract.workflowPlan.strategy === 'fable-staged', 'same-file writers are serialized instead of fable-parallel');
+  check(sharedWrite.contract.workflowPlan.parallelism.allowed === false, 'overlapping writes cannot dispatch in parallel');
+}
+
+const workspace = runTier('Audit the entire repository as a durable multi-session effort with reusable specialists across security and architecture.');
+if (validContract(workspace, 'multi-agent workspace')) {
+  const plan = workspace.contract.workflowPlan;
+  check(plan.strategy === 'fable-multi-agent-workspace', 'durable reusable specialists select workspace topology');
+  check(plan.workspace.required === true, 'workspace topology marks workspace required');
+  check(plan.memory.read === 'workspace-index', 'workspace topology scopes reads through workspace index');
+  check(plan.memory.write === 'propose', 'workspace memory writes remain proposals');
+}
+
+const unknown = runTier('frobnicate the quux');
+if (validContract(unknown, 'unclassified')) {
+  check(unknown.contract.classification.tier === 'unclassified', 'no matched signal remains unclassified');
+  check(unknown.contract.workflowPlan.strategy === null, 'unclassified does not silently select direct-single');
+  check(unknown.contract.workflowPlan.strategySelection === 'deferred', 'unclassified strategy selection is deferred');
+  check(unknown.contract.workflowPlan.reasonCodes.includes('unclassified-strategy-deferred'), 'deferred selection has deterministic reason code');
+}
+
+const destructive = runTier('Drop the prod database table and force push to main.');
+if (validContract(destructive, 'actionGate')) {
+  const gate = destructive.contract.workflowPlan.actionGate;
+  check(gate.required === true, 'irreversible command intent requires actionGate independent of tier');
+  check(gate.reasonCodes.includes('irreversible-action'), 'actionGate records irreversible-action');
+  check(gate.disposition === 'pending-approval', 'actionGate closes at pending approval before execution');
+  check(destructive.contract.workflowPlan.requiredInvariants.includes('pre-action-approval'), 'actionGate adds pre-action invariant');
+}
+
+const external = runTier('Deploy to production and publish the package release.');
+if (validContract(external, 'external side effect')) {
+  check(external.contract.workflowPlan.actionGate.required === true, 'external side effect requires actionGate');
+  check(external.contract.workflowPlan.actionGate.reasonCodes.includes('external-side-effect'), 'external side effect reason is explicit');
+}
+
+const fableTypo = runTier('fix typo in the fable-mode readme');
+if (validContract(fableTypo, 'fable keyword trivial edit')) {
+  check(fableTypo.contract.classification.tier === 'tier1', 'fable keyword does not force Tier 3 on a trivial README typo');
+  check(fableTypo.contract.workflowPlan.strategy === 'direct-single', 'fable README typo remains direct-single');
+}
+
+const explicitFable = runTier('fable on opus audit the entire repository architecture');
+if (validContract(explicitFable, 'explicit Fable model')) {
+  check(explicitFable.contract.workflowPlan.strategy === 'fable-staged', 'explicit Fable request selects Fable topology');
+  check(explicitFable.contract.workflowPlan.modelSelection.requested === 'opus', 'explicit Fable model stays delegated to model selector');
+  check(explicitFable.result.stdout.includes('REQUESTED FABLE MODEL MODE: opus'), 'legacy explicit Fable route remains visible');
+}
+
+const explicitOverride = runTier('Use iterative-single to audit the entire repository architecture.');
+if (validContract(explicitOverride, 'explicit strategy override')) {
+  check(explicitOverride.contract.workflowPlan.strategy === 'iterative-single', 'explicit user strategy overrides derived Tier 3 topology');
+  check(explicitOverride.contract.workflowPlan.reasonCodes.includes('explicit-strategy-request'), 'explicit strategy override is recorded');
+}
+
+const prohibitedParallel = runTier(`${parallelPrompt} Do not use parallel execution.`);
+if (validContract(prohibitedParallel, 'parallel prohibition')) {
+  check(prohibitedParallel.contract.workflowPlan.strategy === 'fable-staged', 'explicit no-parallel prohibition serializes derived parallel work');
+  check(prohibitedParallel.contract.workflowPlan.fallback.disposition === 'reduced', 'parallel prohibition is a visible reduced fallback');
+  check(prohibitedParallel.contract.workflowPlan.fallback.reasonCodes.includes('user-prohibited-parallelism'), 'parallel prohibition reason is recorded');
+}
+
+const noParallelCapability = runTier(parallelPrompt, {
+  context: { hostCapabilities: { parallelCalls: 'unavailable', subagents: 'available' } },
+});
+if (validContract(noParallelCapability, 'parallel capability fallback')) {
+  check(noParallelCapability.contract.workflowPlan.strategy === 'fable-staged', 'missing parallel capability serializes to fable-staged');
+  check(noParallelCapability.contract.workflowPlan.fallback.mode === 'serialized', 'parallel capability fallback is visibly serialized');
+  check(noParallelCapability.contract.workflowPlan.fallback.reasonCodes.includes('parallel-capability-unavailable'), 'missing parallel capability reason is recorded');
+}
+
+const concurrencyOne = runTier(parallelPrompt, {
+  context: { constraints: { concurrency: 1 }, hostCapabilities: { parallelCalls: 'available', subagents: 'available' } },
+});
+if (validContract(concurrencyOne, 'concurrency budget fallback')) {
+  check(concurrencyOne.contract.workflowPlan.strategy === 'fable-staged', 'concurrency=1 serializes fable-parallel');
+  check(concurrencyOne.contract.workflowPlan.fallback.reasonCodes.includes('concurrency-budget-serializes'), 'concurrency budget reason is recorded');
+}
+
+const noWorkspaceState = runTier('Audit the entire repository as a durable multi-session effort with reusable specialists across security and architecture.', {
+  context: { hostCapabilities: { state: 'unavailable', subagents: 'available' } },
+});
+if (validContract(noWorkspaceState, 'workspace state block')) {
+  check(noWorkspaceState.contract.workflowPlan.strategy === 'fable-multi-agent-workspace', 'workspace requirement remains explicit when state is unavailable');
+  check(noWorkspaceState.contract.workflowPlan.fallback.disposition === 'blocked', 'missing durable state blocks rather than silently downgrades');
+  check(noWorkspaceState.contract.workflowPlan.fallback.reasonCodes.includes('workspace-state-unavailable'), 'workspace block reason is recorded');
+}
+
+const unavailableModel = runTier('fable on opus audit the entire repository architecture', {
+  context: { hostCapabilities: { modelAvailability: 'unavailable', subagents: 'available' } },
+});
+if (validContract(unavailableModel, 'model capability block')) {
+  check(unavailableModel.contract.workflowPlan.modelSelection.requested === 'opus', 'router never remaps requested branded model');
+  check(unavailableModel.contract.workflowPlan.fallback.disposition === 'blocked', 'known unavailable model is visibly blocked');
+  check(unavailableModel.contract.workflowPlan.fallback.reasonCodes.includes('requested-model-capability-unavailable'), 'model capability block reason is recorded');
+}
+
+const hookUnavailable = runTier('Drop the prod database table.', {
+  context: { hostCapabilities: { hooks: 'unavailable' } },
+});
+if (validContract(hookUnavailable, 'actionGate capability block')) {
+  check(hookUnavailable.contract.workflowPlan.actionGate.required === true, 'actionGate requirement survives missing hook capability');
+  check(hookUnavailable.contract.workflowPlan.fallback.disposition === 'blocked', 'known missing action-gate hook blocks side effect');
+  check(hookUnavailable.contract.workflowPlan.fallback.reasonCodes.includes('action-gate-hook-unavailable'), 'missing hook capability is visible');
+}
+
+const guideDedup = runTier('security audit login authentication');
+const securityGuideCount = (guideDedup.result.stdout.match(/security-review\/SKILL\.md/g) || []).length;
+check(securityGuideCount === 1, 'duplicate guide matches emit security-review/SKILL.md exactly once by path');
+
+const repeatA = runTier('Fix this checkout bug and add a regression test.');
+const repeatB = runTier('Fix this checkout bug and add a regression test.');
+if (repeatA.contract && repeatB.contract) {
+  check(JSON.stringify(repeatA.contract) === JSON.stringify(repeatB.contract), 'same normalized input produces byte-equivalent structured contracts');
+}
+
+const invalidConfigPath = path.join(os.tmpdir(), `harness-invalid-routing-${process.pid}.json`);
+fs.writeFileSync(invalidConfigPath, '{ invalid json', 'utf8');
+const degraded = runTier('frobnicate the quux', { env: { HARNESS_ROUTING_CONFIG_PATH: invalidConfigPath } });
+fs.rmSync(invalidConfigPath, { force: true });
+if (validContract(degraded, 'degraded routing')) {
+  check(degraded.contract.routingStatus === 'degraded', 'invalid routing config emits routingStatus=degraded');
+  check(degraded.contract.classification.tier === 'unclassified', 'invalid routing config never silently becomes Tier 1');
+  check(degraded.contract.workflowPlan.strategy === null, 'degraded routing does not select a strategy');
+  check(degraded.contract.workflowPlan.reasonCodes.includes('routing-degraded-strategy-deferred'), 'degraded strategy deferral is explicit');
+}
+
+const kernelUnknown = spawnSync(process.execPath, [kernelRouter, 'frobnicate the quux'], {
+  cwd: ROOT,
+  encoding: 'utf8',
+});
+check(kernelUnknown.status === 0, 'kernel consumes selected/deferred structured contract successfully');
+check(kernelUnknown.stdout.includes('ROUTER WORKFLOW PLAN (JSON)'), 'kernel emits the Phase 2 workflow-plan checkpoint');
+check(kernelUnknown.stdout.includes('"strategySelection":"deferred"'), 'kernel exposes deferred unclassified selection');
+check(!/RECOMMENDED TIER:\s*Tier 1/i.test(kernelUnknown.stdout), 'kernel does not expose a silent Tier 1 fallback for unmatched prompts');
+
+const kernelIterative = spawnSync(process.execPath, [kernelRouter, 'Fix this checkout bug and add a regression test.'], {
+  cwd: ROOT,
+  encoding: 'utf8',
+});
+check(kernelIterative.stdout.includes('"strategy":"iterative-single"'), 'kernel consumes strategy from structured plan');
+check(kernelIterative.stdout.includes('loop-budget:'), 'kernel prints dynamic plan invariants');
+check(kernelIterative.stdout.includes('   - tdd'), 'kernel prints advisory skills separately');
+
+for (const schemaPath of [
+  'harness-everything/schemas/router-task-shape.schema.json',
+  'harness-everything/schemas/router-workflow-plan.schema.json',
+]) {
+  try {
+    const schema = JSON.parse(fs.readFileSync(path.join(ROOT, schemaPath), 'utf8'));
+    assert.strictEqual(schema.$schema, 'https://json-schema.org/draft/2020-12/schema');
+    check(true, `${schemaPath} is valid JSON Schema metadata`);
+  } catch (err) {
+    check(false, `${schemaPath} parses: ${err.message}`);
+  }
+}
+
+console.log(`\n${failed === 0 ? 'PASS' : 'FAIL'}: router workflow plan phase 2 (${failed} failure${failed === 1 ? '' : 's'})`);
+process.exit(failed === 0 ? 0 : 1);

package/ci/routing-check.js

@@ -3,8 +3,9 @@
  * Routing Eval Local Gate
  * 
  * Runs `waza spec verify` for all skills' eval.yaml files.
- * A missing waza installation is an explicit non-pass. CI installs waza, and
- * local callers should not mistake an infrastructure skip for coverage.
+ * A missing waza installation is a clear, successful skip. CI installs waza;
+ * local callers should see the infrastructure boundary without a false gate
+ * failure.
  */
 
 const fs = require('fs');
@@ -38,9 +39,9 @@ function runRoutingCheck() {
   console.log('=== Routing Eval Check ===\n');
   
   if (!checkWaza()) {
-    console.error('waza not installed: routing eval check is NOT RUN');
-    console.log('   Provide the waza binary locally; CI installs it from the official installer in .github/workflows/ci.yml.');
-    process.exit(2);
+    console.log('SKIP: waza not installed; routing eval check was not run.');
+    console.log('   CI installs it from the official installer in .github/workflows/ci.yml.');
+    process.exit(0);
   }
   
   const skills = discoverSkills();

package/ci/runner.js

@@ -30,12 +30,14 @@ files.forEach(file => {
       env: { ...process.env, HARNESS_EVAL: 'true' }
     });
     
-    // Parse the output to find the recommended tier (accept the old
-    // "REQUIRED TIER" label too so the runner works across versions)
-    const tierMatch = output.match(/(?:RECOMMENDED|REQUIRED) TIER: (Tier \d+)/i) || output.match(/(Tier \d+)/i);
+    // Parse the human-readable classification. `Unclassified` is a first-class
+    // result: no matched signal must never be converted back into Tier 1 by
+    // the test harness itself.
+    const tierMatch = output.match(/(?:RECOMMENDED|REQUIRED) TIER:\s*(Tier \d+|Unclassified)/i)
+      || output.match(/\b(Tier \d+|Unclassified)\b/i);
     const actualTier = tierMatch ? tierMatch[1] : "Unknown";
     
-    if (actualTier.includes(data.expected_tier)) {
+    if (actualTier.toLowerCase().includes(String(data.expected_tier).toLowerCase())) {
       console.log(`  -> Result: PASS (Got ${actualTier})\n`);
       passed++;
     } else {
@@ -53,4 +55,3 @@ console.log(`--- Summary: ${passed} passed, ${failed} failed ---`);
 if (failed > 0) {
   process.exit(1);
 }
-

package/ci/skill-routing-check.js

@@ -1,8 +1,9 @@
 #!/usr/bin/env node
 /**
- * Deterministic route coverage gate for every skill's positive eval prompts.
- * This complements waza: it executes the real local tier-router and does not
- * require a network-installed evaluator.
+ * Deterministic route coverage gate for directly-routable skills plus an
+ * explicit classification gate for nested skills. A new nested SKILL.md must
+ * declare whether it is parent-routed or internal; otherwise CI fails instead
+ * of silently ignoring it.
  */
 
 const fs = require('fs');
@@ -14,17 +15,38 @@ const ROOT = path.resolve(__dirname, '..');
 const ROUTER = path.join(ROOT, 'harness-everything', 'scripts', 'tier-router.js');
 const EVALS = path.join(ROOT, 'evals');
 
-function walk(dir) {
+const NESTED_ROUTING = new Map([
+  ['fable-mode/execution-guardrails', 'internal'],
+  ['fable-mode/fable-haiku', 'parent'],
+  ['fable-mode/fable-sonnet', 'parent'],
+  ['fable-mode/fable-opus', 'parent'],
+]);
+
+function walkPositiveTasks(dir) {
   if (!fs.existsSync(dir)) return [];
   const files = [];
   for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
     const full = path.join(dir, entry.name);
-    if (entry.isDirectory()) files.push(...walk(full));
+    if (entry.isDirectory()) files.push(...walkPositiveTasks(full));
     else if (/^positive.*\.ya?ml$/i.test(entry.name)) files.push(full);
   }
   return files;
 }
 
+function discoverNestedSkills(skillDir) {
+  const found = [];
+  function visit(dir) {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (!entry.isDirectory() || entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
+      const child = path.join(dir, entry.name);
+      if (fs.existsSync(path.join(child, 'SKILL.md'))) found.push(path.relative(ROOT, child).replace(/\\/g, '/'));
+      visit(child);
+    }
+  }
+  visit(skillDir);
+  return found;
+}
+
 function expectedMarker(skill) {
   if (skill === 'harness-everything') return 'RECOMMENDED TIER';
   if (skill === 'todo-driven-workflow') return 'BASE EXECUTION LOOP';
@@ -38,6 +60,24 @@ const skillDirs = fs.readdirSync(ROOT, { withFileTypes: true })
 
 let failures = 0;
 let cases = 0;
+
+const nestedOnDisk = skillDirs.flatMap(skill => discoverNestedSkills(path.join(ROOT, skill))).sort();
+for (const nested of nestedOnDisk) {
+  const classification = NESTED_ROUTING.get(nested);
+  if (!classification) {
+    console.error(`FAIL ${nested}/SKILL.md: nested skill has no explicit routing classification`);
+    failures++;
+  } else {
+    console.log(`PASS ${nested}/SKILL.md: routing=${classification}`);
+  }
+}
+for (const [nested, classification] of NESTED_ROUTING) {
+  if (!nestedOnDisk.includes(nested)) {
+    console.error(`FAIL routing classification ${nested}=${classification}: SKILL.md no longer exists`);
+    failures++;
+  }
+}
+
 for (const skill of skillDirs) {
   const skillText = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf8');
   const frontmatter = skillText.match(/^---\r?\n([\s\S]*?)\r?\n---/);
@@ -49,7 +89,7 @@ for (const skill of skillDirs) {
     failures++;
     continue;
   }
-  const taskFiles = walk(path.join(EVALS, skill, 'tasks'));
+  const taskFiles = walkPositiveTasks(path.join(EVALS, skill, 'tasks'));
   if (taskFiles.length === 0) {
     console.error(`FAIL ${skill}: no positive task files`);
     failures++;
@@ -79,10 +119,7 @@ for (const skill of skillDirs) {
       continue;
     }
 
-    const result = spawnSync(process.execPath, [ROUTER, prompt], {
-      cwd: ROOT,
-      encoding: 'utf8',
-    });
+    const result = spawnSync(process.execPath, [ROUTER, prompt], { cwd: ROOT, encoding: 'utf8' });
     const output = `${result.stdout || ''}\n${result.stderr || ''}`;
     const marker = expectedMarker(skill);
     if (result.status !== 0 || !output.includes(marker)) {
@@ -94,5 +131,5 @@ for (const skill of skillDirs) {
   }
 }
 
-console.log(`\nRoute coverage: ${cases - failures}/${cases} positive cases passed across ${skillDirs.length} skills.`);
+console.log(`\nRoute coverage: ${cases} positive case(s) checked across ${skillDirs.length} direct skills; ${nestedOnDisk.length} nested skill(s) classified.`);
 process.exit(failures ? 1 : 0);

package/ci/test-behavioral-case-validation.js

@@ -0,0 +1,62 @@
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { parseSimpleYaml } = require('../behavioral-evals/run');
+const { grade, parseTranscriptEvents } = require('../behavioral-evals/run');
+const { validateCase, validateDirectory, negativeControls } = require('../behavioral-evals/case-validator');
+const { canonicalRubric, extractTrace } = require('./ab-test-harness');
+
+const casesDir = path.join(__dirname, '..', 'behavioral-evals', 'cases');
+const failures = validateDirectory(casesDir, parseSimpleYaml);
+assert.deepStrictEqual(failures, [], failures.map(failure => `${failure.file}: ${failure.errors.join('; ')}`).join('\n'));
+assert.strictEqual(negativeControls().length, 0, 'negative controls must be rejected by the validator');
+assert.deepStrictEqual(validateCase({
+  id: 'tool-call-schema',
+  prompt: 'inspect the fixture',
+  max_turns: 1,
+  fixture: { files: [{ path: 'index.js', content: '' }] },
+  expectations: [{ type: 'tool_call', value: 'Bash' }]
+}), [], 'tool_call must remain part of the runner/validator schema');
+
+const transcriptDir = fs.mkdtempSync(path.join(os.tmpdir(), 'behavioral-case-validation-'));
+try {
+  const transcriptPath = path.join(transcriptDir, 'trace.jsonl');
+  const events = [
+    { type: 'tool_use', part: { type: 'tool', tool: 'Edit', callID: 'edit-1', state: { status: 'completed', input: { file_path: 'slug.js' } } } },
+    { type: 'tool_use', part: { type: 'tool', tool: 'Bash', callID: 'test-1', state: { status: 'running', input: { command: 'npm test' } } } },
+    { type: 'tool_use', part: { type: 'tool', tool: 'Bash', callID: 'test-1', state: { status: 'completed', input: { command: 'npm test' } } } },
+    { type: 'tool_use', part: { type: 'tool', tool: 'Bash', callID: 'denied-1', state: { status: 'denied', input: { command: 'npm install' } } } },
+    { type: 'text', part: { type: 'text', text: 'done' } },
+  ];
+  fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+  const parsed = parseTranscriptEvents(transcriptPath);
+  assert.strictEqual(parsed.events.filter(event => event.kind === 'tool').length, 3, 'tool lifecycle events should be merged by call id');
+  const graded = grade({ expectations: [
+    { type: 'tool_executed', value: { command: 'npm test' }, after_edit: true },
+    { type: 'tool_denied', value: { command: 'npm install' } },
+    { type: 'trace_contains', value: 'done' },
+  ] }, transcriptDir, transcriptPath);
+  assert.strictEqual(graded.passed, true, 'execution assertions should use ordered agent events');
+  const spoofPath = path.join(transcriptDir, 'spoof.json');
+  fs.writeFileSync(spoofPath, JSON.stringify({ result: 'I ran npm test after editing.' }));
+  assert.strictEqual(grade({ expectations: [{ type: 'tool_executed', value: { command: 'npm test' } }] }, transcriptDir, spoofPath).passed, false, 'final text must not count as tool execution');
+  assert.deepStrictEqual(
+    canonicalRubric([
+      { type: 'trace_contains', value: 'b', description: 'second' },
+      { type: 'trace_contains', value: 'a', description: 'first' },
+    ]),
+    canonicalRubric([
+      { type: 'trace_contains', value: 'a', description: 'renamed' },
+      { type: 'trace_contains', value: 'b', description: 'also renamed' },
+    ]),
+    'A/B rubric normalization should ignore descriptions and criterion order'
+  );
+  assert.match(extractTrace(transcriptPath), /\[Bash\].*npm test/, 'A/B trace extraction should preserve tool event inputs');
+} finally {
+  fs.rmSync(transcriptDir, { recursive: true, force: true });
+}
+
+console.log(`Behavioral case validation passed (${fs.readdirSync(casesDir).filter(name => name.endsWith('.yaml')).length} cases + negative controls + ordered execution evidence).`);

package/ci/test-helper.js

@@ -1,8 +1,15 @@
 const { execSync } = require('child_process');
 const fs = require('fs');
 const path = require('path');
+const os = require('os');
 
 process.env.CLAUDE = '1'; // Force Claude Code state path
+// Mechanism tests use a fixed session id. Keep each test process hermetic so
+// prior worktrees or an earlier local run cannot make the session registry
+// ambiguous under the global Harness state home.
+const ownsStateHome = !process.env.HARNESS_STATE_HOME;
+const isolatedStateHome = process.env.HARNESS_STATE_HOME || fs.mkdtempSync(path.join(os.tmpdir(), 'harness-mechanism-state-'));
+process.env.HARNESS_STATE_HOME = isolatedStateHome;
 
 const { getWorkspaceRoot, getSessionDir } = require('../hooks/scripts/lib/harness-state');
 
@@ -81,6 +88,9 @@ function cleanup() {
     try { if (fs.existsSync(d)) fs.rmSync(d, { recursive: true, force: true }); } catch (e) { /* best-effort */ }
   }
   try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch (e) { /* best-effort */ }
+  if (ownsStateHome) {
+    try { fs.rmSync(isolatedStateHome, { recursive: true, force: true }); } catch (e) { /* best-effort */ }
+  }
 }
 
 process.on('exit', cleanup);