harness-everything 0.3.6 → 0.4.0

package/ci/ab-test-cases/grill-me-adversarial.yaml

@@ -4,15 +4,25 @@ name: "Grill-me skill makes agent ask adversarial questions instead of agreeing"
 prompt: |
   I want to build a REST API for user management with JWT
   authentication. What do you think?
+success_rubric:
+  - type: trace_contains
+    value: "?"
+    description: "agent asks at least one clarifying question"
+  - type: trace_not_contains
+    value: "POST /auth/register"
+    description: "agent does not immediately scaffold endpoints before clarifying"
 baseline_rubric:
+  - type: trace_contains
+    value: "?"
+    description: "agent asks at least one clarifying question"
   - type: trace_not_contains
-    value: "What's the actual problem"
-    description: "baseline does NOT challenge the fundamental assumption"
+    value: "POST /auth/register"
+    description: "agent does not immediately scaffold endpoints before clarifying"
 treatment_rubric:
   - type: trace_contains
     value: "?"
-    description: "treatment asks at least one question"
+    description: "agent asks at least one clarifying question"
   - type: trace_not_contains
     value: "POST /auth/register"
-    description: "treatment does NOT immediately scaffold endpoints (asks first)"
+    description: "agent does not immediately scaffold endpoints before clarifying"
 verdict_rule: "baseline_fail AND treatment_pass => EFFECTIVE"

package/ci/ab-test-cases/tdd-test-first.yaml

@@ -6,18 +6,25 @@ prompt: |
   the input string is a palindrome (case-insensitive, ignoring
   non-alphanumeric characters). Handle edge cases. Show your
   complete implementation.
+success_rubric:
+  - type: trace_contains
+    value: "def test_"
+    description: "agent writes a test function"
+  - type: trace_contains
+    value: "FAIL"
+    description: "agent records a failing RED phase"
 baseline_rubric:
-  - type: trace_not_contains
+  - type: trace_contains
     value: "def test_"
-    description: "baseline does NOT write test function first"
-  - type: trace_not_contains
+    description: "agent writes a test function"
+  - type: trace_contains
     value: "FAIL"
-    description: "baseline does NOT confirm test failure (no RED phase)"
+    description: "agent records a failing RED phase"
 treatment_rubric:
   - type: trace_contains
     value: "def test_"
-    description: "treatment DOES write test function first"
+    description: "agent writes a test function"
   - type: trace_contains
     value: "FAIL"
-    description: "treatment confirms test failure (RED phase)"
+    description: "agent records a failing RED phase"
 verdict_rule: "baseline_fail AND treatment_pass => EFFECTIVE"

package/ci/ab-test-cases/verification-loop-gates.yaml

@@ -5,21 +5,34 @@ prompt: |
   I've finished implementing the file upload feature. It handles
   multipart uploads, validates file types, and stores to S3.
   Can you verify it's ready for production?
+success_rubric:
+  - type: trace_contains
+    value: "npm run build"
+    description: "agent lists build verification gate"
+  - type: trace_contains
+    value: "npm run lint"
+    description: "agent lists lint verification gate"
+  - type: trace_contains
+    value: "npm run test"
+    description: "agent lists test verification gate"
 baseline_rubric:
-  - type: trace_not_contains
+  - type: trace_contains
     value: "npm run build"
-    description: "baseline does NOT list build verification"
-  - type: trace_not_contains
+    description: "agent lists build verification gate"
+  - type: trace_contains
     value: "npm run lint"
-    description: "baseline does NOT list lint verification"
+    description: "agent lists lint verification gate"
+  - type: trace_contains
+    value: "npm run test"
+    description: "agent lists test verification gate"
 treatment_rubric:
   - type: trace_contains
     value: "npm run build"
-    description: "treatment lists build verification gate"
+    description: "agent lists build verification gate"
   - type: trace_contains
     value: "npm run lint"
-    description: "treatment lists lint verification gate"
+    description: "agent lists lint verification gate"
   - type: trace_contains
     value: "npm run test"
-    description: "treatment lists test verification gate"
+    description: "agent lists test verification gate"
 verdict_rule: "baseline_fail AND treatment_pass => EFFECTIVE"

package/ci/ab-test-cases/verify-before-claim-cites.yaml

@@ -4,15 +4,16 @@ name: "Verify-before-claim skill makes agent cite sources instead of answering f
 prompt: |
   Does Python's `hashlib` module support the SHA3-256 algorithm?
   Give a direct answer.
+success_rubric:
+  - type: trace_contains
+    value: "docs.python.org"
+    description: "agent cites official Python documentation"
 baseline_rubric:
-  - type: trace_not_contains
+  - type: trace_contains
     value: "docs.python.org"
-    description: "baseline does NOT cite official documentation"
-  - type: trace_not_contains
-    value: "Added in version"
-    description: "baseline does NOT provide version history"
+    description: "agent cites official Python documentation"
 treatment_rubric:
   - type: trace_contains
     value: "docs.python.org"
-    description: "treatment DOES cite official documentation"
+    description: "agent cites official Python documentation"
 verdict_rule: "baseline_fail AND treatment_pass => EFFECTIVE"

package/ci/ab-test-harness.js

@@ -21,6 +21,7 @@ const os = require('os');
 const path = require('path');
 const crypto = require('crypto');
 const { execFileSync } = require('child_process');
+const { extractAgentTrace } = require('../behavioral-evals/run');
 
 const ROOT = path.resolve(__dirname, '..');
 const CASES_DIR = path.join(__dirname, 'ab-test-cases');
@@ -122,13 +123,57 @@ function validateCase(c) {
     errors.push('missing or empty baseline_rubric');
   if (!c.treatment_rubric || !Array.isArray(c.treatment_rubric) || c.treatment_rubric.length === 0)
     errors.push('missing or empty treatment_rubric');
+  if (!c.success_rubric || !Array.isArray(c.success_rubric) || c.success_rubric.length === 0) {
+    errors.push('missing or empty success_rubric');
+  } else {
+    const expected = JSON.stringify(canonicalRubric(c.success_rubric));
+    if (JSON.stringify(canonicalRubric(c.baseline_rubric)) !== expected)
+      errors.push('baseline_rubric must use the same success criterion as success_rubric');
+    if (JSON.stringify(canonicalRubric(c.treatment_rubric)) !== expected)
+      errors.push('treatment_rubric must use the same success criterion as success_rubric');
+  }
   if (!c.verdict_rule) errors.push('missing verdict_rule');
+  const rubricTypes = new Set(['trace_contains', 'trace_not_contains', 'trace_matches']);
+  for (const rubricName of ['success_rubric', 'baseline_rubric', 'treatment_rubric']) {
+    for (const item of c[rubricName] || []) {
+      if (!rubricTypes.has(item.type)) errors.push(`${rubricName} has unsupported type: ${item.type}`);
+      if (typeof item.value !== 'string' || !item.value) errors.push(`${rubricName} ${item.type} requires a non-empty value`);
+    }
+  }
   // Check skill file exists
   const skillPath = path.join(SKILLS_DIR, c.skill, 'SKILL.md');
   if (!fs.existsSync(skillPath)) errors.push(`skill file not found: ${skillPath}`);
   return errors;
 }
 
+// A baseline PASS must mean the same desired behavior as a treatment PASS.
+// Inverting the baseline with trace_not_contains makes the A/B verdict
+// compare different questions, so runCase grades the shared criterion below.
+function stableCanonical(value) {
+  if (Array.isArray(value)) return value.map(stableCanonical);
+  if (!value || typeof value !== 'object') return value;
+  return Object.keys(value).sort().reduce((result, key) => {
+    result[key] = stableCanonical(value[key]);
+    return result;
+  }, {});
+}
+
+function canonicalRubric(rubric) {
+  return (rubric || []).map((item) => {
+    const copy = { ...item };
+    delete copy.description;
+    return stableCanonical(copy);
+  }).sort((left, right) => {
+    const a = JSON.stringify(left);
+    const b = JSON.stringify(right);
+    return a < b ? -1 : a > b ? 1 : 0;
+  });
+}
+
+function normalizedRubric(c) {
+  return c.success_rubric || c.treatment_rubric || c.baseline_rubric || [];
+}
+
 // ---------------------------------------------------------------------------
 // Run a headless agent session
 function runHeadless(prompt, cwd, engine) {
@@ -153,36 +198,8 @@ function runHeadless(prompt, cwd, engine) {
 }
 
 function extractTrace(transcriptPath) {
-  try {
-    const raw = fs.readFileSync(transcriptPath, 'utf8');
-    const chunks = [];
-    let structured = false;
-    for (const line of raw.split('\n')) {
-      if (!line.trim()) continue;
-      let e;
-      try { e = JSON.parse(line); } catch { continue; }
-      const p = e && e.part;
-      if (!p) continue;
-      if (e.type === 'text' && typeof p.text === 'string') {
-        chunks.push(p.text);
-        structured = true;
-      } else if (String(e.type).includes('tool')) {
-        const input = p.state && p.state.input !== undefined ? p.state.input : p.input;
-        chunks.push(`[${p.tool || 'tool'}] ` + JSON.stringify(input ?? {}));
-        structured = true;
-      }
-    }
-    if (!structured) {
-      try {
-        const obj = JSON.parse(raw);
-        if (obj && typeof obj.result === 'string') return obj.result;
-      } catch { /* fall through */ }
-      return raw;
-    }
-    return chunks.join('\n');
-  } catch {
-    return '';
-  }
+  try { return extractAgentTrace(transcriptPath); }
+  catch { return ''; }
 }
 
 // ---------------------------------------------------------------------------
@@ -239,6 +256,24 @@ function sha256(value) {
   return crypto.createHash('sha256').update(value).digest('hex');
 }
 
+function sha256File(filePath) {
+  return sha256(fs.readFileSync(filePath));
+}
+
+function cliVersion(command) {
+  try {
+    return execFileSync(command, ['--version'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim().slice(0, 200);
+  } catch {
+    return null;
+  }
+}
+
+function sanitizeTrace(trace) {
+  return String(trace || '')
+    .replace(/[A-Za-z]:[\\/][^\s\]]*?(?:ab-test|harness)[^\s\]]*/gi, '<temp-path>')
+    .replace(/(?:[\\/](?:tmp|var[\\/]folders|Users[\\/][^\\/]+[\\/]AppData[\\/]Local[\\/]Temp)[\\/])[^\s\]]+/gi, '<temp-path>');
+}
+
 function countToolCalls(trace) {
   return (trace.match(/^\[[^\]]+\]/gm) || []).length;
 }
@@ -255,10 +290,22 @@ function wilsonInterval(successes, total) {
 
 // Every run is a paired experiment. The order is randomized to avoid making
 // the second arm systematically benefit from cache/session warm-up.
-function runCase(c, engine) {
+function runCase(c, engine, engineVersion) {
   console.log(`\n=== ${c.id}: ${c.name} ===`);
   console.log(`Skill: ${c.skill}`);
   const skillContent = fs.readFileSync(path.join(SKILLS_DIR, c.skill, 'SKILL.md'), 'utf8');
+  const rubric = normalizedRubric(c);
+  const model = process.env.AB_TEST_MODEL || 'openai/gpt-5-mini';
+  const provenance = {
+    case_sha256: sha256File(c.file),
+    rubric_sha256: sha256(JSON.stringify(canonicalRubric(rubric))),
+    harness_code_sha256: sha256File(__filename),
+    skill_sha256: sha256(skillContent),
+    cli: { command: engine, version: engineVersion },
+    cli_version_sha256: engineVersion ? sha256(engineVersion) : null,
+    model,
+    model_sha256: sha256(model),
+  };
   const treatmentPrompt = `You are following the ${c.skill} discipline. Here is your skill:\n\n${skillContent}\n\n---\n\n${c.prompt}`;
   const pairId = `${c.id}-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`;
   const order = crypto.randomInt(0, 2) === 0 ? ['baseline', 'treatment'] : ['treatment', 'baseline'];
@@ -271,14 +318,19 @@ function runCase(c, engine) {
     } catch (err) {
       console.error(`  ${arm} failed: ${err.message}`);
     }
-    const results = grade(trace, arm === 'treatment' ? c.treatment_rubric : c.baseline_rubric);
+    const results = grade(trace, rubric);
     const pass = results.every(r => r.pass);
+    const sanitizedTrace = sanitizeTrace(trace);
     arms[arm] = {
       pass,
       loaded_skills: arm === 'treatment' ? [c.skill] : [],
       attribution_required: arm === 'treatment',
-      results: results.map(r => ({ description: r.description, pass: r.pass })),
+      results: results.map(r => ({ type: r.type, value: r.value, description: r.description, pass: r.pass })),
       trace_preview: trace.slice(0, 500),
+      evidence: {
+        sanitized_trace: sanitizedTrace,
+        trace_sha256: sha256(sanitizedTrace),
+      },
       tool_call_count: countToolCalls(trace),
     };
     cleanWorkspace(ws);
@@ -298,8 +350,9 @@ function runCase(c, engine) {
     name: c.name,
     date: new Date().toISOString(),
     engine,
-    model: process.env.AB_TEST_MODEL || 'openai/gpt-5-mini',
-    protocol: 'paired-randomized-v1',
+    model,
+    protocol: 'paired-randomized-v2-rubric-normalized',
+    provenance,
     pair_id: pairId,
     arm_order: order,
     fixture_defined: false,
@@ -357,7 +410,7 @@ function main() {
     let effective = 0, ineffective = 0, inconclusive = 0, harmful = 0;
 
     for (const c of cases) {
-      const result = runCase(c, engine);
+      const result = runCase(c, engine, cliVersion(engine));
       results.push(result);
       if (result.verdict === 'EFFECTIVE') effective++;
       else if (result.verdict === 'INEFFECTIVE') ineffective++;
@@ -395,7 +448,18 @@ function main() {
       effective_rate_ci95: wilsonInterval(effective, completed),
       interpretation: 'Prompt-only cases have no workspace boundary; use behavioral-evals/run.js --arm both for fixture-bound evidence.',
     };
-    fs.writeFileSync(resultFile, JSON.stringify({ date, engine, model: process.env.AB_TEST_MODEL || 'openai/gpt-5-mini', protocol: 'paired-randomized-v1', summary, results }, null, 2));
+    fs.writeFileSync(resultFile, JSON.stringify({
+      date,
+      engine,
+      model: process.env.AB_TEST_MODEL || 'openai/gpt-5-mini',
+      protocol: 'paired-randomized-v2-rubric-normalized',
+      harness_code_sha256: sha256File(__filename),
+      cli_version: cliVersion(engine),
+      cli_version_sha256: cliVersion(engine) ? sha256(cliVersion(engine)) : null,
+      model_sha256: sha256(process.env.AB_TEST_MODEL || 'openai/gpt-5-mini'),
+      summary,
+      results,
+    }, null, 2));
     console.log(`\nResults written to: ${resultFile}`);
 
     // Exit non-zero if any EFFECTIVE or HARMFUL (these are signal, not noise)
@@ -407,4 +471,15 @@ function main() {
   process.exit(1);
 }
 
-main();
+if (require.main === module) main();
+
+module.exports = {
+  canonicalRubric,
+  computeVerdict,
+  discoverCases,
+  extractTrace,
+  grade,
+  normalizedRubric,
+  parseSimpleYaml,
+  validateCase,
+};

package/ci/action-gate-platform-doc.test.js

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.resolve(__dirname, '..');
+const matrix = JSON.parse(fs.readFileSync(path.join(ROOT, 'docs', 'platform-compatibility.json'), 'utf8'));
+const gate = matrix.action_gate;
+
+assert.ok(gate, 'platform compatibility matrix must record action_gate evidence');
+assert.strictEqual(gate.mechanism_status, 'Mechanism verified');
+assert.strictEqual(gate.rule_table, 'hooks/scripts/action-gate-rules.json');
+assert.strictEqual(gate.hosts['claude-code'].ask_decision, 'Mechanism verified', 'the opt-in always-ask decision keeps its mechanism evidence');
+assert.strictEqual(gate.hosts['claude-code'].fallback, 'defer-to-host', 'Claude Code defers matched actions to the host permission flow by default (#107)');
+assert.strictEqual(gate.hosts['claude-code'].permission_mode_matrix, 'Mechanism verified', 'deferred permission-mode behavior stays mechanism evidence until live-host evidence is preserved');
+assert.strictEqual(gate.hosts.codex.ask_decision, 'Unsupported', 'Codex currently parses but does not support PreToolUse permissionDecision=ask');
+assert.strictEqual(gate.hosts.codex.fallback, 'exit-2-block');
+assert.match(gate.hosts.codex.notes, /continues? the tool call|continues? execution|continue/i, 'Codex notes must record that unsupported ask cannot be used as a safety gate');
+assert.strictEqual(gate.hosts.other.fallback, 'exit-2-block');
+
+const codex = matrix.platforms.find(platform => platform.id === 'codex');
+assert.ok(codex.official_sources.includes('https://learn.chatgpt.com/docs/hooks'));
+assert.match(codex.capabilities.hooksLifecycle.notes, /PostToolUseFailure.*not declared|does not declare.*PostToolUseFailure/i, 'Codex lifecycle notes must reject unsupported PostToolUseFailure');
+
+const claude = matrix.platforms.find(platform => platform.id === 'claude-code');
+assert.ok(claude.official_sources.includes('https://code.claude.com/docs/en/hooks'));
+assert.ok(claude.official_sources.includes('https://code.claude.com/docs/en/permissions'));
+
+console.log('Action-gate platform evidence boundary verified.');