harness-everything 0.5.3 → 0.6.0

package/behavioral-evals/paired-benchmark.js

@@ -0,0 +1,781 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const crypto = require('crypto');
+const { execFileSync, spawnSync } = require('child_process');
+const {
+  buildWorkspace,
+  grade,
+  parseSimpleYaml,
+} = require('./run');
+const { verifyEvidence: verifyOpenCodeHardlockEvidence } = require('./opencode-hardlock-live');
+
+const ROOT = path.resolve(__dirname, '..');
+const CASES_DIR = path.join(__dirname, 'cases');
+const RESULTS_ROOT = path.join(__dirname, 'results', 'paired');
+const PLUGIN_SOURCE = path.join(ROOT, 'opencode-plugin', 'index.mjs');
+const EFFECT_TYPES = new Set(['skill-text', 'plugin-enforcement']);
+const DEFINITIVE = new Set(['pass', 'fail']);
+const SNAPSHOT_EXCLUDES = new Set(['.git', '.claude', '.opencode', '.harness-src', 'node_modules']);
+
+function sha256(value) {
+  return crypto.createHash('sha256').update(value).digest('hex');
+}
+
+function sha256File(file) {
+  return sha256(fs.readFileSync(file));
+}
+
+function stable(value) {
+  if (Array.isArray(value)) return value.map(stable);
+  if (!value || typeof value !== 'object') return value;
+  return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])]));
+}
+
+function stableJson(value) {
+  return JSON.stringify(stable(value));
+}
+
+function discoverCases() {
+  return fs.readdirSync(CASES_DIR)
+    .filter((file) => file.endsWith('.yaml'))
+    .map((file) => ({
+      file: path.join(CASES_DIR, file),
+      ...parseSimpleYaml(fs.readFileSync(path.join(CASES_DIR, file), 'utf8')),
+    }));
+}
+
+function treatmentSkills(c) {
+  if (Array.isArray(c.loaded_skills)) return c.loaded_skills.slice();
+  if (c.discipline && fs.existsSync(path.join(ROOT, c.discipline, 'SKILL.md'))) return [c.discipline];
+  return [];
+}
+
+function safeCommand(command, args, options = {}) {
+  try {
+    return execFileSync(command, args, {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      ...options,
+    }).trim();
+  } catch {
+    return null;
+  }
+}
+
+function copySkill(ws, skill) {
+  const source = path.join(ROOT, skill);
+  if (!fs.existsSync(path.join(source, 'SKILL.md'))) throw new Error(`unknown skill: ${skill}`);
+  const destination = path.join(ws, '.claude', 'skills', skill);
+  fs.mkdirSync(path.dirname(destination), { recursive: true });
+  fs.cpSync(source, destination, { recursive: true });
+}
+
+function installSkillsOnly(ws, skills, engine) {
+  for (const skill of skills) copySkill(ws, skill);
+  if (engine === 'opencode') {
+    fs.writeFileSync(path.join(ws, 'opencode.json'), JSON.stringify({
+      $schema: 'https://opencode.ai/config.json',
+      instructions: skills.map((skill) => `.claude/skills/${skill}/SKILL.md`),
+    }, null, 2));
+  }
+}
+
+function installOpenCodePlugin(ws) {
+  const pluginDir = path.join(ws, '.opencode', 'plugins');
+  fs.mkdirSync(pluginDir, { recursive: true });
+  const installed = path.join(pluginDir, 'harness-enforcement.js');
+  fs.copyFileSync(PLUGIN_SOURCE, installed);
+  const sourceHash = sha256File(PLUGIN_SOURCE);
+  const installedHash = sha256File(installed);
+  if (sourceHash !== installedHash) throw new Error('installed OpenCode plugin does not match canonical source');
+  return { installed, sha256: sourceHash };
+}
+
+function prepareArmWorkspace(c, effectType, arm, engine) {
+  const ws = buildWorkspace(c);
+  const namedSkills = treatmentSkills(c);
+  const treatment = arm === 'treatment';
+  let skills = [];
+  let plugin = null;
+
+  if (effectType === 'skill-text') {
+    skills = treatment ? namedSkills : [];
+    installSkillsOnly(ws, skills, engine);
+    if (engine === 'opencode' && !fs.existsSync(path.join(ws, 'opencode.json'))) {
+      installSkillsOnly(ws, [], engine);
+    }
+  } else if (effectType === 'plugin-enforcement') {
+    skills = namedSkills;
+    installSkillsOnly(ws, skills, engine);
+    fs.mkdirSync(path.join(ws, '.opencode', 'plugins'), { recursive: true });
+    if (treatment) plugin = installOpenCodePlugin(ws);
+  } else {
+    throw new Error(`unsupported effect type: ${effectType}`);
+  }
+
+  return {
+    ws,
+    namedSkills,
+    loadedSkills: skills,
+    plugin,
+    intervention: {
+      skill_text: effectType === 'skill-text' ? treatment : true,
+      plugin_enforcement: effectType === 'plugin-enforcement' ? treatment : false,
+    },
+  };
+}
+
+function expandWindowsBatchPath(value, batchPath) {
+  const base = path.dirname(batchPath) + path.sep;
+  return value
+    .replace(/%~dp0/gi, base)
+    .replace(/%dp0%/gi, base)
+    .replace(/^"|"$/g, '')
+    .trim();
+}
+
+function windowsCommandCandidates(command) {
+  try {
+    return execFileSync('where.exe', [command], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] })
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter((line) => line && !/^INFO:/i.test(line));
+  } catch {
+    return [];
+  }
+}
+
+function resolveCliInvocation(invocation) {
+  if (process.platform !== 'win32') return invocation;
+  const candidates = windowsCommandCandidates(invocation.command);
+  const native = candidates.find((candidate) => /\.exe$/i.test(candidate));
+  if (native) return { command: native, args: invocation.args.slice() };
+  const shim = candidates.find((candidate) => /\.(?:cmd|bat)$/i.test(candidate));
+  if (shim) {
+    let source = '';
+    try { source = fs.readFileSync(shim, 'utf8'); } catch { source = ''; }
+    const nodeEntry = source.match(/(?:^|\s)(?:node(?:\.exe)?)(?:\s+)(?:"([^"]+\.js)"|([^\s]+\.js))/im);
+    if (nodeEntry) {
+      const entry = expandWindowsBatchPath(nodeEntry[1] || nodeEntry[2], shim);
+      return { command: process.execPath, args: [entry, ...invocation.args] };
+    }
+    const binaryEntry = source.match(/"?([^"\r\n]+\.exe)"?\s+%\*/i);
+    if (binaryEntry) return { command: expandWindowsBatchPath(binaryEntry[1], shim), args: invocation.args.slice() };
+  }
+  const direct = candidates.find((candidate) => !/\.(?:cmd|bat|ps1)$/i.test(candidate));
+  return direct ? { command: direct, args: invocation.args.slice() } : invocation;
+}
+
+function buildInvocation(engine, model, prompt, ws, maxTurns) {
+  if (engine === 'opencode') {
+    return resolveCliInvocation({
+      command: 'opencode',
+      args: ['run', '--format', 'json', '--auto', '--dir', ws, '-m', model, prompt],
+    });
+  }
+  if (engine === 'claude') {
+    return resolveCliInvocation({
+      command: 'claude',
+      args: [
+        '-p', prompt,
+        '--output-format', 'stream-json',
+        '--verbose',
+        '--max-turns', String(maxTurns),
+        '--dangerously-skip-permissions',
+        '--setting-sources', 'project,local',
+        '--model', model,
+      ],
+    });
+  }
+  throw new Error(`unsupported engine: ${engine}`);
+}
+
+function copyWorkspaceSnapshot(ws, destination) {
+  fs.mkdirSync(destination, { recursive: true });
+  fs.cpSync(ws, destination, {
+    recursive: true,
+    filter: (source) => {
+      if (source === ws) return true;
+      const relative = path.relative(ws, source);
+      const first = relative.split(path.sep)[0];
+      return !SNAPSHOT_EXCLUDES.has(first);
+    },
+  });
+}
+
+function recursiveFind(root, names) {
+  if (!fs.existsSync(root)) return [];
+  const out = [];
+  for (const entry of fs.readdirSync(root, { withFileTypes: true })) {
+    const file = path.join(root, entry.name);
+    if (entry.isDirectory()) out.push(...recursiveFind(file, names));
+    else if (entry.isFile() && names.has(entry.name)) out.push(file);
+  }
+  return out;
+}
+
+function inspectPluginAttribution(stateHome) {
+  const stateFiles = recursiveFind(stateHome, new Set(['edit-state.json', 'circuit-breaker.json', 'compliance.json']));
+  return {
+    fired: stateFiles.length > 0,
+    state_files: stateFiles.map((file) => path.relative(stateHome, file).split(path.sep).join('/')).sort(),
+  };
+}
+
+function usageTotal(usage) {
+  if (!usage || typeof usage !== 'object') return null;
+  for (const key of ['total_tokens', 'totalTokens', 'tokens']) {
+    if (typeof usage[key] === 'number') return usage[key];
+  }
+  const input = ['input_tokens', 'inputTokens', 'prompt_tokens', 'promptTokens']
+    .map((key) => usage[key]).find((value) => typeof value === 'number');
+  const output = ['output_tokens', 'outputTokens', 'completion_tokens', 'completionTokens']
+    .map((key) => usage[key]).find((value) => typeof value === 'number');
+  return typeof input === 'number' && typeof output === 'number' ? input + output : null;
+}
+
+function runArm(c, context, arm, pairDir) {
+  const prepared = prepareArmWorkspace(c, context.effect_type, arm, context.engine);
+  const stateHome = fs.mkdtempSync(path.join(os.tmpdir(), `harness-paired-state-${arm}-`));
+  const transcriptFile = path.join(pairDir, `${arm}.transcript.jsonl`);
+  const stderrFile = path.join(pairDir, `${arm}.stderr.txt`);
+  const snapshotDir = path.join(pairDir, `${arm}.workspace`);
+  const stateArchiveDir = path.join(pairDir, `${arm}.state`);
+  const invocation = buildInvocation(context.engine, context.model, c.prompt, prepared.ws, c.max_turns);
+  const started = Date.now();
+  const result = spawnSync(invocation.command, invocation.args, {
+    cwd: prepared.ws,
+    env: { ...process.env, HARNESS_STATE_HOME: stateHome },
+    encoding: 'utf8',
+    timeout: context.engine === 'opencode' ? 20 * 60 * 1000 : 15 * 60 * 1000,
+    maxBuffer: 64 * 1024 * 1024,
+    windowsHide: true,
+  });
+  const durationMs = Date.now() - started;
+  fs.writeFileSync(transcriptFile, result.stdout || '', 'utf8');
+  fs.writeFileSync(stderrFile, result.stderr || '', 'utf8');
+  copyWorkspaceSnapshot(prepared.ws, snapshotDir);
+  if (fs.existsSync(stateHome)) fs.cpSync(stateHome, stateArchiveDir, { recursive: true });
+
+  const base = {
+    arm,
+    intervention: prepared.intervention,
+    loaded_skills: prepared.loadedSkills,
+    plugin_sha256: prepared.plugin ? prepared.plugin.sha256 : null,
+    transcript: path.basename(transcriptFile),
+    stderr: path.basename(stderrFile),
+    workspace_snapshot: path.basename(snapshotDir),
+    state_snapshot: path.basename(stateArchiveDir),
+    cli_exit_status: result.status,
+    cli_signal: result.signal || null,
+    duration_ms_wall: durationMs,
+  };
+
+  if (result.error || result.status !== 0) {
+    fs.rmSync(prepared.ws, { recursive: true, force: true });
+    fs.rmSync(stateHome, { recursive: true, force: true });
+    return {
+      ...base,
+      outcome: 'session-error',
+      infrastructure_reason: result.error
+        ? String(result.error.message || result.error).slice(0, 500)
+        : `CLI exited with status ${result.status}`,
+      model_name: null,
+      cost: null,
+      usage: null,
+      total_tokens: null,
+      tool_call_count: null,
+      expectations: [],
+      plugin_attribution: context.effect_type === 'plugin-enforcement' && arm === 'treatment'
+        ? inspectPluginAttribution(stateArchiveDir)
+        : null,
+    };
+  }
+
+  let graded;
+  try {
+    graded = grade(c, prepared.ws, transcriptFile, context.engine);
+  } catch (error) {
+    fs.rmSync(prepared.ws, { recursive: true, force: true });
+    fs.rmSync(stateHome, { recursive: true, force: true });
+    return {
+      ...base,
+      outcome: 'grader-error',
+      infrastructure_reason: String(error.message || error).slice(0, 500),
+      model_name: null,
+      cost: null,
+      usage: null,
+      total_tokens: null,
+      tool_call_count: null,
+      expectations: [],
+      plugin_attribution: null,
+    };
+  }
+
+  const metadata = graded.parsed.metadata || {};
+  const pluginAttribution = context.effect_type === 'plugin-enforcement' && arm === 'treatment'
+    ? inspectPluginAttribution(stateArchiveDir)
+    : null;
+  let outcome = graded.status;
+  let infrastructureReason = null;
+  if (pluginAttribution && !pluginAttribution.fired) {
+    outcome = 'inconclusive';
+    infrastructureReason = 'plugin treatment produced no retained OpenCode enforcement state; attribution missing';
+  }
+
+  const toolCount = graded.executionEvidence && graded.executionEvidence.counts
+    ? graded.executionEvidence.counts.attempted
+    : null;
+  const armRecord = {
+    ...base,
+    outcome,
+    infrastructure_reason: infrastructureReason,
+    parse_status: graded.parsed.parseStatus,
+    parse_error: graded.parsed.parseError,
+    transcript_format: graded.parsed.format,
+    model_name: metadata.model || null,
+    cost: typeof metadata.cost === 'number' ? metadata.cost : null,
+    usage: metadata.usage || null,
+    total_tokens: usageTotal(metadata.usage),
+    duration_ms: typeof metadata.duration_ms === 'number' ? metadata.duration_ms : durationMs,
+    num_turns: typeof metadata.num_turns === 'number' ? metadata.num_turns : null,
+    tool_call_count: toolCount,
+    expectations: graded.results.map(({ type, value, command, after_edit, informational, description, pass, status, reason }) => ({
+      type, value, command, after_edit, informational: !!informational, description, pass, status, reason,
+      evidence_class: type === 'command_exit_0' ? 'post-hoc-workspace-check' :
+        String(type || '').startsWith('tool_') || type === 'execution_evidence' ? 'observed-tool-event' : 'behavior-or-workspace',
+    })),
+    plugin_attribution: pluginAttribution,
+  };
+
+  fs.rmSync(prepared.ws, { recursive: true, force: true });
+  fs.rmSync(stateHome, { recursive: true, force: true });
+  return armRecord;
+}
+
+function validatePreflight(preflightDir) {
+  if (!preflightDir) return { passed: false, reason: 'plugin-enforcement requires --opencode-preflight <evidence-dir>' };
+  const absolute = path.resolve(preflightDir);
+  let verification;
+  try { verification = verifyOpenCodeHardlockEvidence(absolute); }
+  catch (error) { return { passed: false, reason: `preflight unreadable: ${error.message}` }; }
+  if (!verification.passed) return { passed: false, reason: 'OpenCode hard-lock preflight evidence does not pass verification' };
+  let metadata;
+  try { metadata = JSON.parse(fs.readFileSync(path.join(absolute, 'metadata.json'), 'utf8')); }
+  catch (error) { return { passed: false, reason: `preflight metadata unreadable: ${error.message}` }; }
+  const currentHash = sha256File(PLUGIN_SOURCE);
+  if (!metadata.pluginSha256 || metadata.pluginSha256 !== currentHash) {
+    return { passed: false, reason: 'preflight plugin hash does not match the current canonical OpenCode plugin' };
+  }
+  return {
+    passed: true,
+    evidence_dir: absolute,
+    plugin_sha256: currentHash,
+    repo_commit: metadata.repoCommit || null,
+    opencode_version: metadata.opencodeVersion || null,
+  };
+}
+
+function validateRunConfig(options) {
+  const problems = [];
+  if (!EFFECT_TYPES.has(options.effect_type)) problems.push('effect must be skill-text or plugin-enforcement');
+  if (!['claude', 'opencode'].includes(options.engine)) problems.push('engine must be claude or opencode');
+  if (!options.model || !String(options.model).trim()) problems.push('an explicit --model is required for paired comparability');
+  if (!Number.isFinite(options.min_effect_pp) || options.min_effect_pp < 0 || options.min_effect_pp > 100) {
+    problems.push('--min-effect-pp must be explicitly set between 0 and 100 before the run');
+  }
+  if (!Number.isInteger(options.repeats) || options.repeats < 1) problems.push('--repeats must be an integer >= 1');
+  if (options.effect_type === 'plugin-enforcement' && options.engine !== 'opencode') {
+    problems.push('plugin-enforcement currently supports engine=opencode only');
+  }
+  return problems;
+}
+
+function pairContract(c, context) {
+  return {
+    schema_version: 1,
+    effect_type: context.effect_type,
+    engine: context.engine,
+    requested_model: context.model,
+    case_id: c.id,
+    max_turns: c.max_turns,
+    loaded_skills: treatmentSkills(c).slice().sort(),
+    fixture_sha256: sha256(stableJson(c.fixture)),
+    prompt_sha256: sha256(c.prompt),
+    rubric_sha256: sha256(stableJson(c.expectations)),
+    execution_policy: context.engine === 'claude'
+      ? 'dangerously-skip-permissions/project-local-sources'
+      : 'opencode-auto',
+  };
+}
+
+function validatePairRecord(record) {
+  const baseline = record && record.arms && record.arms.baseline;
+  const treatment = record && record.arms && record.arms.treatment;
+  if (!baseline || !treatment) return { included: false, reason: 'missing-arm' };
+  if (!DEFINITIVE.has(baseline.outcome) || !DEFINITIVE.has(treatment.outcome)) {
+    const outcomes = [baseline.outcome, treatment.outcome].join(' / ');
+    return { included: false, reason: `non-definitive:${outcomes}` };
+  }
+  if (!record.contract || sha256(stableJson(record.contract)) !== record.contract_sha256) {
+    return { included: false, reason: 'pair-contract-hash-mismatch' };
+  }
+  if (baseline.model_name && treatment.model_name && baseline.model_name !== treatment.model_name) {
+    return { included: false, reason: 'actual-model-mismatch' };
+  }
+  if (record.effect_type === 'skill-text') {
+    if (baseline.intervention.skill_text || treatment.intervention.skill_text !== true ||
+        baseline.intervention.plugin_enforcement || treatment.intervention.plugin_enforcement) {
+      return { included: false, reason: 'skill-text-intervention-shape-mismatch' };
+    }
+    if (baseline.loaded_skills.length !== 0 || stableJson(treatment.loaded_skills.slice().sort()) !== stableJson(record.contract.loaded_skills)) {
+      return { included: false, reason: 'skill-text-loaded-skills-mismatch' };
+    }
+  } else if (record.effect_type === 'plugin-enforcement') {
+    if (!baseline.intervention.skill_text || !treatment.intervention.skill_text ||
+        baseline.intervention.plugin_enforcement || treatment.intervention.plugin_enforcement !== true) {
+      return { included: false, reason: 'plugin-intervention-shape-mismatch' };
+    }
+    if (stableJson(baseline.loaded_skills.slice().sort()) !== stableJson(treatment.loaded_skills.slice().sort()) ||
+        stableJson(treatment.loaded_skills.slice().sort()) !== stableJson(record.contract.loaded_skills)) {
+      return { included: false, reason: 'plugin-skill-text-not-held-constant' };
+    }
+    if (!record.preflight || record.preflight.passed !== true || !treatment.plugin_attribution || treatment.plugin_attribution.fired !== true) {
+      return { included: false, reason: 'plugin-attribution-missing' };
+    }
+    if (!treatment.plugin_sha256 || treatment.plugin_sha256 !== record.preflight.plugin_sha256) {
+      return { included: false, reason: 'plugin-hash-mismatch' };
+    }
+  } else {
+    return { included: false, reason: 'unknown-effect-type' };
+  }
+  return { included: true, reason: null };
+}
+
+function transitionOf(record) {
+  const baseline = record.arms.baseline.outcome;
+  const treatment = record.arms.treatment.outcome;
+  return `${baseline}->${treatment}`;
+}
+
+function sampleVariance(values) {
+  if (values.length < 2) return null;
+  const mean = values.reduce((sum, value) => sum + value, 0) / values.length;
+  return values.reduce((sum, value) => sum + ((value - mean) ** 2), 0) / (values.length - 1);
+}
+
+function seedFrom(text) {
+  return parseInt(sha256(text).slice(0, 8), 16) || 1;
+}
+
+function xorshift32(seed) {
+  let state = seed >>> 0;
+  return () => {
+    state ^= state << 13;
+    state ^= state >>> 17;
+    state ^= state << 5;
+    return (state >>> 0) / 0x100000000;
+  };
+}
+
+function percentile(sorted, q) {
+  if (!sorted.length) return null;
+  const position = (sorted.length - 1) * q;
+  const lower = Math.floor(position);
+  const upper = Math.ceil(position);
+  if (lower === upper) return sorted[lower];
+  const weight = position - lower;
+  return sorted[lower] * (1 - weight) + sorted[upper] * weight;
+}
+
+function bootstrapPairedEffect(values, seedText, samples = 5000) {
+  if (!values.length) return null;
+  const random = xorshift32(seedFrom(seedText));
+  const means = [];
+  for (let sample = 0; sample < samples; sample++) {
+    let sum = 0;
+    for (let i = 0; i < values.length; i++) sum += values[Math.floor(random() * values.length)];
+    means.push((sum / values.length) * 100);
+  }
+  means.sort((a, b) => a - b);
+  return [percentile(means, 0.025), percentile(means, 0.975)].map((value) => Number(value.toFixed(2)));
+}
+
+function binomialCoefficient(n, k) {
+  if (k < 0 || k > n) return 0;
+  k = Math.min(k, n - k);
+  let result = 1;
+  for (let i = 1; i <= k; i++) result = result * (n - k + i) / i;
+  return result;
+}
+
+function mcnemarExactP(failToPass, passToFail) {
+  const n = failToPass + passToFail;
+  if (!n) return 1;
+  const k = Math.min(failToPass, passToFail);
+  let tail = 0;
+  for (let i = 0; i <= k; i++) tail += binomialCoefficient(n, i) * (0.5 ** n);
+  return Number(Math.min(1, 2 * tail).toFixed(6));
+}
+
+function meanDelta(records, field) {
+  const values = [];
+  for (const record of records) {
+    const baseline = record.arms.baseline[field];
+    const treatment = record.arms.treatment[field];
+    if (typeof baseline === 'number' && typeof treatment === 'number') values.push(treatment - baseline);
+  }
+  if (!values.length) return { n: 0, mean: null };
+  return { n: values.length, mean: Number((values.reduce((a, b) => a + b, 0) / values.length).toFixed(4)) };
+}
+
+function summarizePairs(records, minEffectPp) {
+  const included = [];
+  const excluded = [];
+  for (const record of records) {
+    const validation = validatePairRecord(record);
+    if (validation.included) included.push(record);
+    else excluded.push({ pair_id: record.pair_id || null, case_id: record.id || null, reason: validation.reason });
+  }
+
+  const transitions = {
+    'fail->pass': 0,
+    'pass->fail': 0,
+    'pass->pass': 0,
+    'fail->fail': 0,
+  };
+  const differences = [];
+  for (const record of included) {
+    const transition = transitionOf(record);
+    transitions[transition]++;
+    const baseline = record.arms.baseline.outcome === 'pass' ? 1 : 0;
+    const treatment = record.arms.treatment.outcome === 'pass' ? 1 : 0;
+    differences.push(treatment - baseline);
+  }
+
+  const baselinePass = included.filter((record) => record.arms.baseline.outcome === 'pass').length;
+  const treatmentPass = included.filter((record) => record.arms.treatment.outcome === 'pass').length;
+  const effectPp = included.length ? ((treatmentPass - baselinePass) / included.length) * 100 : null;
+  const ci = bootstrapPairedEffect(differences, included.map((record) => record.pair_id).sort().join('|'));
+
+  const byCase = {};
+  for (const record of included) {
+    if (!byCase[record.id]) byCase[record.id] = [];
+    const baseline = record.arms.baseline.outcome === 'pass' ? 1 : 0;
+    const treatment = record.arms.treatment.outcome === 'pass' ? 1 : 0;
+    byCase[record.id].push(treatment - baseline);
+  }
+  const repeatVariance = Object.fromEntries(Object.entries(byCase).map(([id, values]) => [id, {
+    repeats: values.length,
+    paired_difference_mean: Number((values.reduce((a, b) => a + b, 0) / values.length).toFixed(4)),
+    paired_difference_variance: sampleVariance(values) === null ? null : Number(sampleVariance(values).toFixed(4)),
+  }]));
+
+  let decision = 'insufficient-data';
+  if (included.length >= 2 && ci) {
+    if (ci[0] >= minEffectPp) decision = 'meets-predeclared-threshold';
+    else if (ci[1] < minEffectPp) decision = 'below-predeclared-threshold';
+    else decision = 'uncertain-at-predeclared-threshold';
+  }
+
+  return {
+    schema_version: 1,
+    requested_pairs: records.length,
+    included_pairs: included.length,
+    excluded_pairs: excluded.length,
+    exclusions: excluded,
+    arm_outcomes: {
+      baseline: { pass: baselinePass, fail: included.length - baselinePass },
+      treatment: { pass: treatmentPass, fail: included.length - treatmentPass },
+    },
+    transitions,
+    paired_effect_pp: effectPp === null ? null : Number(effectPp.toFixed(2)),
+    paired_effect_ci95_pp: ci,
+    mcnemar_exact_p: mcnemarExactP(transitions['fail->pass'], transitions['pass->fail']),
+    decision: {
+      min_effect_pp: minEffectPp,
+      status: decision,
+      rule: 'claim threshold met only when the paired bootstrap 95% CI lower bound is at or above the predeclared minimum effect',
+    },
+    repeat_variance: repeatVariance,
+    cost_and_execution_overhead: {
+      cost_delta: meanDelta(included, 'cost'),
+      token_delta: meanDelta(included, 'total_tokens'),
+      tool_call_delta: meanDelta(included, 'tool_call_count'),
+      duration_ms_delta: meanDelta(included, 'duration_ms'),
+    },
+  };
+}
+
+function flag(args, name) {
+  const index = args.indexOf(name);
+  return index >= 0 ? args[index + 1] : undefined;
+}
+
+function integerFlag(args, name, fallback) {
+  const raw = flag(args, name);
+  return raw === undefined ? fallback : Number(raw);
+}
+
+function loadPairFiles(experimentDir) {
+  const pairsDir = path.join(experimentDir, 'pairs');
+  if (!fs.existsSync(pairsDir)) return [];
+  return fs.readdirSync(pairsDir)
+    .filter((name) => name.endsWith('.json'))
+    .map((name) => JSON.parse(fs.readFileSync(path.join(pairsDir, name), 'utf8')));
+}
+
+function runExperiment(options) {
+  const problems = validateRunConfig(options);
+  if (problems.length) throw new Error(problems.join('; '));
+  const cases = discoverCases().filter((c) => !options.case_id || c.id === options.case_id);
+  if (!cases.length) throw new Error(`no matching case: ${options.case_id || '(none)'}`);
+
+  let preflight = null;
+  if (options.effect_type === 'plugin-enforcement') {
+    preflight = validatePreflight(options.preflight);
+    if (!preflight.passed) throw new Error(preflight.reason);
+  }
+
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const experimentId = `${options.effect_type}-${timestamp}-${crypto.randomBytes(4).toString('hex')}`;
+  const experimentDir = path.join(RESULTS_ROOT, experimentId);
+  const pairsDir = path.join(experimentDir, 'pairs');
+  fs.mkdirSync(pairsDir, { recursive: true });
+
+  const context = {
+    schema_version: 1,
+    experiment_id: experimentId,
+    effect_type: options.effect_type,
+    engine: options.engine,
+    model: options.model,
+    min_effect_pp: options.min_effect_pp,
+    repeats: options.repeats,
+    preflight,
+    repo_commit: safeCommand('git', ['rev-parse', 'HEAD'], { cwd: ROOT }),
+    cli_version: safeCommand(options.engine === 'claude' ? 'claude' : 'opencode', ['--version']),
+    node_version: process.version,
+    platform: process.platform,
+    arch: process.arch,
+    created_at: new Date().toISOString(),
+  };
+  fs.writeFileSync(path.join(experimentDir, 'manifest.json'), JSON.stringify(context, null, 2));
+
+  const records = [];
+  for (let repeat = 1; repeat <= options.repeats; repeat++) {
+    for (const c of cases) {
+      const pairId = `${c.id}-r${repeat}-${crypto.randomUUID()}`;
+      const pairDir = path.join(experimentDir, pairId);
+      fs.mkdirSync(pairDir, { recursive: true });
+      const contract = pairContract(c, context);
+      const contractHash = sha256(stableJson(contract));
+      const order = crypto.randomInt(0, 2) === 0 ? ['baseline', 'treatment'] : ['treatment', 'baseline'];
+      const arms = {};
+      for (const arm of order) arms[arm] = runArm(c, context, arm, pairDir);
+      const record = {
+        schema_version: 1,
+        id: c.id,
+        pair_id: pairId,
+        repeat,
+        effect_type: context.effect_type,
+        engine: context.engine,
+        requested_model: context.model,
+        pressure: !!c.pressure,
+        pressure_category: c.pressure_category || null,
+        arm_order: order,
+        contract,
+        contract_sha256: contractHash,
+        preflight,
+        arms,
+        created_at: new Date().toISOString(),
+      };
+      records.push(record);
+      fs.writeFileSync(path.join(pairsDir, `${pairId}.json`), JSON.stringify(record, null, 2));
+      console.log(`${pairId}: baseline=${arms.baseline.outcome}, treatment=${arms.treatment.outcome}`);
+    }
+  }
+
+  const summary = {
+    experiment_id: experimentId,
+    effect_type: context.effect_type,
+    engine: context.engine,
+    model: context.model,
+    created_at: new Date().toISOString(),
+    ...summarizePairs(records, context.min_effect_pp),
+  };
+  fs.writeFileSync(path.join(experimentDir, 'summary.json'), JSON.stringify(summary, null, 2));
+  console.log(`Summary: ${path.join(experimentDir, 'summary.json')}`);
+  return { experimentDir, summary };
+}
+
+function summarizeExperiment(experimentDir) {
+  const absolute = path.resolve(experimentDir);
+  const manifest = JSON.parse(fs.readFileSync(path.join(absolute, 'manifest.json'), 'utf8'));
+  if (!Number.isFinite(manifest.min_effect_pp)) throw new Error('manifest is missing predeclared min_effect_pp');
+  const records = loadPairFiles(absolute);
+  const summary = {
+    experiment_id: manifest.experiment_id,
+    effect_type: manifest.effect_type,
+    engine: manifest.engine,
+    model: manifest.model,
+    created_at: new Date().toISOString(),
+    ...summarizePairs(records, manifest.min_effect_pp),
+  };
+  fs.writeFileSync(path.join(absolute, 'summary.json'), JSON.stringify(summary, null, 2));
+  return summary;
+}
+
+function main(argv = process.argv.slice(2)) {
+  const command = argv[0];
+  if (command === 'run') {
+    const options = {
+      effect_type: flag(argv, '--effect'),
+      engine: flag(argv, '--engine'),
+      model: flag(argv, '--model'),
+      case_id: flag(argv, '--case'),
+      repeats: integerFlag(argv, '--repeats', 1),
+      min_effect_pp: Number(flag(argv, '--min-effect-pp')),
+      preflight: flag(argv, '--opencode-preflight'),
+    };
+    runExperiment(options);
+    return;
+  }
+  if (command === 'summarize') {
+    const experimentDir = argv[1];
+    if (!experimentDir) throw new Error('summarize requires an experiment directory');
+    const summary = summarizeExperiment(experimentDir);
+    console.log(JSON.stringify(summary, null, 2));
+    return;
+  }
+  console.log([
+    'Usage:',
+    '  node behavioral-evals/paired-benchmark.js run --effect skill-text --engine claude|opencode --model <model> --min-effect-pp <n> [--case <id>] [--repeats <n>]',
+    '  node behavioral-evals/paired-benchmark.js run --effect plugin-enforcement --engine opencode --model <model> --min-effect-pp <n> --opencode-preflight <evidence-dir> [--case <id>] [--repeats <n>]',
+    '  node behavioral-evals/paired-benchmark.js summarize <experiment-dir>',
+  ].join('\n'));
+}
+
+if (require.main === module) {
+  try { main(); }
+  catch (error) {
+    console.error(error && error.stack ? error.stack : error);
+    process.exitCode = 1;
+  }
+}
+
+module.exports = {
+  EFFECT_TYPES,
+  bootstrapPairedEffect,
+  inspectPluginAttribution,
+  mcnemarExactP,
+  pairContract,
+  prepareArmWorkspace,
+  summarizePairs,
+  validatePairRecord,
+  validatePreflight,
+  validateRunConfig,
+};