harness-everything 0.3.6 → 0.4.0

package/behavioral-evals/run.js

@@ -13,7 +13,7 @@
  *  1. Build identical case fixtures in OS temp workspaces.
  *  2. Run a control with no Harness files and a treatment with only the
  *     case's named skill loaded; `--arm both` randomizes this order.
- *  3. Run: claude -p "<prompt>" --output-format json --max-turns N
+ *  3. Run: claude -p "<prompt>" --output-format stream-json --verbose --max-turns N
  *  4. Grade expectations[] against transcript text and workspace state.
  *  5. Write paired results with fixture/prompt hashes and attribution so any
  *     grader verdict can be audited by a human.
@@ -23,6 +23,11 @@ const os = require('os');
 const path = require('path');
 const crypto = require('crypto');
 const { execFileSync } = require('child_process');
+const {
+  gradeExecutionEvidence,
+  parseTranscriptFile,
+} = require('./transcript-parser');
+const { commandMatches, normalizeRepoPath } = require('../scripts/lib/execution-contract');
 
 const ROOT = path.resolve(__dirname, '..');
 const CASES_DIR = path.join(__dirname, 'cases');
@@ -32,7 +37,7 @@ const RESULTS_DIR = path.join(__dirname, 'results');
 // Minimal YAML subset parser for our case files (flat maps, lists of maps,
 // block scalars via |). Avoids adding dependencies to this package.
 function parseSimpleYaml(text) {
-  const lines = text.split('\n');
+  const lines = text.replace(/\r\n?/g, '\n').split('\n');
   let i = 0;
   function parseBlock(indent) {
     const obj = {};
@@ -141,7 +146,10 @@ const PRESSURE_RULES = {
 // --- validate mode ----------------------------------------------------------
 function validate(cases) {
   let failures = 0;
-  const EXPECT_TYPES = new Set(['trace_contains', 'trace_not_contains', 'file_contains', 'file_not_exists', 'command_exit_0']);
+  const EXPECT_TYPES = new Set([
+    'trace_contains', 'trace_not_contains', 'file_contains', 'file_not_exists', 'command_exit_0',
+    'tool_call', 'tool_attempted', 'tool_completed', 'tool_executed', 'tool_denied', 'execution_evidence',
+  ]);
   for (const c of cases) {
     const problems = [];
     if (!c.id) problems.push('missing id');
@@ -155,7 +163,9 @@ function validate(cases) {
     if (!resolveTreatmentSkills(c).length) problems.push('treatment must load at least one named Harness skill');
     for (const e of c.expectations || []) {
       if (!EXPECT_TYPES.has(e.type)) problems.push(`unknown expectation type "${e.type}"`);
-      if (e.value === undefined && e.command === undefined) problems.push(`expectation ${e.type} needs value or command`);
+      if (e.value === undefined && e.command === undefined && e.tool === undefined && e.name === undefined && e.count === undefined) {
+        problems.push(`expectation ${e.type} needs value, tool, name, count, or command`);
+      }
     }
     if (c.pressure && !/skip|don't|not|quick|minutes/i.test(c.prompt)) {
       problems.push('pressure case prompt does not read as pressure');
@@ -180,8 +190,16 @@ function validate(cases) {
 
 // --- live run mode -----------------------------------------------------------
 function buildWorkspace(c) {
+  if (!c || !c.fixture || !Array.isArray(c.fixture.files)) throw new Error('fixture.files must be a non-empty array');
+  const fixtureFiles = c.fixture.files.map((file, index) => {
+    const relative = normalizeRepoPath(file && file.path);
+    if (!relative || relative === '.git' || relative.startsWith('.git/')) {
+      throw new Error(`fixture.files[${index}] path must stay inside the generated workspace`);
+    }
+    return { ...file, path: relative };
+  });
   const ws = fs.mkdtempSync(path.join(os.tmpdir(), `harness-behavioral-${c.id}-`));
-  for (const f of c.fixture.files) {
+  for (const f of fixtureFiles) {
     const target = path.join(ws, f.path);
     fs.mkdirSync(path.dirname(target), { recursive: true });
     fs.writeFileSync(target, typeof f.content === 'string' ? f.content.replace(/\n$/, '') + '\n' : String(f.content));
@@ -260,21 +278,74 @@ function buildEngineInvocation(engine, prompt, ws, maxTurns, arm = 'treatment')
     args.push(prompt);
     return { command: 'opencode', args };
   }
-  const args = ['-p', prompt, '--output-format', 'json', '--max-turns', String(maxTurns), '--permission-mode', 'acceptEdits', '--setting-sources', 'project,local'];
+  // acceptEdits only auto-approves file edits, not Bash/PowerShell tool calls -
+  // a headless -p session has no human to answer those prompts, so any case
+  // needing a shell command (git, npm, node scripts) stalled with every tool
+  // call denied and the model just describing what it would do. This harness
+  // needs the same full autonomy opencode's --auto gives it, in the same kind
+  // of disposable os.tmpdir() fixture workspace.
+  const args = ['-p', prompt, '--output-format', 'stream-json', '--verbose', '--max-turns', String(maxTurns), '--dangerously-skip-permissions', '--setting-sources', 'project,local'];
+  const model = process.env.BEHAVIORAL_MODEL;
+  if (model) args.push('--model', model);
   if (arm === 'baseline') args.push('--safe-mode');
   return { command: 'claude', args };
 }
 
+function expandWindowsBatchPath(value, batchPath) {
+  const base = path.dirname(batchPath) + path.sep;
+  return value
+    .replace(/%~dp0/gi, base)
+    .replace(/%dp0%/gi, base)
+    .replace(/^"|"$/g, '')
+    .trim();
+}
+
+function windowsCommandCandidates(command) {
+  try {
+    return execFileSync('where.exe', [command], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] })
+      .split(/\r?\n/)
+      .map(line => line.trim())
+      .filter(line => line && !/^INFO:/i.test(line));
+  } catch {
+    return [];
+  }
+}
+
+function resolveCliInvocation(invocation) {
+  if (process.platform !== 'win32') return invocation;
+  const candidates = windowsCommandCandidates(invocation.command);
+  const native = candidates.find(candidate => /\.exe$/i.test(candidate));
+  if (native) return { command: native, args: invocation.args.slice() };
+  const shim = candidates.find(candidate => /\.(?:cmd|bat)$/i.test(candidate));
+  if (shim) {
+    let source = '';
+    try { source = fs.readFileSync(shim, 'utf8'); } catch { source = ''; }
+    const nodeEntry = source.match(/(?:^|\s)(?:node(?:\.exe)?)(?:\s+)(?:"([^"]+\.js)"|([^\s]+\.js))/im);
+    if (nodeEntry) {
+      const entry = expandWindowsBatchPath(nodeEntry[1] || nodeEntry[2], shim);
+      return { command: process.execPath, args: [entry, ...invocation.args] };
+    }
+    const binaryEntry = source.match(/"?([^"\r\n]+\.exe)"?\s+%\*/i);
+    if (binaryEntry) {
+      const entry = expandWindowsBatchPath(binaryEntry[1], shim);
+      return { command: entry, args: invocation.args.slice() };
+    }
+  }
+  const direct = candidates.find(candidate => !/\.(?:cmd|bat|ps1)$/i.test(candidate));
+  return direct ? { command: direct, args: invocation.args.slice() } : invocation;
+}
+
 function runHeadless(prompt, ws, maxTurns, engine, arm) {
   // Keep the transcript OUTSIDE the workspace: grader artifacts must never
   // show up in git-status-based scope checks.
-  const outPath = path.join(path.dirname(ws), path.basename(ws) + '.transcript.json');
+  const outPath = path.join(path.dirname(ws), path.basename(ws) + '.transcript.jsonl');
   const invocation = buildEngineInvocation(engine, prompt, ws, maxTurns, arm);
+  const executable = resolveCliInvocation(invocation);
   const outputFd = fs.openSync(outPath, 'w');
   try {
     // Pass the prompt as one argv element. A shell wrapper on Windows can
     // reinterpret spaces and punctuation, truncating prompts to `Add.`.
-    execFileSync(invocation.command, invocation.args, {
+    execFileSync(executable.command, executable.args, {
       cwd: ws,
       stdio: ['ignore', outputFd, 'inherit'],
       timeout: engine === 'opencode' ? 20 * 60 * 1000 : 15 * 60 * 1000,
@@ -290,92 +361,299 @@ function runHeadless(prompt, ws, maxTurns, engine, arm) {
 // Tool outputs embed installed skill text, so keyword graders over raw
 // transcripts produce false positives (a read of zoom-out/SKILL.md counts as
 // "invoked zoom-out"). Trace = assistant text + tool inputs only.
-function extractAgentTrace(transcriptPath) {
+const COMPLETED_TOOL_STATUSES = new Set(['completed', 'complete', 'success', 'succeeded', 'done', 'finished']);
+const DENIED_TOOL_STATUSES = new Set(['denied', 'rejected', 'blocked', 'cancelled', 'canceled', 'error', 'failed']);
+
+function firstDefined(...values) {
+  return values.find(value => value !== undefined && value !== null);
+}
+
+function eventPart(event) {
+  return event && event.part && typeof event.part === 'object' ? event.part : event;
+}
+
+function eventType(event) {
+  return String(event && event.type || '').toLowerCase();
+}
+
+function eventStatus(event, part) {
+  const state = part && part.state && typeof part.state === 'object' ? part.state : {};
+  return String(firstDefined(state.status, part && part.status, event && event.status, '')).toLowerCase();
+}
+
+function eventToolName(event, part) {
+  const state = part && part.state && typeof part.state === 'object' ? part.state : {};
+  return String(firstDefined(part && part.tool, part && part.name, state.tool, event && event.tool, event && event.name, 'tool'));
+}
+
+function eventInput(event, part) {
+  const state = part && part.state && typeof part.state === 'object' ? part.state : {};
+  return firstDefined(state.input, part && part.input, part && part.arguments, event && event.input, event && event.arguments);
+}
+
+function eventId(event, part) {
+  const state = part && part.state && typeof part.state === 'object' ? part.state : {};
+  return firstDefined(part && (part.callID || part.callId || part.id || part.tool_use_id), state.callID, state.callId, event && (event.callID || event.callId || event.id || event.tool_use_id));
+}
+
+function isToolEvent(event, part) {
+  const type = eventType(event);
+  const partType = String(part && part.type || '').toLowerCase();
+  return type.includes('tool') || partType === 'tool' || partType === 'tool_use' ||
+    eventToolName(event, part) !== 'tool' || eventInput(event, part) !== undefined;
+}
+
+function isEditOperation(tool, input) {
+  const name = String(tool || '').toLowerCase();
+  if (/(^|[-_])(edit|write|patch|create|delete|move|rename)([-_]|$)/.test(name) ||
+      /^(edit|write|apply_patch|create_file|delete_file|move_file|rename_file)$/.test(name)) return true;
+  const command = input && typeof input === 'object' ? firstDefined(input.command, input.cmd, input.script) : input;
+  if (typeof command === 'string' && /(?:>>?|\btee\b|\bset-content\b|\bout-file\b|\bwritefile\b|\brename(?:-item)?\b|\bremove(?:-item)?\b|\b(?:sed|perl)\b[^\r\n]*\s-i\b)/i.test(command)) return true;
+  const target = input && typeof input === 'object'
+    ? firstDefined(input.file_path, input.filePath, input.path, input.filename)
+    : null;
+  return typeof target === 'string' && /\.(?:patch|diff)$/i.test(target);
+}
+
+function executionState(status, type) {
+  const normalizedType = String(type || '').toLowerCase();
+  if (DENIED_TOOL_STATUSES.has(status) || normalizedType.includes('denied') || normalizedType.includes('rejected')) return 'denied';
+  if (status && !COMPLETED_TOOL_STATUSES.has(status)) return 'attempted';
+  if (COMPLETED_TOOL_STATUSES.has(status) || (!status && (normalizedType.includes('result') || normalizedType.includes('completed')))) return 'executed';
+  return 'attempted';
+}
+
+// Keep a second, event-oriented view for ordered execution and edit evidence.
+// transcript-parser.js remains the authoritative command-aware grader.
+function parseTranscriptEvents(transcriptPath) {
   const raw = fs.readFileSync(transcriptPath, 'utf8');
-  const chunks = [];
+  const events = [];
+  const pendingById = new Map();
+  let sequence = 0;
+  const addText = text => {
+    if (typeof text === 'string' && text) events.push({ kind: 'text', text, sequence: sequence++ });
+  };
+  const addTool = (event, part) => {
+    const type = eventType(event);
+    const input = eventInput(event, part) ?? {};
+    const tool = eventToolName(event, part);
+    const id = eventId(event, part);
+    const state = executionState(eventStatus(event, part), type);
+    const existing = id && pendingById.get(String(id));
+    if (existing) {
+      if (input && Object.keys(input).length) existing.input = input;
+      existing.tool = tool !== 'tool' ? tool : existing.tool;
+      if (state === 'executed' || state === 'denied') existing.state = state;
+      existing.executed = existing.state === 'executed';
+      existing.denied = existing.state === 'denied';
+      return;
+    }
+    const item = {
+      kind: 'tool', tool, input, id: id === undefined ? null : String(id), state,
+      attempted: true, executed: state === 'executed', denied: state === 'denied',
+      edit: isEditOperation(tool, input), sequence: sequence++, raw_type: type,
+    };
+    events.push(item);
+    if (id !== undefined) pendingById.set(String(id), item);
+  };
+  const visit = (value, includeText = true) => {
+    if (!value || typeof value !== 'object') return;
+    if (Array.isArray(value)) { value.forEach(item => visit(item, includeText)); return; }
+    const part = eventPart(value);
+    const type = eventType(value);
+    if (type === 'text' || String(part && part.type || '').toLowerCase() === 'text') {
+      if (includeText) addText(part && part.text !== undefined ? part.text : value.text);
+      if (value.part || !value.message) return;
+    }
+    if (value.part || isToolEvent(value, part)) {
+      if (isToolEvent(value, part)) addTool(value, part);
+      if (value.part) return;
+    }
+    if (value.message && value.message.content) visit(value.message.content, value.type !== 'user');
+    if (value.content && value.content !== value.message) visit(value.content, includeText);
+    if (!value.part && typeof value.result === 'string' && includeText) addText(value.result);
+  };
   let structured = false;
   for (const line of raw.split('\n')) {
     if (!line.trim()) continue;
-    let e;
-    try { e = JSON.parse(line); } catch { continue; }
-    const p = e && e.part;
-    if (!p) continue;
-    if (e.type === 'text' && typeof p.text === 'string') {
-      chunks.push(p.text);
-      structured = true;
-    } else if (String(e.type).includes('tool')) {
-      const input = p.state && p.state.input !== undefined ? p.state.input : p.input;
-      chunks.push(`[${p.tool || 'tool'}] ` + JSON.stringify(input ?? {}));
-      structured = true;
-    }
+    try {
+      const value = JSON.parse(line);
+      const before = events.length;
+      visit(value);
+      if (events.length > before) structured = true;
+    } catch { /* malformed/non-JSON lines are retained only by the raw fallback */ }
   }
-  // claude -p --output-format json emits one object; grade its result text.
   if (!structured) {
     try {
-      const obj = JSON.parse(raw);
-      if (obj && typeof obj.result === 'string') return obj.result;
-    } catch { /* fall through */ }
-    return raw;
+      const value = JSON.parse(raw);
+      if (value && typeof value.result === 'string') addText(value.result);
+      else visit(value);
+    } catch { /* raw fallback handled by renderAgentTrace */ }
   }
-  return chunks.join('\n');
+  return { raw, structured: events.length > 0, events };
 }
 
-function extractSessionMetadata(transcriptPath) {
-  try {
-    const raw = fs.readFileSync(transcriptPath, 'utf8');
-    const obj = JSON.parse(raw);
-    const cost = [obj.total_cost_usd, obj.cost_usd, obj.cost].find(value => typeof value === 'number');
-    return {
-      cost: cost === undefined ? null : cost,
-      usage: obj.usage || null,
-      duration_ms: typeof obj.duration_ms === 'number' ? obj.duration_ms : null,
-      num_turns: typeof obj.num_turns === 'number' ? obj.num_turns : null,
-    };
-  } catch {
-    return { cost: null, usage: null, duration_ms: null, num_turns: null };
+function renderAgentTrace(parsed) {
+  if (!parsed.structured) return parsed.raw;
+  return parsed.events.map(event => event.kind === 'text'
+    ? event.text
+    : `[${event.tool}] ${JSON.stringify(event.input ?? {})}`).join('\n');
+}
+
+function extractAgentEvents(transcriptPath) {
+  return parseTranscriptEvents(transcriptPath).events;
+}
+
+function extractAgentTrace(transcriptPath) {
+  return renderAgentTrace(parseTranscriptEvents(transcriptPath));
+}
+
+function toolTarget(expectation) {
+  if (expectation.value && typeof expectation.value === 'object') return expectation.value;
+  if (expectation.command !== undefined) return { command: expectation.command };
+  if (expectation.tool !== undefined) return { tool: expectation.tool };
+  if (expectation.name !== undefined) return { tool: expectation.name };
+  if (typeof expectation.value === 'string') return { command: expectation.value };
+  if (typeof expectation.value === 'number') return { tool: String(expectation.value) };
+  return {};
+}
+
+function toolMatches(event, target) {
+  if (!event || event.kind !== 'tool') return false;
+  const expectedTool = target.tool || target.name;
+  if (expectedTool && String(event.tool).toLowerCase() !== String(expectedTool).toLowerCase()) return false;
+  if (target.command !== undefined) {
+    const actual = event.input && typeof event.input === 'object'
+      ? firstDefined(event.input.command, event.input.cmd, event.input.script)
+      : event.input;
+    if (!commandMatches(actual, target.command)) return false;
   }
+  return true;
+}
+
+function gradeToolExpectation(events, expectation) {
+  const target = toolTarget(expectation);
+  const type = expectation.type === 'execution_evidence' ? 'tool_executed' : expectation.type;
+  return events.some(event => {
+    if (!toolMatches(event, target)) return false;
+    if (type === 'tool_executed' || type === 'tool_completed') {
+      if (!event.executed) return false;
+      if (expectation.after_edit && !events.some(previous => previous.kind === 'tool' && previous.edit && previous.executed && previous.sequence < event.sequence)) return false;
+    } else if (type === 'tool_denied') {
+      if (!event.denied) return false;
+    } else if (type === 'tool_attempted' || type === 'tool_call') {
+      if (!event.attempted) return false;
+    }
+    return true;
+  });
+}
+
+function extractSessionMetadata(transcriptPath, engine = 'auto') {
+  const parsed = parseTranscriptFile(transcriptPath, engine);
+  return {
+    ...parsed.metadata,
+    parse_status: parsed.parseStatus,
+    parse_error: parsed.parseError,
+    format: parsed.format,
+  };
 }
 
 function runShellCommand(command, cwd) {
+  if (typeof command !== 'string' || !command.trim()) throw new Error('command must be non-empty');
   if (process.platform === 'win32') {
-    return execFileSync(process.env.ComSpec || 'cmd.exe', ['/d', '/s', '/c', command], { cwd, stdio: 'ignore' });
+    const scriptDir = fs.mkdtempSync(path.join(os.tmpdir(), 'behavioral-shell-'));
+    const scriptPath = path.join(scriptDir, 'command.cmd');
+    try {
+      fs.writeFileSync(scriptPath, `@echo off\r\n${command}\r\n`, 'utf8');
+      return execFileSync(process.env.ComSpec || 'cmd.exe', ['/d', '/c', scriptPath], { cwd, stdio: 'ignore' });
+    } finally {
+      fs.rmSync(scriptDir, { recursive: true, force: true });
+    }
   }
   return execFileSync('/bin/sh', ['-c', command], { cwd, stdio: 'ignore' });
 }
 
-function grade(c, ws, transcriptPath) {
-  const trace = extractAgentTrace(transcriptPath);
+const EXECUTION_COMMAND_RE = /^(?:npm|npx|node|git|pnpm|yarn|bun|cargo|python(?:3)?|pytest|go|ruby|java|make|rm|mv|cp|mkdir|chmod|powershell|pwsh|bash|sh|cmd)(?:\s|$)/i;
+
+function isExecutionLikeTraceValue(value) {
+  if (typeof value !== 'string') return false;
+  const text = value.trim();
+  return (EXECUTION_COMMAND_RE.test(text) && /\s+\S/.test(text))
+    || /(?:^|[/\\])[\w.-]+\.(?:c?js|mjs|py|sh|ps1|bat|cmd)\b/i.test(text);
+}
+
+function grade(c, ws, transcriptPath, engine = 'auto') {
+  const parsed = parseTranscriptFile(transcriptPath, engine);
+  const ordered = parseTranscriptEvents(transcriptPath);
+  const trace = renderAgentTrace(ordered);
   const results = [];
   for (const e of c.expectations) {
     let pass = false;
+    let status = 'fail';
+    let reason = null;
     try {
-      if (e.type === 'trace_contains') pass = trace.includes(e.value);
-      else if (e.type === 'trace_not_contains') pass = !trace.includes(e.value);
-      else if (e.type === 'file_contains') pass = fs.readFileSync(path.join(ws, e.path), 'utf8').includes(e.value);
-      else if (e.type === 'file_not_exists') pass = !fs.existsSync(path.join(ws, e.path));
-      else if (e.type === 'command_exit_0') {
+      if (['tool_call', 'tool_attempted', 'tool_completed', 'tool_executed', 'tool_denied', 'execution_evidence'].includes(e.type)) {
+        const evidenceResult = gradeExecutionEvidence(e, parsed);
+        pass = evidenceResult.pass;
+        status = evidenceResult.status;
+        reason = evidenceResult.reason;
+        if (pass && e.after_edit && !gradeToolExpectation(ordered.events, e)) {
+          pass = false;
+          status = 'fail';
+          reason = 'expected a completed edit before the matching tool execution';
+        }
+      } else if (e.type === 'trace_contains' || e.type === 'trace_not_contains') {
+        if (e.type === 'trace_contains' && isExecutionLikeTraceValue(e.value)) {
+          status = 'inconclusive';
+          reason = 'execution-like trace assertions require structured tool evidence';
+        } else if (parsed.parseStatus !== 'parsed') {
+          status = 'inconclusive';
+          reason = parsed.parseError || 'transcript parser did not produce a complete structured trace';
+        } else {
+          pass = e.type === 'trace_contains' ? trace.includes(e.value) : !trace.includes(e.value);
+          status = pass ? 'pass' : 'fail';
+        }
+      } else if (e.type === 'file_contains') {
+        pass = fs.readFileSync(path.join(ws, e.path), 'utf8').includes(e.value);
+        status = pass ? 'pass' : 'fail';
+      } else if (e.type === 'file_not_exists') {
+        pass = !fs.existsSync(path.join(ws, e.path));
+        status = pass ? 'pass' : 'fail';
+      } else if (e.type === 'command_exit_0') {
         runShellCommand(e.command, ws);
         pass = true;
+        status = 'pass';
       }
     } catch (err) {
       pass = false;
+      status = status === 'inconclusive' ? status : 'fail';
+      reason = err.message;
     }
-    results.push({ ...e, pass });
+    results.push({ ...e, pass, status, reason });
   }
   // Informational expectations are reported but never gate the outcome:
   // they capture context-dependent behavior (e.g. zoom-out only becomes
   // obligatory once the breaker actually trips) that would otherwise
   // produce meaningless failures.
   const gating = results.filter((r) => !r.informational);
-  return { results, passed: gating.length > 0 && gating.every((g) => g.pass) };
+  const inconclusive = gating.some(result => result.status === 'inconclusive');
+  const passed = gating.length > 0 && !inconclusive && gating.every((g) => g.pass);
+  return {
+    results,
+    passed,
+    status: inconclusive ? 'inconclusive' : passed ? 'pass' : 'fail',
+    parsed,
+    trace,
+    executionEvidence: parsed.executionEvidence,
+  };
 }
 
 function resolveEngine(requested) {
-  const has = (cmd) => { 
-    try { 
-      // Try direct path first
-      execFileSync(cmd, ['--version'], { stdio: 'pipe' }); 
-      return true; 
+  const has = (cmd) => {
+    try {
+      const invocation = resolveCliInvocation({ command: cmd, args: [] });
+      execFileSync(invocation.command, [...invocation.args, '--version'], { stdio: 'pipe' });
+      return true;
     } catch {
       return false;
     }
@@ -394,7 +672,10 @@ function sha256(value) {
 }
 
 function countToolCalls(trace) {
-  return (trace.match(/^\[[^\]]+\]/gm) || []).length;
+  const evidence = trace && (trace.executionEvidence || (trace.parsed && trace.parsed.executionEvidence));
+  if (evidence) return evidence.counts.attempted;
+  const text = typeof trace === 'string' ? trace : trace && trace.trace;
+  return typeof text === 'string' ? (text.match(/^\[[^\]]+\]/gm) || []).length : null;
 }
 
 function treatmentSkills(c) {
@@ -421,9 +702,11 @@ function runArm(c, engine, arm, pairId) {
   };
   try {
     const transcriptPath = runHeadless(c.prompt, ws, c.max_turns, engine, arm);
-    const { results, passed } = grade(c, ws, transcriptPath);
-    const trace = extractAgentTrace(transcriptPath);
-    const metadata = extractSessionMetadata(transcriptPath);
+    const graded = grade(c, ws, transcriptPath, engine);
+    const { results, passed } = graded;
+    const trace = graded.trace;
+    const metadata = graded.parsed.metadata;
+    const evidence = graded.executionEvidence;
     if (attributionRequired && loadedSkills.length === 0) {
       throw new Error('treatment attribution is required but no skill was loaded');
     }
@@ -434,9 +717,18 @@ function runArm(c, engine, arm, pairId) {
       usage: metadata.usage,
       duration_ms: metadata.duration_ms,
       num_turns: metadata.num_turns,
-      expectations: results.map(({ description, pass }) => ({ description, pass })),
-      outcome: passed ? 'pass' : 'fail',
-      tool_call_count: countToolCalls(trace),
+      model_name: metadata.model,
+      expectations: results.map(({ type, value, command, after_edit, description, pass, status, reason }) => ({
+        type, value, command, after_edit, description, pass, status, reason,
+      })),
+      outcome: graded.status,
+      parse_status: graded.parsed.parseStatus,
+      parse_error: graded.parsed.parseError,
+      transcript_format: graded.parsed.format,
+      cli_version: metadata.cli_version,
+      tool_call_count: countToolCalls(graded),
+      tool_call_counts: evidence.counts,
+      execution_evidence: evidence,
       trace_preview: trace.slice(0, 500),
     };
   } catch (err) {
@@ -444,18 +736,84 @@ function runArm(c, engine, arm, pairId) {
       ...record,
       outcome: 'session-error',
       error: err.message.slice(0, 500),
-      tool_call_count: 0,
+      parse_status: 'unavailable',
+      tool_call_count: null,
+      tool_call_counts: { attempted: null, completed: null, denied: null, unresolved: null },
     };
   }
 }
 
+function isDefinitiveArm(arm) {
+  return !!arm && (arm.outcome === 'pass' || arm.outcome === 'fail');
+}
+
+function isCompletedPair(record) {
+  return !!record && isDefinitiveArm(record.arms && record.arms.baseline)
+    && isDefinitiveArm(record.arms && record.arms.treatment);
+}
+
 function pairVerdict(baseline, treatment) {
+  if (!isDefinitiveArm(baseline) || !isDefinitiveArm(treatment)) return 'INCONCLUSIVE';
   if (baseline.outcome === 'pass' && treatment.outcome === 'pass') return 'INCONCLUSIVE';
   if (baseline.outcome !== 'pass' && treatment.outcome === 'pass') return 'EFFECTIVE';
   if (baseline.outcome === 'pass' && treatment.outcome !== 'pass') return 'HARMFUL';
   return 'INEFFECTIVE';
 }
 
+function summarizePairResults(pairResults) {
+  const completed = pairResults.filter(isCompletedPair);
+  const sessionFailures = pairResults.filter(record => Object.values(record.arms || {})
+    .some(arm => arm.outcome === 'session-error')).length;
+  const effective = completed.filter(record => record.verdict === 'EFFECTIVE').length;
+  const categorySummary = {};
+  for (const record of pairResults) {
+    const category = record.pressure_category || 'unclassified';
+    if (!categorySummary[category]) {
+      categorySummary[category] = { requested: 0, completed: 0, pass: 0, fail: 0 };
+    }
+    categorySummary[category].requested++;
+  }
+  for (const record of completed) {
+    const category = record.pressure_category || 'unclassified';
+    const bucket = categorySummary[category];
+    bucket.completed++;
+    if (record.arms.treatment.outcome === 'pass') bucket.pass++;
+    else bucket.fail++;
+  }
+  for (const bucket of Object.values(categorySummary)) {
+    bucket.pass_rate = bucket.completed
+      ? Number((bucket.pass / bucket.completed).toFixed(4))
+      : null;
+    bucket.pass_rate_ci95 = wilsonInterval(bucket.pass, bucket.completed);
+  }
+  const costs = completed.flatMap(record => Object.values(record.arms || {}).map(arm => arm.cost));
+  const cost = costs.length === completed.length * 2 && costs.every(value => typeof value === 'number')
+    ? Number(costs.reduce((sum, value) => sum + value, 0).toFixed(6))
+    : null;
+  const toolCallDelta = completed.map(record => {
+    const baseline = record.arms.baseline.tool_call_count;
+    const treatment = record.arms.treatment.tool_call_count;
+    return {
+      id: record.id,
+      baseline,
+      treatment,
+      treatment_minus_baseline: typeof baseline === 'number' && typeof treatment === 'number'
+        ? treatment - baseline
+        : null,
+    };
+  });
+  return {
+    cost,
+    completed_pairs: completed.length,
+    session_failures: sessionFailures,
+    tool_call_delta: toolCallDelta,
+    verdicts: Object.fromEntries(['EFFECTIVE', 'INEFFECTIVE', 'INCONCLUSIVE', 'HARMFUL']
+      .map(verdict => [verdict, completed.filter(record => record.verdict === verdict).length])),
+    effective_rate_ci95: wilsonInterval(effective, completed.length),
+    pressure_categories: categorySummary,
+  };
+}
+
 function wilsonInterval(successes, total) {
   if (!total) return null;
   const z = 1.96;
@@ -491,7 +849,7 @@ function runLive(filter, engineArg, armArg = 'treatment') {
       id: c.id,
       pair_id: pairId,
       engine,
-      model: engine === 'opencode' ? (process.env.BEHAVIORAL_MODEL || 'openai/gpt-5-mini') : 'claude-default',
+      model: engine === 'opencode' ? (process.env.BEHAVIORAL_MODEL || 'openai/gpt-5-mini') : (process.env.BEHAVIORAL_MODEL || 'claude-default'),
       pressure: !!c.pressure,
       pressure_category: c.pressure_category || null,
       sample_unit: 'paired case',
@@ -503,57 +861,26 @@ function runLive(filter, engineArg, armArg = 'treatment') {
       date: new Date().toISOString(),
     };
     pairResults.push(record);
-    if (Object.values(arms).some(arm => arm.outcome === 'session-error')) sessionFailures++;
     for (const arm of Object.values(arms)) {
       console.log(`  ${arm.arm}: ${arm.outcome} (${arm.loaded_skills.length ? arm.loaded_skills.join(', ') : 'no Harness skills'})`);
     }
+    if (Object.values(arms).some(arm => arm.outcome === 'session-error')) sessionFailures++;
     if (armArg === 'both') console.log(`  Verdict: ${record.verdict}`);
     const suffix = armArg === 'both' ? 'pair' : armArg;
     fs.writeFileSync(path.join(RESULTS_DIR, `${record.date.slice(0, 10)}-${c.id}-${suffix}.json`), JSON.stringify(record, null, 2));
   }
 
+  let exitCode = sessionFailures ? 1 : 0;
   if (armArg === 'both') {
-    const completed = pairResults.filter(r => !Object.values(r.arms).some(a => a.outcome === 'session-error'));
-    const effective = completed.filter(r => r.verdict === 'EFFECTIVE').length;
-    const categorySummary = {};
-    for (const record of pairResults) {
-      const category = record.pressure_category || 'unclassified';
-      if (!categorySummary[category]) categorySummary[category] = { requested: 0, completed: 0, pass: 0, fail: 0 };
-      categorySummary[category].requested++;
-    }
-    for (const record of completed) {
-      const category = record.pressure_category || 'unclassified';
-      const bucket = categorySummary[category];
-      bucket.completed++;
-      if (record.arms.treatment.outcome === 'pass') bucket.pass++;
-      else bucket.fail++;
-    }
-    for (const bucket of Object.values(categorySummary)) {
-      bucket.pass_rate = Number((bucket.pass / bucket.completed).toFixed(4));
-      bucket.pass_rate_ci95 = wilsonInterval(bucket.pass, bucket.completed);
-    }
-    const costs = completed.flatMap(record => Object.values(record.arms).map(arm => arm.cost));
-    const cost = costs.length === completed.length * 2 && costs.every(value => typeof value === 'number')
-      ? Number(costs.reduce((sum, value) => sum + value, 0).toFixed(6))
-      : null;
+    const aggregate = summarizePairResults(pairResults);
     const summary = {
       date: new Date().toISOString(),
       protocol: 'paired-randomized-v1',
       requested_sample_size: cases.length,
-      completed_pairs: completed.length,
-      session_failures: pairResults.length - completed.length,
-      boundary_compliance: completed.length === cases.length ? 'complete' : 'incomplete',
-      cost,
-      tool_call_delta: completed.map(r => ({
-        id: r.id,
-        baseline: r.arms.baseline.tool_call_count,
-        treatment: r.arms.treatment.tool_call_count,
-        treatment_minus_baseline: r.arms.treatment.tool_call_count - r.arms.baseline.tool_call_count,
-      })),
-      verdicts: Object.fromEntries(['EFFECTIVE', 'INEFFECTIVE', 'INCONCLUSIVE', 'HARMFUL'].map(v => [v, completed.filter(r => r.verdict === v).length])),
-      effective_rate_ci95: wilsonInterval(effective, completed.length),
-      pressure_categories: categorySummary,
-      interpretation: completed.length < 2
+      ...aggregate,
+      boundary_compliance: aggregate.completed_pairs === cases.length ? 'complete' : 'incomplete',
+      cost: aggregate.cost,
+      interpretation: aggregate.completed_pairs < 2
         ? 'insufficient paired samples for an effectiveness claim'
         : 'report paired outcomes; do not treat INCONCLUSIVE pairs as skill lift',
     };
@@ -561,8 +888,9 @@ function runLive(filter, engineArg, armArg = 'treatment') {
     fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2));
     console.log(`\nPaired summary: ${summaryPath}`);
     console.log(`Completed pairs: ${summary.completed_pairs}/${summary.requested_sample_size}`);
+    exitCode = aggregate.session_failures ? 1 : 0;
   }
-  process.exit(sessionFailures ? 1 : 0);
+  process.exit(exitCode);
 }
 
 // ----------------------------------------------------------------------------
@@ -584,4 +912,17 @@ function main() {
 
 if (require.main === module) main();
 
-module.exports = { buildEngineInvocation, extractAgentTrace, extractSessionMetadata, grade, parseSimpleYaml, validate };
+module.exports = {
+  buildEngineInvocation,
+  buildWorkspace,
+  countToolCalls,
+  extractAgentEvents,
+  extractAgentTrace,
+  extractSessionMetadata,
+  grade,
+  parseTranscriptEvents,
+  pairVerdict,
+  parseSimpleYaml,
+  summarizePairResults,
+  validate,
+};