harness-everything 0.26.0 → 0.28.0

package/behavioral-evals/run.js

@@ -22,12 +22,13 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const crypto = require('crypto');
-const { execFileSync } = require('child_process');
+const { execFileSync, spawnSync } = require('child_process');
 const {
   gradeExecutionEvidence,
   parseTranscriptFile,
 } = require('./transcript-parser');
 const { commandMatches, normalizeRepoPath } = require('../scripts/lib/execution-contract');
+const { resolveFixtureSetup, validateFixture } = require('./case-validator');
 
 const ROOT = path.resolve(__dirname, '..');
 const CASES_DIR = path.join(__dirname, 'cases');
@@ -155,7 +156,8 @@ function validate(cases) {
     if (!c.id) problems.push('missing id');
     if (!c.prompt) problems.push('missing prompt');
     if (!c.max_turns) problems.push('missing max_turns');
-    if (!c.fixture || !Array.isArray(c.fixture.files)) problems.push('fixture.files missing');
+    if (!c.fixture) problems.push('fixture missing');
+    else problems.push(...validateFixture(c.fixture));
     if (!Array.isArray(c.expectations) || c.expectations.length === 0) problems.push('expectations missing');
     if (c.loaded_skills !== undefined && (!Array.isArray(c.loaded_skills) || c.loaded_skills.some(skill => !fs.existsSync(path.join(ROOT, skill, 'SKILL.md'))))) {
       problems.push('loaded_skills must name existing skills');
@@ -207,6 +209,66 @@ function buildWorkspace(c) {
   return ws;
 }
 
+function runFixtureSetup(c, ws) {
+  const resolved = resolveFixtureSetup(c && c.fixture && c.fixture.setup);
+  if (!resolved) return { path: null, script_sha256: null, status: 'not-requested', exit_code: null, duration_ms: 0 };
+  const scriptSha256 = sha256(fs.readFileSync(resolved.absolute));
+  const startedAt = Date.now();
+  const result = spawnSync(process.execPath, [resolved.absolute], {
+    cwd: ws,
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      HARNESS_BEHAVIORAL_WORKSPACE: ws,
+      HARNESS_BEHAVIORAL_CASE_ID: String(c.id || ''),
+    },
+    timeout: 60 * 1000,
+    windowsHide: true,
+  });
+  const evidence = {
+    path: resolved.relative,
+    script_sha256: scriptSha256,
+    status: result.status === 0 && !result.error ? 'pass' : 'fail',
+    exit_code: result.status,
+    duration_ms: Date.now() - startedAt,
+  };
+  if (result.error || result.status !== 0) {
+    const detail = (result.stderr || result.stdout || (result.error && result.error.message) || '').trim();
+    const error = new Error(
+      `fixture setup failed: ${resolved.relative}${result.status === null ? '' : ` (exit ${result.status})`}${detail ? `: ${detail.slice(0, 300)}` : ''}`
+    );
+    error.fixtureSetup = evidence;
+    throw error;
+  }
+  return evidence;
+}
+
+function prepareWorkspace(c) {
+  let workspace = null;
+  try {
+    workspace = buildWorkspace(c);
+    return {
+      ok: true,
+      workspace,
+      setup: runFixtureSetup(c, workspace),
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      workspace,
+      outcome: 'fixture-error',
+      error: error.message.slice(0, 500),
+      setup: error.fixtureSetup || {
+        path: c && c.fixture ? c.fixture.setup || null : null,
+        script_sha256: null,
+        status: 'fail',
+        exit_code: null,
+        duration_ms: null,
+      },
+    };
+  }
+}
+
 // Capture the requested fixture baseline after treatment installation so the
 // installer's own artifacts (.claude/, .harness-src/, opencode.json) show up
 // in neither arm's post-run scope check.
@@ -685,9 +747,8 @@ function treatmentSkills(c) {
 function runArm(c, engine, arm, pairId) {
   const loadedSkills = arm === 'treatment' ? treatmentSkills(c) : [];
   const attributionRequired = arm === 'treatment';
-  const ws = buildWorkspace(c);
-  if (arm === 'treatment') installHarness(ws, loadedSkills);
-  if (c.fixture.git) gitSnapshot(ws);
+  const prepared = prepareWorkspace(c);
+  const ws = prepared.workspace;
 
   const record = {
     arm,
@@ -698,9 +759,22 @@ function runArm(c, engine, arm, pairId) {
     workspace: ws,
     fixture_sha256: sha256(JSON.stringify(c.fixture)),
     prompt_sha256: sha256(c.prompt),
+    fixture_setup: prepared.setup,
     cost: null,
   };
+  if (!prepared.ok) {
+    return {
+      ...record,
+      outcome: 'fixture-error',
+      error: prepared.error,
+      parse_status: 'unavailable',
+      tool_call_count: null,
+      tool_call_counts: { attempted: null, completed: null, denied: null, unresolved: null },
+    };
+  }
   try {
+    if (arm === 'treatment') installHarness(ws, loadedSkills);
+    if (c.fixture.git) gitSnapshot(ws);
     const transcriptPath = runHeadless(c.prompt, ws, c.max_turns, engine, arm);
     const graded = grade(c, ws, transcriptPath, engine);
     const { results, passed } = graded;
@@ -764,6 +838,8 @@ function summarizePairResults(pairResults) {
   const completed = pairResults.filter(isCompletedPair);
   const sessionFailures = pairResults.filter(record => Object.values(record.arms || {})
     .some(arm => arm.outcome === 'session-error')).length;
+  const fixtureFailures = pairResults.filter(record => Object.values(record.arms || {})
+    .some(arm => arm.outcome === 'fixture-error')).length;
   const effective = completed.filter(record => record.verdict === 'EFFECTIVE').length;
   const categorySummary = {};
   for (const record of pairResults) {
@@ -806,6 +882,7 @@ function summarizePairResults(pairResults) {
     cost,
     completed_pairs: completed.length,
     session_failures: sessionFailures,
+    fixture_failures: fixtureFailures,
     tool_call_delta: toolCallDelta,
     verdicts: Object.fromEntries(['EFFECTIVE', 'INEFFECTIVE', 'INCONCLUSIVE', 'HARMFUL']
       .map(verdict => [verdict, completed.filter(record => record.verdict === verdict).length])),
@@ -835,7 +912,7 @@ function runLive(filter, engineArg, armArg = 'treatment') {
   fs.mkdirSync(RESULTS_DIR, { recursive: true });
 
   const pairResults = [];
-  let sessionFailures = 0;
+  let infrastructureFailures = 0;
   for (const c of cases) {
     const pairId = `${c.id}-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`;
     const order = armArg === 'both'
@@ -864,13 +941,13 @@ function runLive(filter, engineArg, armArg = 'treatment') {
     for (const arm of Object.values(arms)) {
       console.log(`  ${arm.arm}: ${arm.outcome} (${arm.loaded_skills.length ? arm.loaded_skills.join(', ') : 'no Harness skills'})`);
     }
-    if (Object.values(arms).some(arm => arm.outcome === 'session-error')) sessionFailures++;
+    if (Object.values(arms).some(arm => arm.outcome === 'session-error' || arm.outcome === 'fixture-error')) infrastructureFailures++;
     if (armArg === 'both') console.log(`  Verdict: ${record.verdict}`);
     const suffix = armArg === 'both' ? 'pair' : armArg;
     fs.writeFileSync(path.join(RESULTS_DIR, `${record.date.slice(0, 10)}-${c.id}-${suffix}.json`), JSON.stringify(record, null, 2));
   }
 
-  let exitCode = sessionFailures ? 1 : 0;
+  let exitCode = infrastructureFailures ? 1 : 0;
   if (armArg === 'both') {
     const aggregate = summarizePairResults(pairResults);
     const summary = {
@@ -888,7 +965,7 @@ function runLive(filter, engineArg, armArg = 'treatment') {
     fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2));
     console.log(`\nPaired summary: ${summaryPath}`);
     console.log(`Completed pairs: ${summary.completed_pairs}/${summary.requested_sample_size}`);
-    exitCode = aggregate.session_failures ? 1 : 0;
+    exitCode = (aggregate.session_failures || aggregate.fixture_failures) ? 1 : 0;
   }
   process.exit(exitCode);
 }
@@ -915,6 +992,8 @@ if (require.main === module) main();
 module.exports = {
   buildEngineInvocation,
   buildWorkspace,
+  runFixtureSetup,
+  prepareWorkspace,
   countToolCalls,
   extractAgentEvents,
   extractAgentTrace,

package/ci/mechanism-2j-behavioral-runner.test.js

@@ -2,7 +2,8 @@ const helper = require('./test-helper');
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
-const { buildEngineInvocation, buildWorkspace } = require('../behavioral-evals/run');
+const { spawnSync } = require('child_process');
+const { buildEngineInvocation, buildWorkspace, runFixtureSetup, prepareWorkspace, summarizePairResults } = require('../behavioral-evals/run');
 
 console.log('\n[2j] Behavioral runner argv integrity...');
 const prompt = "Add punctuation stripping to slug.js, then run npm test. Do not truncate this request.";
@@ -59,4 +60,123 @@ helper.check(
   traversalError && traversalError.stack ? traversalError.stack : traversalTarget
 );
 
+let setupWorkspace = null;
+try {
+  setupWorkspace = buildWorkspace({
+    id: 'fixture-setup-valid',
+    fixture: {
+      files: [{ path: 'seed.txt', content: 'seed' }],
+      setup: 'behavioral-evals/fixtures/setup/noop.js',
+    },
+  });
+  const setupResult = runFixtureSetup({
+    id: 'fixture-setup-valid',
+    fixture: { setup: 'behavioral-evals/fixtures/setup/noop.js' },
+  }, setupWorkspace);
+  helper.check(
+    '2j. fixture setup runs via Node in the generated workspace',
+    setupResult.status === 'pass' &&
+      fs.readFileSync(path.join(setupWorkspace, '.fixture-setup-ok'), 'utf8') === 'ok\n',
+    JSON.stringify(setupResult)
+  );
+} catch (error) {
+  helper.check('2j. fixture setup runs via Node in the generated workspace', false, error && error.stack ? error.stack : String(error));
+} finally {
+  if (setupWorkspace) fs.rmSync(setupWorkspace, { recursive: true, force: true });
+}
+
+let failedPreparation = null;
+try {
+  failedPreparation = prepareWorkspace({
+    id: 'fixture-setup-fails',
+    fixture: {
+      files: [{ path: 'seed.txt', content: 'seed' }],
+      setup: 'behavioral-evals/fixtures/setup/fail.js',
+    },
+  });
+} catch (error) {
+  failedPreparation = { threw: error && error.stack ? error.stack : String(error) };
+}
+helper.check(
+  '2j. setup failure is classified as fixture-error before any model session',
+  Boolean(
+    failedPreparation &&
+    failedPreparation.ok === false &&
+    failedPreparation.outcome === 'fixture-error' &&
+    failedPreparation.setup &&
+    failedPreparation.setup.status === 'fail' &&
+    failedPreparation.setup.exit_code === 17
+  ),
+  JSON.stringify(failedPreparation)
+);
+if (failedPreparation && failedPreparation.workspace) {
+  fs.rmSync(failedPreparation.workspace, { recursive: true, force: true });
+}
+
+let topologyPreparation = null;
+try {
+  topologyPreparation = prepareWorkspace({
+    id: 'worktree-submodule-setup-smoke',
+    fixture: {
+      files: [{ path: 'README.md', content: 'topology smoke' }],
+      setup: 'behavioral-evals/fixtures/setup/worktree-submodule.js',
+    },
+  });
+  const linkedSub = topologyPreparation.ok
+    ? path.join(topologyPreparation.workspace, 'super-wt', 'libs', 'sub')
+    : null;
+  const branchResult = linkedSub
+    ? spawnSync('git', ['-C', linkedSub, 'branch', '--show-current'], { encoding: 'utf8', windowsHide: true })
+    : null;
+  const helperResult = topologyPreparation.ok
+    ? spawnSync(process.execPath, [
+        path.join(topologyPreparation.workspace, '.fixture-tools', 'submodule-reachability.js'),
+        '--json', '--root', 'super-wt',
+      ], { cwd: topologyPreparation.workspace, encoding: 'utf8', windowsHide: true })
+    : null;
+  helper.check(
+    '2j. worktree/submodule setup creates the real #243 topology before model execution',
+    Boolean(
+      topologyPreparation.ok &&
+      fs.existsSync(path.join(topologyPreparation.workspace, 'super-wt', '.git')) &&
+      fs.existsSync(path.join(linkedSub, '.git')) &&
+      branchResult && branchResult.status === 0 && branchResult.stdout.trim() === '' &&
+      helperResult && helperResult.status === 0
+    ),
+    JSON.stringify({
+      preparation: topologyPreparation,
+      branchStatus: branchResult && branchResult.status,
+      branch: branchResult && branchResult.stdout,
+      helperStatus: helperResult && helperResult.status,
+      helperStderr: helperResult && helperResult.stderr,
+    })
+  );
+} catch (error) {
+  helper.check(
+    '2j. worktree/submodule setup creates the real #243 topology before model execution',
+    false,
+    error && error.stack ? error.stack : String(error)
+  );
+} finally {
+  if (topologyPreparation && topologyPreparation.workspace) {
+    fs.rmSync(topologyPreparation.workspace, { recursive: true, force: true });
+  }
+}
+
+const fixtureFailureSummary = summarizePairResults([{
+  id: 'fixture-error-pair',
+  pressure_category: null,
+  arms: {
+    baseline: { outcome: 'fixture-error', cost: null, tool_call_count: null },
+    treatment: { outcome: 'pass', cost: null, tool_call_count: 1 },
+  },
+}]);
+helper.check(
+  '2j. fixture errors remain distinct aggregate infrastructure failures',
+  fixtureFailureSummary.fixture_failures === 1 &&
+    fixtureFailureSummary.session_failures === 0 &&
+    fixtureFailureSummary.completed_pairs === 0,
+  JSON.stringify(fixtureFailureSummary)
+);
+
 helper.finish();

package/ci/mechanism-2j-fixture-entrypoints.test.js

@@ -0,0 +1,153 @@
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { parseSimpleYaml, prepareWorkspace } = require('../behavioral-evals/run');
+const paired = require('../behavioral-evals/paired-benchmark');
+const ROOT = path.resolve(__dirname, '..');
+const topology = parseSimpleYaml(fs.readFileSync(path.join(ROOT, 'behavioral-evals/cases/baseline-worktree-submodule-reachability.yaml'), 'utf8'));
+const failure = { ...topology, fixture: { ...topology.fixture, setup: 'behavioral-evals/fixtures/setup/fail.js' } };
+const paths = [];
+const topologyFingerprints = [];
+const setupHashes = [];
+function track(ws) { if (ws) paths.push(ws); return ws; }
+function gitText(cwd, args) {
+  const result = spawnSync('git', args, { cwd, encoding: 'utf8', windowsHide: true });
+  assert.strictEqual(result.status, 0, result.stderr || result.stdout);
+  return result.stdout.trim();
+}
+function topologyFingerprint(ws) {
+  return {
+    child_head: gitText(path.join(ws, 'child-origin'), ['rev-parse', 'HEAD']),
+    super_head: gitText(path.join(ws, 'super'), ['rev-parse', 'HEAD']),
+    linked_head: gitText(path.join(ws, 'super-wt'), ['rev-parse', 'HEAD']),
+    gitlink: gitText(path.join(ws, 'super-wt'), ['rev-parse', 'HEAD:libs/sub']),
+  };
+}
+function assertTopology(ws) {
+  assert.ok(fs.existsSync(path.join(ws, 'super-wt', '.git')), 'setup must construct the linked worktree');
+  assert.ok(fs.existsSync(path.join(ws, '.fixture-tools', 'verify-worktree-submodule.js')), 'setup must install the grader');
+}
+try {
+  for (const arm of ['baseline', 'treatment']) {
+    for (const effect of ['skill-text', 'plugin-enforcement']) {
+      const prepared = paired.prepareArmWorkspace(topology, effect, arm, 'opencode');
+      track(prepared.ws);
+      assertTopology(prepared.ws);
+      assert.strictEqual(prepared.fixture_setup.status, 'pass');
+      assert.match(prepared.fixture_setup.script_sha256, /^[0-9a-f]{64}$/);
+      topologyFingerprints.push(topologyFingerprint(prepared.ws));
+      setupHashes.push(prepared.fixture_setup.script_sha256);
+      const failed = paired.prepareArmWorkspace(failure, effect, arm, 'opencode');
+      track(failed.ws);
+      assert.strictEqual(failed.ok, false);
+      assert.strictEqual(failed.outcome, 'fixture-error');
+      assert.strictEqual(failed.fixture_setup.exit_code, 17);
+      assert.ok(!fs.existsSync(path.join(failed.ws, '.claude')), 'setup failure must precede skill installation');
+      assert.ok(!fs.existsSync(path.join(failed.ws, '.opencode')), 'setup failure must precede plugin installation');
+    }
+  }
+  assert.ok(topologyFingerprints.length >= 4);
+  for (const fingerprint of topologyFingerprints.slice(1)) {
+    assert.deepStrictEqual(fingerprint, topologyFingerprints[0], 'fixture setup must produce identical Git object identities across independent arms/runs');
+  }
+  assert.strictEqual(new Set(setupHashes).size, 1, 'all arms must retain the same checked-in setup script hash');
+
+  const pairDir = track(fs.mkdtempSync(path.join(os.tmpdir(), 'fixture-entrypoints-')));
+  let launches = 0;
+  const context = { effect_type: 'skill-text', engine: 'claude', model: 'test' };
+  const arms = {};
+  for (const arm of ['baseline', 'treatment']) {
+    arms[arm] = paired.runArm(failure, context, arm, pairDir, () => { launches++; throw new Error('model must not start'); });
+    assert.strictEqual(arms[arm].outcome, 'fixture-error');
+    assert.strictEqual(arms[arm].fixture_setup.exit_code, 17);
+    assert.strictEqual(arms[arm].tool_call_count, null);
+  }
+  assert.strictEqual(launches, 0);
+  const summary = paired.summarizePairs([{ id: topology.id, arms }], 10);
+  assert.strictEqual(summary.fixture_failures, 1);
+  assert.strictEqual(summary.included_pairs, 0);
+  assert.deepStrictEqual(summary.arm_outcomes, { baseline: { pass: 0, fail: 0 }, treatment: { pass: 0, fail: 0 } });
+
+  const plugin = require('../behavioral-evals/run-with-plugin');
+  const failed = plugin.runCase(failure, () => { launches++; throw new Error('model must not start'); });
+  track(failed.workspace);
+  assert.strictEqual(failed.outcome, 'fixture-error');
+  assert.strictEqual(failed.fixture_setup.exit_code, 17);
+  assert.strictEqual(launches, 0);
+  assert.ok(!fs.existsSync(path.join(failed.workspace, '.harness-src')), 'failed setup must precede Harness installation');
+
+  // Exercise the same public case path as the plugin CLI, without calling a model.
+  const result = plugin.runCase(topology, (_prompt, ws) => {
+    launches++;
+    assertTopology(ws);
+    const transcript = path.join(pairDir, 'plugin-transcript.jsonl');
+    fs.writeFileSync(transcript, '');
+    return transcript;
+  });
+  track(result.workspace);
+  assert.strictEqual(launches, 1);
+  assert.strictEqual(result.fixture_setup.status, 'pass');
+  assert.strictEqual(result.outcome, 'fail', 'an untouched topology must fail the actual final-state graders');
+
+  for (const fixture of [
+    { files: [{ path: '../escape', content: 'bad' }] },
+    { files: [{ path: 'seed', content: 'ok' }], setup: '../escape.js' },
+  ]) {
+    const result = prepareWorkspace({ id: 'invalid-fixture', fixture });
+    track(result.workspace);
+    assert.strictEqual(result.outcome, 'fixture-error');
+  }
+  // The public CLIs must retain fixture-error evidence and exit nonzero.
+  const caseId = `fixture-entrypoints-failure-${process.pid}`;
+  const casePath = path.join(ROOT, 'behavioral-evals/cases', `${caseId}.yaml`);
+  fs.writeFileSync(casePath, `id: ${caseId}
+name: Fixture failure CLI control
+loaded_skills:
+  - git-commit
+fixture:
+  files:
+    - path: seed.txt
+      content: seed
+  setup: behavioral-evals/fixtures/setup/fail.js
+prompt: This model session must never start.
+max_turns: 1
+expectations:
+  - type: file_contains
+    path: seed.txt
+    value: seed
+`);
+  paths.push(casePath);
+  const pairedCli = spawnSync(process.execPath, [
+    path.join(ROOT, 'behavioral-evals/paired-benchmark.js'), 'run',
+    '--effect', 'skill-text', '--engine', 'claude', '--model', 'fixture-test',
+    '--min-effect-pp', '10', '--case', caseId,
+  ], { cwd: ROOT, encoding: 'utf8', timeout: 30000 });
+  const summaryMatch = pairedCli.stdout.match(/Summary: (.+)/);
+  assert.ok(summaryMatch, pairedCli.stderr || pairedCli.stdout);
+  const summaryPath = summaryMatch[1].trim();
+  track(path.dirname(summaryPath));
+  assert.strictEqual(pairedCli.status, 1, pairedCli.stderr);
+  const cliSummary = JSON.parse(fs.readFileSync(summaryPath, 'utf8'));
+  assert.strictEqual(cliSummary.fixture_failures, 1);
+  assert.strictEqual(cliSummary.included_pairs, 0);
+  const pluginCli = spawnSync(process.execPath, [
+    path.join(ROOT, 'behavioral-evals/run-with-plugin.js'), 'run', '--case', caseId,
+  ], { cwd: ROOT, encoding: 'utf8', timeout: 30000 });
+  assert.strictEqual(pluginCli.status, 1, pluginCli.stderr);
+  const resultMatch = pluginCli.stdout.match(/fixture-error -> (.+)/);
+  assert.ok(resultMatch, pluginCli.stdout);
+  const resultPath = resultMatch[1].trim();
+  paths.push(resultPath);
+  const pluginFailure = JSON.parse(fs.readFileSync(resultPath, 'utf8'));
+  track(pluginFailure.workspace);
+  assert.strictEqual(pluginFailure.fixture_setup.exit_code, 17);
+  assert.ok(!fs.existsSync(path.join(pluginFailure.workspace, '.harness-src')));
+
+  console.log('Fixture entrypoints: paired baseline/treatment and plugin execute setup; failures skip installation/model and are excluded from scores.');
+} finally {
+  for (const dir of paths) fs.rmSync(dir, { recursive: true, force: true });
+}