harness-everything 0.27.0 → 0.29.0

package/behavioral-evals/run.js

@@ -22,12 +22,13 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const crypto = require('crypto');
-const { execFileSync } = require('child_process');
+const { execFileSync, spawnSync } = require('child_process');
 const {
   gradeExecutionEvidence,
   parseTranscriptFile,
 } = require('./transcript-parser');
 const { commandMatches, normalizeRepoPath } = require('../scripts/lib/execution-contract');
+const { resolveFixtureSetup, validateFixture } = require('./case-validator');
 
 const ROOT = path.resolve(__dirname, '..');
 const CASES_DIR = path.join(__dirname, 'cases');
@@ -155,7 +156,8 @@ function validate(cases) {
     if (!c.id) problems.push('missing id');
     if (!c.prompt) problems.push('missing prompt');
     if (!c.max_turns) problems.push('missing max_turns');
-    if (!c.fixture || !Array.isArray(c.fixture.files)) problems.push('fixture.files missing');
+    if (!c.fixture) problems.push('fixture missing');
+    else problems.push(...validateFixture(c.fixture));
     if (!Array.isArray(c.expectations) || c.expectations.length === 0) problems.push('expectations missing');
     if (c.loaded_skills !== undefined && (!Array.isArray(c.loaded_skills) || c.loaded_skills.some(skill => !fs.existsSync(path.join(ROOT, skill, 'SKILL.md'))))) {
       problems.push('loaded_skills must name existing skills');
@@ -207,6 +209,66 @@ function buildWorkspace(c) {
   return ws;
 }
 
+function runFixtureSetup(c, ws) {
+  const resolved = resolveFixtureSetup(c && c.fixture && c.fixture.setup);
+  if (!resolved) return { path: null, script_sha256: null, status: 'not-requested', exit_code: null, duration_ms: 0 };
+  const scriptSha256 = sha256(fs.readFileSync(resolved.absolute));
+  const startedAt = Date.now();
+  const result = spawnSync(process.execPath, [resolved.absolute], {
+    cwd: ws,
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+      HARNESS_BEHAVIORAL_WORKSPACE: ws,
+      HARNESS_BEHAVIORAL_CASE_ID: String(c.id || ''),
+    },
+    timeout: 60 * 1000,
+    windowsHide: true,
+  });
+  const evidence = {
+    path: resolved.relative,
+    script_sha256: scriptSha256,
+    status: result.status === 0 && !result.error ? 'pass' : 'fail',
+    exit_code: result.status,
+    duration_ms: Date.now() - startedAt,
+  };
+  if (result.error || result.status !== 0) {
+    const detail = (result.stderr || result.stdout || (result.error && result.error.message) || '').trim();
+    const error = new Error(
+      `fixture setup failed: ${resolved.relative}${result.status === null ? '' : ` (exit ${result.status})`}${detail ? `: ${detail.slice(0, 300)}` : ''}`
+    );
+    error.fixtureSetup = evidence;
+    throw error;
+  }
+  return evidence;
+}
+
+function prepareWorkspace(c) {
+  let workspace = null;
+  try {
+    workspace = buildWorkspace(c);
+    return {
+      ok: true,
+      workspace,
+      setup: runFixtureSetup(c, workspace),
+    };
+  } catch (error) {
+    return {
+      ok: false,
+      workspace,
+      outcome: 'fixture-error',
+      error: error.message.slice(0, 500),
+      setup: error.fixtureSetup || {
+        path: c && c.fixture ? c.fixture.setup || null : null,
+        script_sha256: null,
+        status: 'fail',
+        exit_code: null,
+        duration_ms: null,
+      },
+    };
+  }
+}
+
 // Capture the requested fixture baseline after treatment installation so the
 // installer's own artifacts (.claude/, .harness-src/, opencode.json) show up
 // in neither arm's post-run scope check.
@@ -685,9 +747,8 @@ function treatmentSkills(c) {
 function runArm(c, engine, arm, pairId) {
   const loadedSkills = arm === 'treatment' ? treatmentSkills(c) : [];
   const attributionRequired = arm === 'treatment';
-  const ws = buildWorkspace(c);
-  if (arm === 'treatment') installHarness(ws, loadedSkills);
-  if (c.fixture.git) gitSnapshot(ws);
+  const prepared = prepareWorkspace(c);
+  const ws = prepared.workspace;
 
   const record = {
     arm,
@@ -698,9 +759,22 @@ function runArm(c, engine, arm, pairId) {
     workspace: ws,
     fixture_sha256: sha256(JSON.stringify(c.fixture)),
     prompt_sha256: sha256(c.prompt),
+    fixture_setup: prepared.setup,
     cost: null,
   };
+  if (!prepared.ok) {
+    return {
+      ...record,
+      outcome: 'fixture-error',
+      error: prepared.error,
+      parse_status: 'unavailable',
+      tool_call_count: null,
+      tool_call_counts: { attempted: null, completed: null, denied: null, unresolved: null },
+    };
+  }
   try {
+    if (arm === 'treatment') installHarness(ws, loadedSkills);
+    if (c.fixture.git) gitSnapshot(ws);
     const transcriptPath = runHeadless(c.prompt, ws, c.max_turns, engine, arm);
     const graded = grade(c, ws, transcriptPath, engine);
     const { results, passed } = graded;
@@ -764,6 +838,8 @@ function summarizePairResults(pairResults) {
   const completed = pairResults.filter(isCompletedPair);
   const sessionFailures = pairResults.filter(record => Object.values(record.arms || {})
     .some(arm => arm.outcome === 'session-error')).length;
+  const fixtureFailures = pairResults.filter(record => Object.values(record.arms || {})
+    .some(arm => arm.outcome === 'fixture-error')).length;
   const effective = completed.filter(record => record.verdict === 'EFFECTIVE').length;
   const categorySummary = {};
   for (const record of pairResults) {
@@ -806,6 +882,7 @@ function summarizePairResults(pairResults) {
     cost,
     completed_pairs: completed.length,
     session_failures: sessionFailures,
+    fixture_failures: fixtureFailures,
     tool_call_delta: toolCallDelta,
     verdicts: Object.fromEntries(['EFFECTIVE', 'INEFFECTIVE', 'INCONCLUSIVE', 'HARMFUL']
       .map(verdict => [verdict, completed.filter(record => record.verdict === verdict).length])),
@@ -835,7 +912,7 @@ function runLive(filter, engineArg, armArg = 'treatment') {
   fs.mkdirSync(RESULTS_DIR, { recursive: true });
 
   const pairResults = [];
-  let sessionFailures = 0;
+  let infrastructureFailures = 0;
   for (const c of cases) {
     const pairId = `${c.id}-${Date.now()}-${crypto.randomBytes(3).toString('hex')}`;
     const order = armArg === 'both'
@@ -864,13 +941,13 @@ function runLive(filter, engineArg, armArg = 'treatment') {
     for (const arm of Object.values(arms)) {
       console.log(`  ${arm.arm}: ${arm.outcome} (${arm.loaded_skills.length ? arm.loaded_skills.join(', ') : 'no Harness skills'})`);
     }
-    if (Object.values(arms).some(arm => arm.outcome === 'session-error')) sessionFailures++;
+    if (Object.values(arms).some(arm => arm.outcome === 'session-error' || arm.outcome === 'fixture-error')) infrastructureFailures++;
     if (armArg === 'both') console.log(`  Verdict: ${record.verdict}`);
     const suffix = armArg === 'both' ? 'pair' : armArg;
     fs.writeFileSync(path.join(RESULTS_DIR, `${record.date.slice(0, 10)}-${c.id}-${suffix}.json`), JSON.stringify(record, null, 2));
   }
 
-  let exitCode = sessionFailures ? 1 : 0;
+  let exitCode = infrastructureFailures ? 1 : 0;
   if (armArg === 'both') {
     const aggregate = summarizePairResults(pairResults);
     const summary = {
@@ -888,7 +965,7 @@ function runLive(filter, engineArg, armArg = 'treatment') {
     fs.writeFileSync(summaryPath, JSON.stringify(summary, null, 2));
     console.log(`\nPaired summary: ${summaryPath}`);
     console.log(`Completed pairs: ${summary.completed_pairs}/${summary.requested_sample_size}`);
-    exitCode = aggregate.session_failures ? 1 : 0;
+    exitCode = (aggregate.session_failures || aggregate.fixture_failures) ? 1 : 0;
   }
   process.exit(exitCode);
 }
@@ -915,6 +992,8 @@ if (require.main === module) main();
 module.exports = {
   buildEngineInvocation,
   buildWorkspace,
+  runFixtureSetup,
+  prepareWorkspace,
   countToolCalls,
   extractAgentEvents,
   extractAgentTrace,

package/benchmarks/fixtures/system-one-holdout-reviews.json

@@ -410,15 +410,15 @@
     {
       "id": "h-fix-cli-error-log-3",
       "promptHash": "09d285719d1f96cd6740d19922cafc6d8dd0cf2c69107c6684406e4c71a754f3",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
       "id": "h-fix-cli-error-log-4",
       "promptHash": "cf19ff4c74605c34d4e8743ff71dd61978178b070c79ea392b7c9c56ea44a933",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
@@ -438,8 +438,8 @@
     {
       "id": "h-fix-failing-test-3",
       "promptHash": "a06402d978c65a532d23ee2381e4e062154adfffb446021c8c5102cad93a8202",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
@@ -452,22 +452,22 @@
     {
       "id": "h-add-cli-flag-1",
       "promptHash": "ae03dbf3193395e0de90b58741c3e866ce2cd3bf08a501820d6a13632d1cd1bb",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
       "id": "h-add-cli-flag-2",
       "promptHash": "dee4d30095d0eb8a53d969dd58773cf9d74c27766b67ec0e2871e4d28bf1f12a",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
       "id": "h-add-cli-flag-3",
       "promptHash": "5ba561160b690b0725c6d40956d2ee1deb97745adc418fc137e6774448295f66",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
@@ -613,15 +613,15 @@
     {
       "id": "h-hash-algorithm-support-1",
       "promptHash": "f0da51303eeb3a54985a29bc91cc456438222a68d0632cab72b07edfb18a3795",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
       "id": "h-hash-algorithm-support-2",
       "promptHash": "74b9af0f162bd5f25b8c7aa032120c4c6fd950fa0e4badfe65ede93adba251c6",
-      "decision": "relabel",
-      "gold": "tier3",
+      "decision": "accept",
+      "gold": "tier2",
       "reviewer": "human:repository-owner"
     },
     {
@@ -1579,8 +1579,8 @@
     {
       "id": "h-long-git-ops-1",
       "promptHash": "38d447c5f21ff104d786066a36ff1a76d11886c1a6361248e478cf953121a888",
-      "decision": "relabel",
-      "gold": "tier2",
+      "decision": "accept",
+      "gold": "tier1",
       "reviewer": "human:repository-owner"
     }
   ]

package/benchmarks/fixtures/system-one-holdout.json

@@ -2005,7 +2005,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "f3cf716b74e5fe928f8047acda0267e8490cd69ff17db017eb47ca77cd35a4a4"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-fix-cli-error-log-4",
@@ -2039,7 +2039,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "fdd46662e02f8d3230b840fe089b4e4cb4a37aef1aa998f59e3ffaa4637093bc"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-fix-failing-test-1",
@@ -2141,7 +2141,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "dd3cf4d63ff04266de08e6235641a3e191a2485cb849a5dc3e44eeacfc7238a3"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-fix-failing-test-4",
@@ -2209,7 +2209,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "b9438ca5119663f987c91d4b851b71e1f0f9e15547e4405b752302116cfe2de1"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-add-cli-flag-2",
@@ -2243,7 +2243,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "89dc4acaee4ca2c8f070fe1eb7e8da8a11025f78d3cf9d8b15f14cc22af7538a"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-add-cli-flag-3",
@@ -2277,7 +2277,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "426171654d98fe77a50a5dced0053d3539595cbf28d23259e680fffbad603230"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-nondeterministic-output-1",
@@ -2991,7 +2991,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "cb231efa719d2585111733df0da9907aa0d8b046d806a8f77a49a62c22e8bf5a"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-hash-algorithm-support-2",
@@ -3025,7 +3025,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "d6f57302ad014bf5dc923fffd50895568941dde27b4966d8bf9b428d7dc827fa"
       },
-      "gold": "tier3"
+      "gold": "tier2"
     },
     {
       "id": "h-toolchain-check-1",
@@ -7309,7 +7309,7 @@
         "catalogHash": "b625bd9390f1d163f8fbace0b9c4768166bfc613aed04d87b2246b2c6e966d40",
         "requestHash": "d48c25217d11d4d8881768a373dc92125fef499d200ce49d9a0b7d461e13831f"
       },
-      "gold": "tier2"
+      "gold": "tier1"
     }
   ]
 }

package/benchmarks/results/system-one/harness-routing-v1-cpu-2026-09-24/README.md

@@ -0,0 +1,75 @@
+# System One: first harness-routing-v1 checkpoint (2026-09-24)
+
+This is local real-checkpoint evidence for a candidate tier checkpoint trained on the owner's
+own prompts, evaluated once on the owner-reviewed holdout. It is not live-host evidence.
+Issue [#233](https://github.com/dyphn1/Harness-everything/issues/233); the pipeline is in
+[system-one-training.md](../../../../docs/system-one-training.md). **Result: the checkpoint is
+not released.**
+
+## Data and labels
+
+The training data stays local and is not committed.
+
+- **Prompts:** 5,625 from the owner's Claude Code, Codex (interactive only), Copilot CLI and
+  VS Code Copilot Chat histories. That is 1,401 session families, split 4,898 train / 727
+  validation, and 93% of the prompts are zh-TW.
+- **Labels:** assigned under the refined rules (rules SHA-256 prefix `e153bd5ff747d6c9`).
+  - 3,300 by Claude Sonnet, which agreed with the holdout at 83.7%
+    ([check](../labeler-holdout-check-2026-09-24/report.json)). This run stopped when the
+    Claude session limit was reached.
+  - 2,325 by Codex `gpt-5.6-luna` in stateless per-batch calls, which agreed at 81.9%
+    ([check](../labeler-holdout-check-codex-2026-09-24/report.json)).
+- **Rejected labeling run:** two long-running Codex lane sessions were rejected. One labeled
+  with a keyword regex, and the long sessions consumed most of the Codex budget.
+
+## Training (CPU, Intel Core Ultra 5 225H, 4 torch threads)
+
+The model is a fresh `cua_s1` `tinyx` with width 128, 2 layers, 4 heads and `context_tokens` 1024. It was trained for 12 epochs at about 6 minutes each. The best validation NLL came at epoch 7:
+
+| Validation | Value |
+| --- | --- |
+| NLL | 0.911 |
+| Top-1 accuracy | 65.1% |
+
+Checkpoint SHA-256 values: weights `d0744d894eb3…`, config `c9833f3791eb…`.
+
+## Calibration on validation (727)
+
+| Min confidence | Coverage | Accepted precision |
+| --- | --- | --- |
+| 0.5 | 85.1% | 67.9% |
+| 0.7 | 55.8% | 71.7% |
+| 0.8 | 29.4% | 75.7% |
+| 0.9 | 3.7% | 81.5% |
+| 0.95 (chosen for ≥98%) | 0.1% | 100% (1 case) |
+
+Precision never approaches 98% at useful coverage.
+
+## Holdout (215 owner-reviewed cases, one run)
+
+[`report.json`](report.json) is the unmodified evaluator output. It was produced with the resident transport and the calibrated `acceptance` (0.95 / 0).
+
+| Metric | Value |
+| --- | --- |
+| Accepted | 0 of 215; every decision is `low-confidence` |
+| Raw top-1, ignoring thresholds | 35.3% |
+| Raw macro-F1 | 0.255 (the lexical baseline is 0.254) |
+| tier3 predicted correctly | 0 of 51 |
+| Warm p95 | 59.8 ms (the `warmLatency` gate passes) |
+| Repeatability | 1.0 |
+
+**Gates.** The `reviewedHoldout`, `repeatability`, `warmLatency` and `sourceProvenance` gates pass. The `acceptedPrecision`, `coverage`, `macroF1`, `policyEvidence` and `liveHostEvidence` gates fail. `rolloutReady` is false.
+
+## Reading
+
+- **Generalization gap.** Validation top-1 is 65%, which is in-distribution with the owner's
+  real session prompts. On the holdout it falls to 35%. The holdout consists of authored
+  prompts and de-identified rewrites with balanced classes.
+- **Label skew.** The training labels are about 61% tier2. The model mostly predicts tier2 or
+  unclassified and never tier3.
+- **Label noise.** The labels disagree with the owner's gold roughly 16–18% of the time, which
+  also caps accuracy.
+- **Model capacity.** A 4-way byte-level scorer trained from scratch on about 5k noisy
+  examples does not reach the rollout gates. Better class balance, more tier3 and tier1
+  examples, or a pretrained multilingual encoder are the likely next levers. None of them is
+  measured yet.