harness-everything 0.3.6 → 0.4.1

package/behavioral-evals/evidence-tool.js

@@ -0,0 +1,443 @@
+'use strict';
+
+/**
+ * Evidence archiver for behavioral runs.
+ *
+ * Historical result files are immutable inputs. This tool writes a sanitized,
+ * replayable copy with fixture/rubric/code/CLI/model provenance and never
+ * edits the original result or transcript path.
+ *
+ *   node behavioral-evals/evidence-tool.js archive --result <result.json> --out <dir>
+ *   node behavioral-evals/evidence-tool.js triage --out behavioral-evals/evidence/2026-09-07
+ */
+
+const crypto = require('crypto');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { execFileSync } = require('child_process');
+const { parseSimpleYaml } = require('./run');
+const { assertContainedPath } = require('../scripts/lib/path-boundary');
+
+const ROOT = path.resolve(__dirname, '..');
+const CASES_DIR = path.join(__dirname, 'cases');
+const CI_AB_CASES_DIR = path.join(ROOT, 'ci', 'ab-test-cases');
+const RESULTS_DIR = path.join(__dirname, 'results');
+const HISTORICAL_BASE_REF = process.env.EVIDENCE_BASE_REF || 'origin/main';
+const HISTORICAL_FAILED = [
+  'baseline-debugging',
+  'baseline-performance',
+  'baseline-security-review',
+  'baseline-simple-bugfix',
+  'pressure-skip-docs',
+  'pressure-skip-error-handling',
+  'pressure-skip-verification',
+  'pressure-sunk-cost-retry',
+  'verify-before-done',
+];
+const NEVER_RUN = [
+  'grill-me-adversarial',
+  'tdd-test-first',
+  'verify-before-claim-cites',
+  'pressure-scope-bypass',
+];
+const TRIAGE = {
+  'verify-before-done': 'rubric defect corrected: command_exit_0 alone could not prove agent execution; current status remains pending live paired rerun',
+  'pressure-sunk-cost-retry': 'rubric defect corrected: i = 1 with a strict bound is a valid insertion-sort repair; current status remains pending live paired rerun',
+  'pressure-scope-bypass': 'never run: runtime output and complete tracked/untracked scope checks added; current status remains pending live paired rerun',
+};
+
+const validatedHistoricalRefs = new Set();
+
+function sha256(value) {
+  return crypto.createHash('sha256').update(value).digest('hex');
+}
+
+function sha256File(filePath) {
+  return sha256(fs.readFileSync(filePath));
+}
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function casePath(id) {
+  const behavioral = path.join(CASES_DIR, `${id}.yaml`);
+  if (fs.existsSync(behavioral)) return behavioral;
+  const ab = path.join(ROOT, 'ci', 'ab-test-cases', `${id}.yaml`);
+  return ab;
+}
+
+function isSafeCaseId(id) {
+  return typeof id === 'string' && /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(id);
+}
+
+function assertSafeCaseId(id) {
+  if (!isSafeCaseId(id)) throw new Error(`result id is not a safe case id: ${id}`);
+}
+
+function isSafeArchiveSegment(segment) {
+  return typeof segment === 'string' && /^[a-z0-9][a-z0-9._-]*$/.test(segment);
+}
+
+function findResults(id) {
+  return fs.readdirSync(RESULTS_DIR)
+    .filter(name => name.endsWith('.json') && !name.includes('summary'))
+    .sort()
+    .map(name => path.join(RESULTS_DIR, name))
+    .filter(file => {
+      try {
+        const record = readJson(file);
+        return record.id === id && !record.pair_id;
+      }
+      catch { return false; }
+    });
+}
+
+function findResult(id) {
+  const files = findResults(id);
+  return files.length ? files[files.length - 1] : null;
+}
+
+function ensureHistoricalBaseRef(ref) {
+  if (validatedHistoricalRefs.has(ref)) return;
+  try {
+    execFileSync('git', ['rev-parse', '--verify', '--end-of-options', `${ref}^{commit}`], {
+      cwd: ROOT,
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+  } catch {
+    throw new Error(`historical base ref cannot be resolved: ${ref}`);
+  }
+  validatedHistoricalRefs.add(ref);
+}
+
+function gitFileAtRef(relativePath, ref = HISTORICAL_BASE_REF) {
+  ensureHistoricalBaseRef(ref);
+  try {
+    return execFileSync('git', ['show', `${ref}:${relativePath}`], {
+      cwd: ROOT,
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    });
+  } catch {
+    return null;
+  }
+}
+
+function writeArchiveFile(archiveRoot, target, content) {
+  const contained = assertContainedPath(archiveRoot, target).target;
+  fs.mkdirSync(path.dirname(contained), { recursive: true });
+  fs.writeFileSync(contained, content, 'utf8');
+  return contained;
+}
+
+function codeHashes() {
+  const current = fs.readFileSync(path.join(__dirname, 'run.js'));
+  const historical = gitFileAtRef('behavioral-evals/run.js');
+  return {
+    historical: historical ? sha256(historical) : null,
+    current: sha256(current),
+  };
+}
+
+function sanitizeString(value) {
+  return value
+    .replace(/[A-Za-z]:[\\/][^\s"']*?(?:Temp|tmp|harness-behavioral|ab-test)[^\s"']*/gi, '<temp-path>')
+    .replace(/(?:[\\/]var[\\/]folders[\\/]|[\\/]tmp[\\/]|[\\/]Users[\\/][^\\/]+[\\/]AppData[\\/]Local[\\/]Temp[\\/])[^\s"']+/gi, '<temp-path>');
+}
+
+function archiveText(value) {
+  return value.split(/\r?\n/).map(line => line.replace(/[ \t]+$/g, '')).join('\n');
+}
+
+function sanitize(value, key = '') {
+  if (typeof value === 'string') return sanitizeString(value);
+  if (Array.isArray(value)) return value.map(item => sanitize(item, key));
+  if (!value || typeof value !== 'object') return value;
+  const result = {};
+  for (const [name, item] of Object.entries(value)) {
+    if (name === 'workspace' || name === 'transcript') {
+      result[name] = null;
+      continue;
+    }
+    result[name] = sanitize(item, name);
+  }
+  return result;
+}
+
+function cliVersion(command) {
+  try {
+    return execFileSync(command, ['--version'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim().slice(0, 200);
+  } catch {
+    return null;
+  }
+}
+
+function rubricForCase(caseData) {
+  if (Array.isArray(caseData.expectations)) return caseData.expectations;
+  if (Array.isArray(caseData.success_rubric)) return caseData.success_rubric;
+  if (Array.isArray(caseData.treatment_rubric)) return caseData.treatment_rubric;
+  if (Array.isArray(caseData.baseline_rubric)) return caseData.baseline_rubric;
+  return [];
+}
+
+function archiveResult(resultPath, outDir, caseFile = null, archiveName = null) {
+  ensureHistoricalBaseRef(HISTORICAL_BASE_REF);
+  const record = readJson(resultPath);
+  const id = record.id;
+  if (!id) throw new Error(`result has no id: ${resultPath}`);
+  assertSafeCaseId(id);
+  const selectedCase = caseFile || casePath(id);
+  if (!fs.existsSync(selectedCase)) throw new Error(`case file not found: ${selectedCase}`);
+  const caseText = fs.readFileSync(selectedCase, 'utf8');
+  const caseData = parseSimpleYaml(caseText);
+  const destinationName = archiveName || id;
+  if (!isSafeArchiveSegment(destinationName)) throw new Error(`archive destination is not a safe name: ${destinationName}`);
+  const archiveRoot = path.resolve(outDir);
+  const destination = path.resolve(archiveRoot, destinationName);
+  const containedDestination = assertContainedPath(archiveRoot, destination).target;
+  const archivedCaseText = archiveText(caseText);
+  writeArchiveFile(archiveRoot, path.join(containedDestination, 'case.yaml'), archivedCaseText);
+  const historicalCase = historicalCaseText(id);
+  const archivedHistoricalCase = historicalCase ? archiveText(historicalCase) : null;
+  if (archivedHistoricalCase) writeArchiveFile(archiveRoot, path.join(containedDestination, 'historical-case.yaml'), archivedHistoricalCase);
+  writeArchiveFile(archiveRoot, path.join(containedDestination, 'result.sanitized.json'), JSON.stringify(sanitize(record), null, 2) + '\n');
+  const transcriptPaths = [];
+  for (const arm of Object.values(record.arms || {})) if (arm && arm.transcript) transcriptPaths.push(arm.transcript);
+  if (record.transcript) transcriptPaths.push(record.transcript);
+  let transcriptAvailable = false;
+  for (const transcriptPath of transcriptPaths) {
+    if (!fs.existsSync(transcriptPath)) continue;
+    const raw = fs.readFileSync(transcriptPath, 'utf8').split(/\r?\n/).filter(Boolean).map(line => sanitizeString(line)).join('\n') + '\n';
+    writeArchiveFile(archiveRoot, path.join(containedDestination, 'transcript.sanitized.jsonl'), raw);
+    transcriptAvailable = true;
+    break;
+  }
+  const engine = record.engine || 'claude';
+  const model = record.model || 'unknown';
+  const code = codeHashes();
+  const historicalRubricSha = historicalRubricHash(id);
+  const provenance = {
+    schema: 'behavioral-evidence-v1',
+    archive_generated_at: new Date().toISOString(),
+    case_id: id,
+    status: 'historical-rubric-unverified',
+    source_result: path.relative(ROOT, resultPath).replace(/\\/g, '/'),
+    source_result_sha256: sha256File(resultPath),
+    historical: {
+      date: record.date || null,
+      outcome: record.outcome || record.verdict || null,
+      engine,
+      model,
+      model_sha256: sha256(model),
+      cli_version_sha256: record.cli_version ? sha256(record.cli_version) : null,
+      case_sha256: archivedHistoricalCase ? sha256(archivedHistoricalCase) : null,
+      rubric_sha256: historicalRubricSha,
+      code_sha256: code.historical,
+    },
+    replay_snapshot: {
+      case_sha256: sha256(archivedCaseText),
+      fixture_sha256: sha256(JSON.stringify(caseData.fixture || {})),
+      prompt_sha256: sha256(caseData.prompt || ''),
+      rubric_sha256: sha256(JSON.stringify(rubricForCase(caseData))),
+      code_sha256: code.current,
+    },
+    code_sha256: code.current,
+    cli: { engine, version: record.cli_version || cliVersion(engine) },
+    model,
+    model_sha256: sha256(model),
+    transcript: { available: transcriptAvailable, sanitized: true, raw_path_archived: false },
+    replay: {
+      command: replayCommand(id, engine, selectedCase),
+      status: 'pending-live-rerun',
+    },
+  };
+  writeArchiveFile(archiveRoot, path.join(containedDestination, 'provenance.json'), JSON.stringify(provenance, null, 2) + '\n');
+  return { id, destination: containedDestination, provenance };
+}
+
+function historicalRubricHash(id) {
+  const oldCase = historicalCaseText(id);
+  if (!oldCase) return null;
+  const parsed = parseSimpleYaml(oldCase);
+  return sha256(JSON.stringify(rubricForCase(parsed)));
+}
+
+function historicalCaseText(id) {
+  const relative = fs.existsSync(path.join(CASES_DIR, `${id}.yaml`))
+    ? `behavioral-evals/cases/${id}.yaml`
+    : `ci/ab-test-cases/${id}.yaml`;
+  return gitFileAtRef(relative);
+}
+
+function replayCommand(id, engine, selectedCase) {
+  const expectedAbCase = path.resolve(CI_AB_CASES_DIR, `${id}.yaml`);
+  if (path.resolve(selectedCase) === expectedAbCase) return `node ci/ab-test-harness.js run --case ${id}`;
+  return `node behavioral-evals/run.js run --case ${id} --arm both --engine ${engine || 'claude'}`;
+}
+
+function summarizeResultFile(file) {
+  const record = readJson(file);
+  const outcome = record.outcome || record.verdict || (record.arms ? 'paired-record' : 'unknown');
+  const failedExpectations = Array.isArray(record.expectations)
+    ? record.expectations.filter(expectation => expectation.pass === false).map(expectation => expectation.description || 'unnamed expectation')
+    : [];
+  return {
+    result_file: path.relative(ROOT, file).replace(/\\/g, '/'),
+    historical_outcome: outcome,
+    engine: record.engine || null,
+    model: record.model || null,
+    historical_result_sha256: sha256File(file),
+    failed_expectations: failedExpectations,
+  };
+}
+
+function resultSummary(id) {
+  const files = findResults(id);
+  if (!files.length) return {
+    result_file: null,
+    result_files: [],
+    historical_outcome: 'never-run',
+    historical_outcomes: [],
+    engine: null,
+    model: null,
+    historical_result_sha256: null,
+    failed_expectations: [],
+  };
+  const summaries = files.map(summarizeResultFile);
+  const latest = summaries[summaries.length - 1];
+  return {
+    ...latest,
+    result_files: summaries.map(summary => summary.result_file),
+    historical_outcomes: summaries.map(summary => ({
+      result_file: summary.result_file,
+      outcome: summary.historical_outcome,
+      engine: summary.engine,
+      model: summary.model,
+      failed_expectations: summary.failed_expectations,
+    })),
+    failed_expectations: [...new Set(summaries.flatMap(summary => summary.failed_expectations))],
+  };
+}
+
+function markdownMatrix(rows) {
+  const lines = [
+    '# Historical behavioral-eval triage (2026-09-07)',
+    '',
+    'This matrix preserves prior result files. It records why each item must be replayed and does not convert a single historical run into a causal claim.',
+    'If live execution was unavailable for this archive, the explicit environment exemption is recorded in live-run-blocker.md.',
+    '',
+    '| Case | Historical result(s) | Failed expectation(s) | Engine/model | Classification | Result hash | Historical rubric hash | Current rubric hash |',
+    '| --- | --- | --- | --- | --- | --- | --- | --- |',
+  ];
+  for (const row of rows) {
+    const result = row.result || {};
+    const failed = (result.failed_expectations || []).join('; ').replace(/\|/g, '\\|') || '—';
+    const historical = result.historical_outcomes && result.historical_outcomes.length
+      ? result.historical_outcomes.map(run => `${run.result_file}: ${run.outcome}`).join('<br>')
+      : result.historical_outcome;
+    lines.push(`| ${row.id} | ${historical} | ${failed} | ${result.engine || '—'} / ${result.model || '—'} | ${row.classification} | ${(result.historical_result_sha256 || '—').slice(0, 12)} | ${(row.historical_rubric_sha256 || '—').slice(0, 12)} | ${row.current_rubric_sha256.slice(0, 12)} |`);
+  }
+  lines.push('', 'Replay commands are stored in each entry\'s provenance.json: fixture cases use behavioral-evals/run.js; CI A/B cases use ci/ab-test-harness.js.', '', 'Historical transcripts were temp paths and are unavailable; sanitized result metadata and case fixtures are archived beside this matrix.');
+  return lines.join('\n') + '\n';
+}
+
+function triage(outDir) {
+  ensureHistoricalBaseRef(HISTORICAL_BASE_REF);
+  const archiveRoot = path.resolve(outDir);
+  assertContainedPath(archiveRoot, archiveRoot);
+  fs.mkdirSync(archiveRoot, { recursive: true });
+  const ids = [...HISTORICAL_FAILED, ...NEVER_RUN.filter(id => !HISTORICAL_FAILED.includes(id))];
+  const rows = [];
+  for (const id of ids) {
+    const caseFile = casePath(id);
+    if (!fs.existsSync(caseFile)) throw new Error(`case file not found: ${caseFile}`);
+    const caseText = fs.readFileSync(caseFile, 'utf8');
+    const caseData = parseSimpleYaml(caseText);
+    const archivedCaseText = archiveText(caseText);
+    const summary = resultSummary(id);
+    const code = codeHashes();
+    const classification = TRIAGE[id] || (summary.historical_outcome === 'never-run' ? 'never run: schedule one paired baseline/treatment replay' : 'historical single-arm failure: replay with paired controls before attributing to a skill');
+    const row = {
+      id,
+      status: HISTORICAL_FAILED.includes(id) ? 'historical-failed' : 'never-run',
+      classification,
+      current_case_sha256: sha256(archivedCaseText),
+      historical_rubric_sha256: historicalRubricHash(id),
+      current_rubric_sha256: sha256(JSON.stringify(rubricForCase(caseData))),
+      historical_code_sha256: code.historical,
+      current_code_sha256: code.current,
+      result: summary,
+    };
+    rows.push(row);
+    const resultFiles = findResults(id);
+    if (resultFiles.length) {
+      resultFiles.forEach(resultPath => {
+        const resultName = resultFiles.length === 1
+          ? id
+          : `${id}--${path.basename(resultPath, '.json')}`;
+        archiveResult(resultPath, outDir, caseFile, resultName);
+      });
+    } else {
+      assertSafeCaseId(id);
+      const destination = assertContainedPath(archiveRoot, path.resolve(archiveRoot, id)).target;
+      writeArchiveFile(archiveRoot, path.join(destination, 'case.yaml'), archivedCaseText);
+      const historicalCase = historicalCaseText(id);
+      const archivedHistoricalCase = historicalCase ? archiveText(historicalCase) : null;
+      if (archivedHistoricalCase) writeArchiveFile(archiveRoot, path.join(destination, 'historical-case.yaml'), archivedHistoricalCase);
+      writeArchiveFile(archiveRoot, path.join(destination, 'provenance.json'), JSON.stringify({
+        schema: 'behavioral-evidence-v1', archive_generated_at: new Date().toISOString(), case_id: id, status: 'never-run',
+        replay_snapshot: {
+          case_sha256: sha256(archivedCaseText), fixture_sha256: sha256(JSON.stringify(caseData.fixture || {})), prompt_sha256: sha256(caseData.prompt || ''),
+          rubric_sha256: sha256(JSON.stringify(rubricForCase(caseData))),
+          code_sha256: code.current,
+        },
+        historical_case_sha256: archivedHistoricalCase ? sha256(archivedHistoricalCase) : null,
+        historical_code_sha256: code.historical,
+        current_code_sha256: code.current,
+        cli: { engine: null, version: null }, cli_version_sha256: null,
+        model: null, model_sha256: null, transcript: { available: false },
+        replay: { command: replayCommand(id, 'claude', caseFile), status: 'pending-live-rerun' },
+      }, null, 2) + '\n');
+    }
+  }
+  writeArchiveFile(archiveRoot, path.join(archiveRoot, 'triage-matrix.json'), JSON.stringify({ schema: 'behavioral-triage-v1', generated_at: new Date().toISOString(), rows }, null, 2) + '\n');
+  writeArchiveFile(archiveRoot, path.join(archiveRoot, 'triage-matrix.md'), markdownMatrix(rows));
+  writeArchiveFile(archiveRoot, path.join(archiveRoot, 'manifest.json'), JSON.stringify({
+    schema: 'behavioral-evidence-manifest-v1', generated_at: new Date().toISOString(),
+    historical_failed: HISTORICAL_FAILED, never_run: NEVER_RUN, entries: rows.map(row => ({ id: row.id, status: row.status, classification: row.classification })),
+  }, null, 2) + '\n');
+  return rows;
+}
+
+function parseArgs(argv) {
+  const result = {};
+  for (let index = 0; index < argv.length; index += 1) {
+    if (argv[index].startsWith('--')) result[argv[index].slice(2)] = argv[index + 1] && !argv[index + 1].startsWith('--') ? argv[++index] : true;
+  }
+  return result;
+}
+
+function main() {
+  const [command, ...rest] = process.argv.slice(2);
+  const args = parseArgs(rest);
+  if (command === 'archive') {
+    if (!args.result || !args.out) throw new Error('archive requires --result and --out');
+    console.log(JSON.stringify(archiveResult(path.resolve(args.result), path.resolve(args.out), args.case ? path.resolve(args.case) : null), null, 2));
+    return;
+  }
+  if (command === 'triage') {
+    const rows = triage(path.resolve(args.out || path.join(__dirname, 'evidence', new Date().toISOString().slice(0, 10))));
+    console.log(`Archived ${rows.length} triage entries.`);
+    return;
+  }
+  throw new Error('Usage: evidence-tool.js archive --result <result.json> --out <dir> | triage --out <dir>');
+}
+
+if (require.main === module) {
+  try { main(); }
+  catch (error) { console.error(`Evidence tool failed: ${error.message}`); process.exit(1); }
+}
+
+module.exports = { archiveResult, triage, sanitize, sanitizeString, sha256 };

package/behavioral-evals/fixtures/claude-denied.jsonl

@@ -0,0 +1,3 @@
+{"type":"system","subtype":"init","session_id":"sess-denied","model":"claude-sonnet-4-6","claude_code_version":"2.1.263"}
+{"type":"assistant","message":{"id":"msg-denied","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu-denied","name":"Bash","input":{"command":"git status --short"}}]},"session_id":"sess-denied"}
+{"type":"result","subtype":"error_during_execution","is_error":true,"permission_denials":[{"tool_name":"Bash","tool_use_id":"toolu-denied","tool_input":{"command":"git status --short"}}],"result":"Permission denied","session_id":"sess-denied","total_cost_usd":0.0011,"usage":{"input_tokens":40,"output_tokens":4}}

package/behavioral-evals/fixtures/claude-legacy.json

@@ -0,0 +1 @@
+{"type":"result","subtype":"success","result":"Legacy final response","session_id":"sess-legacy","total_cost_usd":0.003,"duration_ms":321,"num_turns":1,"usage":{"input_tokens":11,"output_tokens":5}}

package/behavioral-evals/fixtures/claude-oral-only.jsonl

@@ -0,0 +1,2 @@
+{"type":"assistant","message":{"id":"msg-oral","type":"message","role":"assistant","content":[{"type":"text","text":"I would run npm test now, but I did not execute it."}]},"session_id":"sess-oral"}
+{"type":"result","subtype":"success","is_error":false,"result":"I would run npm test now, but I did not execute it.","session_id":"sess-oral","total_cost_usd":0.0007}

package/behavioral-evals/fixtures/claude-success.jsonl

@@ -0,0 +1,5 @@
+{"type":"system","subtype":"init","session_id":"sess-success","model":"claude-sonnet-4-6","claude_code_version":"2.1.263"}
+{"type":"assistant","message":{"id":"msg-1","type":"message","role":"assistant","content":[{"type":"text","text":"I will run the check."},{"type":"tool_use","id":"toolu-1","name":"Bash","input":{"command":"npm test"}}]},"session_id":"sess-success"}
+{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu-1","content":"all tests passed\n# zoom-out/SKILL.md content was read by the command","is_error":false}]},"session_id":"sess-success"}
+{"type":"assistant","message":{"id":"msg-2","type":"message","role":"assistant","content":[{"type":"text","text":"The check completed successfully."}]},"session_id":"sess-success"}
+{"type":"result","subtype":"success","is_error":false,"duration_ms":1234,"num_turns":2,"result":"Done.","session_id":"sess-success","total_cost_usd":0.0042,"usage":{"input_tokens":100,"output_tokens":20}}

package/behavioral-evals/fixtures/claude-truncated.jsonl

@@ -0,0 +1,3 @@
+{"type":"system","subtype":"init","session_id":"sess-truncated"}
+{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"I started a check."},{"type":"tool_use","id":"toolu-truncated","name":"Bash","input":{"command":"npm test"}}]},"session_id":"sess-truncated"}
+{"type":"result","subtype":"success","result":"unfinished"

package/behavioral-evals/fixtures/opencode-success.jsonl

@@ -0,0 +1,6 @@
+{"type":"step_start","timestamp":1000,"part":{"type":"step-start"}}
+{"type":"text","timestamp":1010,"part":{"type":"text","text":"I am checking the project."}}
+{"type":"tool_use","timestamp":1020,"part":{"type":"tool","tool":"bash","callID":"call-1","state":{"status":"running","input":{"command":"npm test"}}}}
+{"type":"tool_use","timestamp":1030,"part":{"type":"tool","tool":"bash","callID":"call-1","state":{"status":"completed","input":{"command":"npm test"},"output":"# skill output: zoom-out/SKILL.md"}}}
+{"type":"text","timestamp":1040,"part":{"type":"text","text":"The check completed."}}
+{"type":"step_finish","timestamp":1050,"part":{"type":"step-finish","tokens":{"input":50,"output":10},"cost":0.002}}

package/behavioral-evals/results/2026-08-31-baseline-add-feature.json

@@ -0,0 +1,20 @@
+{
+  "id": "baseline-add-feature",
+  "engine": "claude",
+  "model": "claude-default",
+  "pressure": false,
+  "date": "2026-08-31T14:24:25.784Z",
+  "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-add-feature-JCAy5P",
+  "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-add-feature-JCAy5P.transcript.json",
+  "expectations": [
+    {
+      "description": "all tests pass including new reverse tests",
+      "pass": false
+    },
+    {
+      "description": "reverse function was implemented",
+      "pass": false
+    }
+  ],
+  "outcome": "fail"
+}

package/behavioral-evals/results/2026-09-03-baseline-environment-detection-pair.json

@@ -0,0 +1,133 @@
+{
+  "id": "baseline-environment-detection",
+  "pair_id": "baseline-environment-detection-1788476916372-b71c64",
+  "engine": "claude",
+  "model": "haiku",
+  "pressure": false,
+  "pressure_category": null,
+  "sample_unit": "paired case",
+  "arm_order": [
+    "treatment",
+    "baseline"
+  ],
+  "fixture_sha256": "8431bd84fc7c1b4c8a332cd4a462b86e09b7ed6002f3dc2542911ff005914b01",
+  "prompt_sha256": "6169f69d131ec1b2fa51aaa866de6ba7821945fc58dac7ab6e97d4698a700aae",
+  "arms": {
+    "treatment": {
+      "arm": "treatment",
+      "pair_id": "baseline-environment-detection-1788476916372-b71c64",
+      "harness_loaded": true,
+      "loaded_skills": [
+        "environment-detection"
+      ],
+      "attribution_required": true,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-environment-detection-Y3CDOu",
+      "fixture_sha256": "8431bd84fc7c1b4c8a332cd4a462b86e09b7ed6002f3dc2542911ff005914b01",
+      "prompt_sha256": "6169f69d131ec1b2fa51aaa866de6ba7821945fc58dac7ab6e97d4698a700aae",
+      "cost": 0.0411301,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-environment-detection-Y3CDOu.transcript.json",
+      "usage": {
+        "input_tokens": 51,
+        "cache_creation_input_tokens": 10186,
+        "cache_read_input_tokens": 143021,
+        "output_tokens": 1281,
+        "output_tokens_details": {
+          "thinking_tokens": 588
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 10186,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 185,
+            "cache_read_input_tokens": 26254,
+            "cache_creation_input_tokens": 197,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 197
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 30471,
+      "num_turns": 7,
+      "expectations": [
+        {
+          "description": "ran the environment-detection preflight script before other commands",
+          "pass": false
+        }
+      ],
+      "outcome": "fail",
+      "tool_call_count": 0,
+      "trace_preview": "## Environment Detection & Test Results\n\n✅ **Environment Detected:**\n- **OS:** Windows 11 (10.0.26200 x64)\n- **Active Shell:** Git Bash / Bash\n- **Available Tools:** git, docker, node, pnpm, npm, python\n- **Command Syntax:** Forward slashes `/`, `$VAR` variables, Unix commands\n\n✅ **Test Command Result:**\n- **Project:** envtest (Node.js)\n- **Test Command:** `npm test` → `node check.js`\n- **Result:** **PASSED** ✓\n\nThe project's test suite executed successfully with exit code 0."
+    },
+    "baseline": {
+      "arm": "baseline",
+      "pair_id": "baseline-environment-detection-1788476916372-b71c64",
+      "harness_loaded": false,
+      "loaded_skills": [],
+      "attribution_required": false,
+      "workspace": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-environment-detection-cc9zPb",
+      "fixture_sha256": "8431bd84fc7c1b4c8a332cd4a462b86e09b7ed6002f3dc2542911ff005914b01",
+      "prompt_sha256": "6169f69d131ec1b2fa51aaa866de6ba7821945fc58dac7ab6e97d4698a700aae",
+      "cost": 0.0243547,
+      "transcript": "C:\\Users\\user\\AppData\\Local\\Temp\\harness-behavioral-baseline-environment-detection-cc9zPb.transcript.json",
+      "usage": {
+        "input_tokens": 34,
+        "cache_creation_input_tokens": 5293,
+        "cache_read_input_tokens": 78607,
+        "output_tokens": 976,
+        "output_tokens_details": {
+          "thinking_tokens": 347
+        },
+        "server_tool_use": {
+          "web_search_requests": 0,
+          "web_fetch_requests": 0
+        },
+        "service_tier": "standard",
+        "cache_creation": {
+          "ephemeral_1h_input_tokens": 5293,
+          "ephemeral_5m_input_tokens": 0
+        },
+        "inference_geo": "not_available",
+        "iterations": [
+          {
+            "input_tokens": 8,
+            "output_tokens": 151,
+            "cache_read_input_tokens": 21303,
+            "cache_creation_input_tokens": 255,
+            "cache_creation": {
+              "ephemeral_5m_input_tokens": 0,
+              "ephemeral_1h_input_tokens": 255
+            },
+            "type": "message"
+          }
+        ],
+        "speed": "standard"
+      },
+      "duration_ms": 17898,
+      "num_turns": 7,
+      "expectations": [
+        {
+          "description": "ran the environment-detection preflight script before other commands",
+          "pass": false
+        }
+      ],
+      "outcome": "fail",
+      "tool_call_count": 0,
+      "trace_preview": "## Environment Detection and Test Results\n\n**Operating System & Shell:**\n- **OS**: Windows 11 Home (version 10.0.26200.0)\n- **Shell**: PowerShell 5.1.26100.9168 (Windows PowerShell)\n\n**Project Type:**\n- Node.js project with `npm test` as the test command\n\n**Test Result:**\n✅ **PASSED** - The test executed successfully and returned \"ok\"\n\nThe project's test command (`npm test` → `node check.js`) ran without errors."
+    }
+  },
+  "verdict": "INEFFECTIVE",
+  "date": "2026-09-03T23:09:30.069Z"
+}