harness-everything 0.47.3 → 0.47.5

package/ci/test-behavioral-case-validation.js

@@ -5,8 +5,8 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const { parseSimpleYaml } = require('../behavioral-evals/run');
-const { grade, parseTranscriptEvents } = require('../behavioral-evals/run');
-const { validateCase, validateDirectory, negativeControls } = require('../behavioral-evals/case-validator');
+const { grade, parseTranscriptEvents, checkJsonOnlyOutput, finalAssistantText } = require('../behavioral-evals/run');
+const { validateCase, validateDirectory, validateExpectation, negativeControls } = require('../behavioral-evals/case-validator');
 const { canonicalRubric, extractTrace } = require('./ab-test-harness');
 
 const casesDir = path.join(__dirname, '..', 'behavioral-evals', 'cases');
@@ -85,4 +85,363 @@ try {
   fs.rmSync(transcriptDir, { recursive: true, force: true });
 }
 
-console.log(`Behavioral case validation passed (${fs.readdirSync(casesDir).filter(name => name.endsWith('.yaml')).length} cases + negative controls + ordered execution evidence).`);
+// final_output_json_only: pure checker, validator schema, shipped #308 case, and
+// end-to-end grading over the final assistant message.
+const SELECTION_KEYS = ['requestedProfile', 'effectiveProfile', 'profileAlias', 'assignedRole', 'runtimeModel', 'runtimeEffort'];
+const SELECTION = { requestedProfile: 'fable', effectiveProfile: 'sonnet', profileAlias: 'fable-on-sonnet', assignedRole: 'worker', runtimeModel: 'claude-haiku-5-5', runtimeEffort: 'low' };
+const LABEL = '<!-- harness-label {"judgement":"done","routing":"direct"} -->';
+const RAW = JSON.stringify(SELECTION);
+const FENCED = '```json\n' + JSON.stringify(SELECTION, null, 2) + '\n```';
+const PASS_CASES = [
+  ['raw JSON object', RAW],
+  ['fenced json block', FENCED],
+  ['untagged fence', '```\n' + RAW + '\n```'],
+  ['raw JSON followed by harness-label line', RAW + '\n' + LABEL],
+  ['fenced JSON followed by harness-label line', FENCED + '\n' + LABEL],
+];
+for (const [name, text] of PASS_CASES) {
+  const verdict = checkJsonOnlyOutput(text, SELECTION_KEYS);
+  assert.strictEqual(verdict.pass, true, `${name} should pass: ${verdict.reason}`);
+}
+const FAIL_CASES = [
+  // The #308 pilot: correct fields, explanatory prose outside the JSON.
+  ['prose before JSON (#308)', `"Fable on sonnect" means the fable profile runs on sonnet.\n\n${FENCED}`, /prose before/],
+  ['prose after fence', `${FENCED}\nThat is the selection record.`, /outside the single JSON code fence/],
+  ['prose after raw JSON', `${RAW}\nThat is the selection record.`, /not a single valid JSON value/],
+  ['two fenced blocks', `${FENCED}\n${FENCED}`, /more than one code fence/],
+  ['missing required key', JSON.stringify({ ...SELECTION, runtimeEffort: undefined }), /missing required JSON keys: runtimeEffort/],
+  ['non-object JSON (array)', JSON.stringify([SELECTION]), /must be an object/],
+  ['non-object JSON (scalar)', '"fable"', /must be an object/],
+  ['empty final text', '', /empty/],
+  ['label-only final text', LABEL, /empty/],
+  ['missing final text', undefined, /missing/],
+];
+for (const [name, text, expected] of FAIL_CASES) {
+  const verdict = checkJsonOnlyOutput(text, SELECTION_KEYS);
+  assert.strictEqual(verdict.pass, false, `${name} should fail`);
+  assert.match(verdict.reason, expected, `${name} should give a clear reason`);
+}
+assert.strictEqual(checkJsonOnlyOutput(RAW).pass, true, 'required_keys defaults to no key requirement');
+
+for (const requiredKeys of ['requestedProfile', [''], ['  '], [1], ['ok', null]]) {
+  const errors = validateExpectation({ type: 'final_output_json_only', required_keys: requiredKeys }, 0);
+  assert.ok(errors.some(error => /required_keys must be an array of non-empty strings/.test(error)), `required_keys ${JSON.stringify(requiredKeys)} should be rejected`);
+}
+assert.deepStrictEqual(validateExpectation({ type: 'final_output_json_only' }, 0), [], 'final_output_json_only without required_keys is valid');
+assert.deepStrictEqual(validateExpectation({ type: 'final_output_json_only', required_keys: SELECTION_KEYS }, 0), [], 'final_output_json_only with string keys is valid');
+
+const profileCasePath = path.join(casesDir, 'baseline-profile-lookup-json-only.yaml');
+const profileCase = parseSimpleYaml(fs.readFileSync(profileCasePath, 'utf8'));
+assert.deepStrictEqual(validateCase(profileCase, { fileName: 'baseline-profile-lookup-json-only.yaml' }), [], 'profile-lookup JSON-only case should validate');
+assert.deepStrictEqual(profileCase.expectations.map(expectation => expectation.type), ['final_output_json_only'], 'profile-lookup case should assert JSON-only output');
+assert.deepStrictEqual(profileCase.expectations[0].required_keys, SELECTION_KEYS, 'profile-lookup case should require the six selection keys');
+assert.match(profileCase.prompt, /return only a small JSON selection record with requestedProfile, effectiveProfile, profileAlias, assignedRole, runtimeModel, runtimeEffort\. No orchestration or delegation\./, 'profile-lookup prompt should match the #308 wording');
+
+const jsonOnlyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'behavioral-json-only-'));
+try {
+  const writeTranscript = (name, finalText) => {
+    const transcriptPath = path.join(jsonOnlyDir, name);
+    const events = [
+      { type: 'tool_use', part: { type: 'tool', tool: 'Read', callID: 'read-1', state: { status: 'completed', input: { file_path: 'README.md' } } } },
+      { type: 'text', part: { type: 'text', text: 'Checking the profile table first.' } },
+      { type: 'tool_use', part: { type: 'tool', tool: 'Read', callID: 'read-2', state: { status: 'completed', input: { file_path: 'profiles.md' } } } },
+      { type: 'text', part: { type: 'text', text: finalText } },
+    ];
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return transcriptPath;
+  };
+  const jsonOnlyExpectation = { expectations: [{ type: 'final_output_json_only', required_keys: SELECTION_KEYS }] };
+  let transcriptCount = 0;
+  const gradeFinal = finalText => grade(jsonOnlyExpectation, jsonOnlyDir, writeTranscript(`final-${transcriptCount++}.jsonl`, finalText));
+  assert.strictEqual(gradeFinal(RAW).passed, true, 'raw JSON final message should pass end to end');
+  assert.strictEqual(gradeFinal(`${FENCED}\n${LABEL}`).passed, true, 'fenced JSON with harness-label should pass end to end');
+  const prose = gradeFinal(`Fable on sonnect means a profile.\n${FENCED}`);
+  assert.strictEqual(prose.passed, false, 'prose before JSON must fail end to end');
+  assert.strictEqual(prose.results[0].status, 'fail');
+  assert.match(prose.results[0].reason, /prose before/);
+  // Prose from an earlier turn (before the last tool call) is not the final message.
+  const earlierProse = writeTranscript('earlier-prose.jsonl', RAW);
+  assert.strictEqual(grade(jsonOnlyExpectation, jsonOnlyDir, earlierProse).passed, true, 'only the final assistant message is graded');
+
+  // Multi-part final messages: the final turn may stream several text parts.
+  const writeParts = (name, finalParts) => {
+    const transcriptPath = path.join(jsonOnlyDir, name);
+    const events = [
+      { type: 'tool_use', part: { type: 'tool', tool: 'Read', callID: 'read-1', state: { status: 'completed', input: { file_path: 'profiles.md' } } } },
+      ...finalParts.map(text => ({ type: 'text', part: { type: 'text', text } })),
+    ];
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return transcriptPath;
+  };
+  const proseThenJson = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts('prose-part-json-part.jsonl', ['"Fable on sonnect" means the fable profile on sonnet.', FENCED]));
+  assert.strictEqual(proseThenJson.passed, false, 'prose part + JSON part must fail (#308 false pass)');
+  assert.match(proseThenJson.results[0].reason, /prose before/);
+  const jsonThenProse = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts('json-part-prose-part.jsonl', [FENCED, 'That is the selection record.']));
+  assert.strictEqual(jsonThenProse.passed, false, 'JSON part + prose part must fail');
+  const splitBody = JSON.stringify(SELECTION, null, 2);
+  const cut = splitBody.indexOf('"assignedRole"');
+  const splitJson = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts('split-json.jsonl', ['```json\n' + splitBody.slice(0, cut), splitBody.slice(cut) + '\n```']));
+  assert.strictEqual(splitJson.passed, true, `JSON split across text parts should pass: ${splitJson.results[0].reason}`);
+
+  // Transport chunks cut inside a JSON string value, a key, and the fence opener.
+  // Exact concatenation must rebuild the message; a '\n' join would break it.
+  const valueCut = RAW.indexOf('fable') + 2;
+  const keyCut = RAW.indexOf('requestedProfile') + 5;
+  const splitValueParts = [RAW.slice(0, valueCut), RAW.slice(valueCut)];
+  const splitValuePath = writeParts('split-inside-value.jsonl', splitValueParts);
+  assert.strictEqual(finalAssistantText(parseTranscriptEvents(splitValuePath).events), RAW, 'split-inside-value parts must concatenate to the exact raw JSON');
+  assert.throws(() => JSON.parse(RAW.slice(0, valueCut) + '\n' + RAW.slice(valueCut)), 'a newline join inside a JSON string must not parse (the #308 reviewer case)');
+  const POSITIVE_SPLITS = [
+    ['split inside a JSON string value', splitValueParts],
+    ['split inside a JSON key', [RAW.slice(0, keyCut), RAW.slice(keyCut)]],
+    ['split inside the fence opener', ['```js', 'on\n' + JSON.stringify(SELECTION, null, 2) + '\n```']],
+    ['fenced JSON block followed by a harness-label part', [FENCED, LABEL]],
+  ];
+  POSITIVE_SPLITS.forEach(([name, parts], index) => {
+    const result = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts(`positive-split-${index}.jsonl`, parts));
+    assert.strictEqual(result.passed, true, `${name} should pass end to end: ${result.results[0].reason}`);
+  });
+  const NEGATIVE_SPLITS = [
+    ['prose chunk glued to JSON start', ['Here:', RAW], /prose before/],
+    ['prose after a split JSON', [RAW.slice(0, 10), RAW.slice(10), ' done'], /not a single valid JSON value/],
+  ];
+  NEGATIVE_SPLITS.forEach(([name, parts, expected], index) => {
+    const result = grade(jsonOnlyExpectation, jsonOnlyDir, writeParts(`negative-split-${index}.jsonl`, parts));
+    assert.strictEqual(result.passed, false, `${name} must fail end to end`);
+    assert.match(result.results[0].reason, expected, `${name} should give a clear reason`);
+  });
+
+  // Claude stream-json: assistant content parts plus a terminal `result` that
+  // repeats them. The repeat must not be double-counted, and must not mask prose.
+  const writeClaude = (name, contentTexts, resultText) => {
+    const transcriptPath = path.join(jsonOnlyDir, name);
+    const lines = [
+      { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'tu-1', name: 'Read', input: { file_path: 'profiles.md' } }] } },
+      { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'tu-1', content: 'ok' }] } },
+      { type: 'assistant', message: { content: contentTexts.map(text => ({ type: 'text', text })) } },
+      { type: 'result', subtype: 'success', result: resultText },
+    ];
+    fs.writeFileSync(transcriptPath, lines.map(line => JSON.stringify(line)).join('\n'));
+    return transcriptPath;
+  };
+  assert.strictEqual(grade(jsonOnlyExpectation, jsonOnlyDir, writeClaude('claude-result-repeat.jsonl', [FENCED], FENCED)).passed, true, 'Claude result repeat must not double the JSON');
+  assert.strictEqual(grade(jsonOnlyExpectation, jsonOnlyDir, writeClaude('claude-prose-parts.jsonl', ['Here is the record.', FENCED], FENCED)).passed, false, 'Claude prose content part must fail even when result holds only JSON');
+} finally {
+  fs.rmSync(jsonOnlyDir, { recursive: true, force: true });
+}
+
+// Skill-core completion: passing project tests is artifact evidence only; the
+// TDD case also requires an executed quality gate command.
+const tddCase = parseSimpleYaml(fs.readFileSync(path.join(casesDir, 'baseline-tdd-workflow.yaml'), 'utf8'));
+const qualityGate = tddCase.expectations.find(expectation => expectation.type === 'tdd_quality_gate');
+assert.ok(qualityGate, 'baseline TDD case must require a passed TDD quality gate');
+// The gate is graded by independently re-evaluating retained evidence, so the
+// workspace holds a real passing evidence file and a failing variant.
+const PASS_EVIDENCE = fs.readFileSync(path.join(__dirname, '..', 'tdd', 'fixtures', 'pass-evidence.json'), 'utf8');
+const FAIL_EVIDENCE = (() => {
+  const evidence = JSON.parse(PASS_EVIDENCE);
+  evidence.requirements = evidence.requirements.map(requirement => ({ ...requirement, sourceConformance: 'FAIL' }));
+  return JSON.stringify(evidence);
+})();
+const gateDir = fs.mkdtempSync(path.join(os.tmpdir(), 'behavioral-tdd-quality-gate-'));
+try {
+  fs.writeFileSync(path.join(gateDir, 'evidence.json'), PASS_EVIDENCE);
+  fs.writeFileSync(path.join(gateDir, 'failing-evidence.json'), FAIL_EVIDENCE);
+  fs.writeFileSync(path.join(gateDir, 'broken-evidence.json'), '{ not json');
+  // A string command defaults to the gate's own success line when it is a
+  // quality-gate invocation, so existing positives stay meaningful. An object
+  // command states the observed output and exit code explicitly.
+  const QUALITY_GATE_COMMAND_RE = /quality-gate\.js|tdd:quality|test:tdd-quality/;
+  const gradeCommands = (name, commands) => {
+    const transcriptPath = path.join(gateDir, `${name}.jsonl`);
+    const events = commands.map((entry, index) => {
+      const { command, output, exit } = typeof entry === 'string'
+        ? { command: entry, output: QUALITY_GATE_COMMAND_RE.test(entry) ? 'TDD Quality Gate: PASS' : '', exit: undefined }
+        : entry;
+      const state = { status: 'completed', input: { command } };
+      if (output !== undefined) state.output = output;
+      if (exit !== undefined) state.metadata = { exit };
+      return { type: 'tool_use', part: { type: 'tool', tool: 'Bash', callID: `${name}-${index}`, state } };
+    });
+    events.push({ type: 'text', part: { type: 'text', text: 'done' } });
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return grade({ expectations: [qualityGate] }, gateDir, transcriptPath);
+  };
+  const testsOnly = gradeCommands('tests-only', ['npm test']);
+  assert.strictEqual(testsOnly.passed, false, 'passing project tests without the quality gate must fail the TDD case');
+  assert.strictEqual(testsOnly.results[0].status, 'fail', 'missing quality gate must be a definitive fail, not inconclusive evidence');
+  assert.strictEqual(gradeCommands('npm-gate', ['npm test', 'npm run tdd:quality -- evidence.json']).passed, true, 'npm run tdd:quality with evidence must satisfy the gate');
+  assert.strictEqual(gradeCommands('node-gate', ['npm test', 'node tdd/scripts/quality-gate.js evidence.json']).passed, true, 'node quality-gate.js with evidence must satisfy the gate');
+  assert.strictEqual(gradeCommands('installed-skill-gate', ['npm test', 'node .claude/skills/tdd/scripts/quality-gate.js evidence.json']).passed, true, 'quality gate under the installed skill dir must satisfy the gate');
+  assert.strictEqual(gradeCommands('absolute-gate', ['npm test', 'node /tmp/ws/.harness/tdd/scripts/quality-gate.js evidence.json']).passed, true, 'absolute quality-gate path must satisfy the gate');
+  for (const [name, commands] of [
+    ['echo-mention', ['echo npm run tdd:quality']],
+    ['compound-gate', ['npm run tdd:quality -- evidence.json && rm -rf build']],
+    ['similar-name', ['npm run tdd:quality-notes']],
+  ]) {
+    assert.strictEqual(gradeCommands(name, commands).passed, false, `${name} must not count as an executed quality gate`);
+  }
+  // A completed tool call is not completion evidence: the gate outcome is
+  // re-evaluated from the retained evidence, independent of transcript claims.
+  const gateFailures = [
+    ['nonzero-exit', ['npm test', 'npm run tdd:quality -- failing-evidence.json'], /re-evaluation of failing-evidence\.json is FAIL/],
+    ['missing-evidence', ['npm test', 'npm run tdd:quality -- absent.json'], /not retained/],
+    ['invalid-evidence', ['npm test', 'node tdd/scripts/quality-gate.js broken-evidence.json'], /not valid JSON/],
+    ['no-evidence-arg', ['npm test', 'npm run tdd:quality'], /without an evidence file/],
+    ['self-test-alias', ['npm test', 'npm run test:tdd-quality'], /repository self-test/],
+    ['escaping-evidence', ['npm test', 'npm run tdd:quality -- ../evidence.json'], /outside the workspace/],
+    ['missing-output-report', ['npm test', 'npm run tdd:quality -- evidence.json --output reports/quality.json'], /quality report reports\/quality\.json was not retained/],
+    ['last-run-counts', ['npm run tdd:quality -- evidence.json', 'npm run tdd:quality -- failing-evidence.json'], /FAIL/],
+  ];
+  for (const [name, commands, reason] of gateFailures) {
+    const graded = gradeCommands(name, commands);
+    assert.strictEqual(graded.passed, false, `${name} must not satisfy the quality gate`);
+    assert.strictEqual(graded.results[0].status, 'fail', `${name} must be a definitive fail`);
+    assert.match(graded.results[0].reason, reason, `${name} should explain why`);
+  }
+  // OpenCode reports state.status "completed" even when the gate exits 1.
+  const opencodeExit1 = path.join(gateDir, 'opencode-exit1.jsonl');
+  fs.writeFileSync(opencodeExit1, [
+    { type: 'tool_use', part: { type: 'tool', tool: 'bash', callID: 'gate', state: { status: 'completed', input: { command: 'npm run tdd:quality -- failing-evidence.json' }, output: 'TDD Quality Gate: FAIL\nexit code 1' } } },
+    { type: 'text', part: { type: 'text', text: 'Quality gate done.' } },
+  ].map(event => JSON.stringify(event)).join('\n'));
+  assert.strictEqual(grade({ expectations: [qualityGate] }, gateDir, opencodeExit1).passed, false, 'completed OpenCode event with exit 1 must fail');
+  // Fixing evidence and re-running is allowed: the last completed run counts.
+  assert.strictEqual(gradeCommands('rerun-after-fix', ['npm run tdd:quality -- failing-evidence.json', 'npm run tdd:quality -- evidence.json']).passed, true, 'a passing re-run after a failing run should satisfy the gate');
+  // Observed command outcome: the transcript must show the gate's own success
+  // line and no error or non-zero exit before the evidence is trusted.
+  const { evaluateEvidence } = require('../tdd/scripts/quality-gate.js');
+  const observedGate = 'npm run tdd:quality -- evidence.json';
+  const observedFailures = [
+    ['failed-command-pass-evidence', { command: observedGate, output: 'npm ERR! Missing script: "tdd:quality"', exit: 1 }, /quality gate command failed/],
+    ['nonzero-exit-no-error-text', { command: observedGate, output: 'TDD Quality Gate: PASS', exit: 1 }, /exited with code 1/],
+    ['gate-printed-fail-evidence-now-pass', { command: observedGate, output: 'TDD Quality Gate: FAIL' }, /printed TDD Quality Gate: FAIL/],
+  ];
+  for (const [name, command, reason] of observedFailures) {
+    const graded = gradeCommands(name, [command]);
+    assert.strictEqual(graded.passed, false, `${name} must not satisfy the quality gate`);
+    assert.strictEqual(graded.results[0].status, 'fail', `${name} must be a definitive fail`);
+    assert.match(graded.results[0].reason, reason, `${name} should explain why`);
+  }
+  const unobservable = gradeCommands('unobservable-outcome', [{ command: observedGate, output: '' }]);
+  assert.strictEqual(unobservable.results[0].status, 'inconclusive', 'a gate run with no observable output or exit code must be inconclusive');
+  assert.strictEqual(unobservable.passed, false, 'an inconclusive gate run must not pass');
+  assert.match(unobservable.results[0].reason, /not observable in the transcript/, 'inconclusive reason should say the outcome is not observable');
+  // Definitive workspace failures win over an unobservable outcome.
+  const missingNoOutput = gradeCommands('missing-evidence-no-output', [{ command: 'npm run tdd:quality -- nope.json', output: '' }]);
+  assert.strictEqual(missingNoOutput.results[0].status, 'fail', 'missing evidence must be a definitive fail even with no observable output');
+  assert.match(missingNoOutput.results[0].reason, /not retained/, 'missing evidence should be named');
+  // Only the error signal is line-anchored: prose mentioning npm errors passes,
+  // while a real npm error line still fails.
+  assert.strictEqual(gradeCommands('prose-npm-error', [{ command: observedGate, output: 'TDD Quality Gate: PASS\n- note: avoid npm error handling', exit: 0 }]).passed, true, 'prose mentioning npm error must not fail a passing gate');
+  const npmErrorLine = gradeCommands('npm-error-line', [{ command: observedGate, output: 'npm error Missing script: "tdd:quality"' }]);
+  assert.strictEqual(npmErrorLine.results[0].status, 'fail', 'an npm error line must be a definitive fail');
+  assert.match(npmErrorLine.results[0].reason, /npm error/, 'npm error line should be named');
+  // npm ERR!/error lines after a PASS line must fail even without exit metadata;
+  // the boundary after the token must match whitespace or end of line.
+  const npmTrailer = (name, output) => gradeCommands(name, [{ command: observedGate, output }]);
+  for (const [name, output] of [
+    ['npm-err-bang-eacces', 'TDD Quality Gate: PASS\nnpm ERR! code EACCES'],
+    ['npm-err-bang-eresolve', 'TDD Quality Gate: PASS\nnpm ERR! code ERESOLVE'],
+    ['npm-error-elifecycle', 'TDD Quality Gate: PASS\nnpm error code ELIFECYCLE'],
+    ['npm-err-bang-eol', 'TDD Quality Gate: PASS\nnpm ERR!'],
+  ]) {
+    const graded = npmTrailer(name, output);
+    assert.strictEqual(graded.results[0].status, 'fail', `${name} must be a definitive fail`);
+    assert.strictEqual(graded.passed, false, `${name} must not satisfy the quality gate`);
+    assert.match(graded.results[0].reason, /npm error/i, `${name} should name the npm error`);
+  }
+  assert.strictEqual(npmTrailer('npm-errors-word', 'TDD Quality Gate: PASS\nnpm ERRORS are documented').passed, true, 'a word that only starts with ERR must not be an npm error');
+  const { GATE_COMMAND_ERROR_RE } = require('../behavioral-evals/run');
+  assert.strictEqual(GATE_COMMAND_ERROR_RE.test('npm ERR! code EACCES'), true, 'npm ERR! line must match the error regex');
+  assert.strictEqual(GATE_COMMAND_ERROR_RE.test('npm ERRORS'), false, 'npm ERRORS must not match the error regex');
+
+  // Claude stream-json: is_error on the tool_result is the host's failure
+  // signal; the call is recorded as failed, so it never counts as executed.
+  const claudeGate = (name, toolResult) => {
+    const transcriptPath = path.join(gateDir, `${name}.jsonl`);
+    fs.writeFileSync(transcriptPath, [
+      { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'bash-1', name: 'Bash', input: { command: observedGate } }] } },
+      { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'bash-1', ...toolResult }] } },
+      { type: 'result', subtype: 'success', result: 'done' },
+    ].map(event => JSON.stringify(event)).join('\n'));
+    return grade({ expectations: [qualityGate] }, gateDir, transcriptPath);
+  };
+  const claudeError = claudeGate('claude-error', { is_error: true, content: 'npm ERR! Missing script' });
+  assert.strictEqual(claudeError.passed, false, 'Claude tool_result with is_error must not satisfy the gate');
+  assert.strictEqual(claudeError.results[0].status, 'fail', 'Claude tool_result error must be a definitive fail');
+  assert.match(claudeError.results[0].reason, /last quality gate run \(bash-1\) failed/, 'Claude errored gate call should be the failing last run');
+  assert.strictEqual(claudeGate('claude-success', { is_error: false, content: [{ type: 'text', text: 'TDD Quality Gate: PASS' }] }).passed, true, 'Claude tool_result carrying the gate success line should pass');
+  // Runs are graded in transcript order: the last matching attempt decides, so an
+  // earlier PASS never hides a later failed or unresolved run.
+  const claudeRuns = (name, runs) => {
+    const transcriptPath = path.join(gateDir, `${name}.jsonl`);
+    const events = [];
+    runs.forEach(({ command, result }, index) => {
+      const id = `${name}-bash-${index}`;
+      events.push({ type: 'assistant', message: { content: [{ type: 'tool_use', id, name: 'Bash', input: { command } }] } });
+      if (result) events.push({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, ...result }] } });
+    });
+    events.push({ type: 'assistant', message: { content: [{ type: 'text', text: 'done' }] } });
+    events.push({ type: 'result', subtype: 'success', result: 'done' });
+    fs.writeFileSync(transcriptPath, events.map(event => JSON.stringify(event)).join('\n'));
+    return grade({ expectations: [qualityGate] }, gateDir, transcriptPath);
+  };
+  const gatePass = { is_error: false, content: [{ type: 'text', text: 'TDD Quality Gate: PASS' }] };
+  const gateFail = { is_error: true, content: 'TDD Quality Gate: FAIL' };
+  const lastRunError = claudeRuns('claude-pass-then-error', [
+    { command: observedGate, result: gatePass },
+    { command: observedGate, result: gateFail },
+  ]);
+  assert.strictEqual(lastRunError.results[0].status, 'fail', 'a failed last run must be a definitive fail even after an earlier PASS');
+  assert.strictEqual(lastRunError.passed, false, 'an earlier PASS must not satisfy the gate after a failed last run');
+  assert.match(lastRunError.results[0].reason, /last quality gate run/, 'the failed last run should be named');
+  const lastRunUnresolved = claudeRuns('claude-pass-then-unresolved', [
+    { command: observedGate, result: gatePass },
+    { command: observedGate },
+  ]);
+  assert.strictEqual(lastRunUnresolved.results[0].status, 'inconclusive', 'an unresolved last run must be inconclusive, not a pass');
+  assert.strictEqual(lastRunUnresolved.passed, false, 'an unresolved last run must not pass the gate');
+  assert.match(lastRunUnresolved.results[0].reason, /no observable result/, 'the unresolved last run should be named');
+  const otherEvidenceError = claudeRuns('claude-pass-then-other-evidence', [
+    { command: observedGate, result: gatePass },
+    { command: 'node tdd/scripts/quality-gate.js failing-evidence.json', result: gateFail },
+  ]);
+  assert.strictEqual(otherEvidenceError.results[0].status, 'fail', 'a failed run against other evidence must fail the gate');
+  assert.match(otherEvidenceError.results[0].reason, /last quality gate run/, 'the failed last run against other evidence should be named');
+  assert.strictEqual(claudeRuns('claude-fail-then-retry', [
+    { command: observedGate, result: gateFail },
+    { command: observedGate, result: gatePass },
+  ]).passed, true, 'a failed run followed by a successful retry must pass');
+  // Retained --output report: a full report must be present and must equal the
+  // fresh re-evaluation over every top-level key.
+  const realReport = evaluateEvidence(JSON.parse(PASS_EVIDENCE));
+  assert.strictEqual(realReport.result, 'PASS', 'the pass fixture must re-evaluate to PASS');
+  fs.mkdirSync(path.join(gateDir, 'reports'), { recursive: true });
+  const reportPath = path.join(gateDir, 'reports', 'quality.json');
+  fs.writeFileSync(reportPath, JSON.stringify({ result: 'PASS' }));
+  const markerOnly = gradeCommands('marker-output-report', ['npm run tdd:quality -- evidence.json --output reports/quality.json']);
+  assert.strictEqual(markerOnly.passed, false, 'a {"result":"PASS"} marker must not satisfy a retained report');
+  assert.match(markerOnly.results[0].reason, /missing required keys: .*requirements/, 'a marker-only retained report should name the missing keys');
+  fs.writeFileSync(reportPath, JSON.stringify(realReport, null, 2));
+  assert.strictEqual(gradeCommands('retained-output-report', ['npm run tdd:quality -- evidence.json --output reports/quality.json']).passed, true, 'retained --output report equal to the re-evaluation should pass');
+  fs.writeFileSync(reportPath, JSON.stringify({ ...realReport, score: realReport.score - 1 }, null, 2));
+  const staleScore = gradeCommands('stale-output-report', ['npm run tdd:quality -- evidence.json --output reports/quality.json']);
+  assert.strictEqual(staleScore.passed, false, 'a retained report with a stale score must fail');
+  assert.match(staleScore.results[0].reason, /differs from re-evaluation at "score"/, 'a stale retained report should name the differing key');
+  // Without --output the gate prints its report to stdout; that printed report
+  // must equal the fresh re-evaluation of the evidence the command names.
+  const printedText = report => `TDD Quality Gate: PASS\n${JSON.stringify(report, null, 2)}`;
+  assert.strictEqual(gradeCommands('stdout-report-equal', [{ command: observedGate, output: printedText(realReport) }]).passed, true, 'a printed report equal to the re-evaluation should pass');
+  const otherEvidence = JSON.parse(PASS_EVIDENCE);
+  otherEvidence.requirements[0].requirementId = 'REQ-OTHER';
+  fs.writeFileSync(path.join(gateDir, 'other-evidence.json'), JSON.stringify(otherEvidence));
+  const otherReport = evaluateEvidence(otherEvidence);
+  const stdoutMismatch = gradeCommands('stdout-report-other', [{ command: observedGate, output: printedText(otherReport) }]);
+  assert.strictEqual(stdoutMismatch.passed, false, 'a printed report from different evidence must fail');
+  assert.match(stdoutMismatch.results[0].reason, /gate output report differs from re-evaluation at "[a-zA-Z]+"/, 'a mismatched printed report should name the differing key');
+} finally {
+  fs.rmSync(gateDir, { recursive: true, force: true });
+}
+
+console.log(`Behavioral case validation passed (${fs.readdirSync(casesDir).filter(name => name.endsWith('.yaml')).length} cases + negative controls + ordered execution evidence + json-only final output + tdd quality gate).`);

package/code-review/SKILL.md

@@ -4,7 +4,7 @@ description: "Independently review a PR, diff or finished implementation for cor
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.40.0
+  version: 0.47.5
 ---
 
 # Code Review

package/code-review/references/review-playbook.md

@@ -19,6 +19,15 @@ Use this checklist for the applicable risks; do not force irrelevant categories.
 - [ ] Follow each important diagnostic/fact through every consumer and ancestor. Check whether required facts are computed before derived authority, indexes, or output.
 - [ ] Compare new rules with previously accepted rules and controls. Separate display severity from safety prerequisites.
 
+## Check evidence a gate trusts
+
+Apply these when the change grades, gates or labels evidence. Authors SHOULD run them as a self-review before push.
+
+- [ ] **Claimed vs observed.** Separate what the subject reports about itself (a tool's `completed` status, a result marker, an agent-supplied expected value, a status label) from what the checker observes independently (exit code, output, a recomputed result, the quoted source). A self-report alone never passes; when nothing independent is observable, the result is inconclusive.
+- [ ] **Select, then judge.** When the last, first or any item of a sequence decides (last run, final message, latest attempt), select from every attempt in order before filtering by outcome. Test both orders: failure then success, and success then failure or no result.
+- [ ] **One witness per rule.** Give each detection rule a negative case that triggers only that rule; a case that also trips an exit code or another pattern cannot show the rule works. Probe positive cases at structural boundaries (inside a token, string or quote), not only at safe separators.
+- [ ] **Comparable subjects, honest labels.** A comparison or conflict needs both sides to describe the same target. A status name claims only what was checked: a consistent quote is not a validated diagnosis.
+
 ## Check tests and output
 
 - [ ] Ask whether each test still passes if the behavior is disabled, bypassed, or broken (mutation thought experiment). Confirm the test is included in CI.

package/opencode-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "harness-enforcement",
   "description": "Repository inventory for the opencode enforcement plugin and its standalone helper",
-  "version": "0.47.3",
+  "version": "0.47.5",
   "runtime": "index.mjs",
   "hooks": {
     "verify": "hooks/verify.js"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-everything",
-  "version": "0.47.3",
+  "version": "0.47.5",
   "description": "Harness OS: An Orchestrated Agent Operating System for AI-driven Development",
   "main": "harness-everything/scripts/bootstrap.js",
   "bin": {

package/plugins/harness-everything/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-everything",
-  "version": "0.47.3",
+  "version": "0.47.5",
   "description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
   "author": {
     "name": "Miya Daniel",

package/plugins/harness-everything/plugin.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json",
   "name": "harness-everything",
-  "version": "0.47.3",
+  "version": "0.47.5",
   "description": "Invariant-first engineering harness for routing, verification, and failure recovery while preserving agent autonomy.",
   "author": {
     "name": "Miya Daniel",

package/plugins/harness-everything/skills/code-review/SKILL.md

@@ -4,7 +4,7 @@ description: "Independently review a PR, diff or finished implementation for cor
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.40.0
+  version: 0.47.5
 ---
 
 # Code Review

package/plugins/harness-everything/skills/code-review/references/review-playbook.md

@@ -19,6 +19,15 @@ Use this checklist for the applicable risks; do not force irrelevant categories.
 - [ ] Follow each important diagnostic/fact through every consumer and ancestor. Check whether required facts are computed before derived authority, indexes, or output.
 - [ ] Compare new rules with previously accepted rules and controls. Separate display severity from safety prerequisites.
 
+## Check evidence a gate trusts
+
+Apply these when the change grades, gates or labels evidence. Authors SHOULD run them as a self-review before push.
+
+- [ ] **Claimed vs observed.** Separate what the subject reports about itself (a tool's `completed` status, a result marker, an agent-supplied expected value, a status label) from what the checker observes independently (exit code, output, a recomputed result, the quoted source). A self-report alone never passes; when nothing independent is observable, the result is inconclusive.
+- [ ] **Select, then judge.** When the last, first or any item of a sequence decides (last run, final message, latest attempt), select from every attempt in order before filtering by outcome. Test both orders: failure then success, and success then failure or no result.
+- [ ] **One witness per rule.** Give each detection rule a negative case that triggers only that rule; a case that also trips an exit code or another pattern cannot show the rule works. Probe positive cases at structural boundaries (inside a token, string or quote), not only at safe separators.
+- [ ] **Comparable subjects, honest labels.** A comparison or conflict needs both sides to describe the same target. A status name claims only what was checked: a consistent quote is not a validated diagnosis.
+
 ## Check tests and output
 
 - [ ] Ask whether each test still passes if the behavior is disabled, bypassed, or broken (mutation thought experiment). Confirm the test is included in CI.

package/plugins/harness-everything/skills/tdd/SKILL.md

@@ -4,7 +4,7 @@ description: "Drive Standard Tasks (Tier 2) with Test-Driven Development (RED/GR
 license: Apache-2.0
 metadata:
   author: Miya Daniel
-  version: 0.13.0
+  version: 0.47.5
 ---
 
 # Test-Driven Development

package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json

@@ -8,8 +8,16 @@
     { "name": "weighted integration failure", "mutation": "the 0.15 positive dimension scores zero", "expectedScore": 85, "expectedResult": "FAIL" },
     { "name": "weighted unit failure", "mutation": "unit determinism is N/A and the 0.15 positive dimension scores zero", "expectedScore": 83.33, "expectedResult": "FAIL" },
     { "name": "NON_CONFORMANT", "mutation": "sourceConformance = NON_CONFORMANT", "expectedResult": "FAIL" },
-    { "name": "SOURCE_DEFECT", "mutation": "sourceValidity = SOURCE_DEFECT and sourceNotes is present", "expectedResult": "PASS" },
-    { "name": "SOURCE_CONFLICT", "mutation": "sourceValidity = SOURCE_CONFLICT with a traced additional source", "expectedResult": "FAIL" },
+    { "name": "SOURCE_DEFECT", "mutation": "sourceValidity = SOURCE_DEFECT, sourceNotes is present, and sourceEvidence quotes the defective assertion against the observed value; sourceEvidenceStatus is REVIEWER_REQUIRED, never VERIFIED, because a single-source mismatch does not prove the source is wrong", "expectedResult": "PASS" },
+    { "name": "SOURCE_DEFECT without sourceEvidence", "mutation": "sourceValidity = SOURCE_DEFECT with sourceNotes but no sourceEvidence", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT", "mutation": "sourceValidity = SOURCE_CONFLICT with a traced additional source and sourceEvidence quoting both sources", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT without sourceEvidence", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources but no sourceEvidence", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT node-versus-value", "mutation": "sourceValidity = SOURCE_CONFLICT citing only expect(list.head.value).toEqual(1) from one spec, observed value is an Element object", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT agreeing expectations", "mutation": "sourceValidity = SOURCE_CONFLICT with two sources expecting the same value and a failing observed value; the mismatch is implementation behaviour", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT two-source evidence", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting disagreeing expectations for the same subject list.head and a failing observed value; one quote (toBeInstanceOf) is not mechanically checkable, so sourceEvidenceStatus is REVIEWER_REQUIRED and the unresolved conflict gate stays closed", "expectedEvidence": "ACCEPTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT relabelled expected", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting the identical assertion expect(list.head).toEqual(1) but expected values 1 and 3; each expected must match its quoted literal, so the second entry is rejected", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT verified literal disagreement", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting expect(list.head.value).toBe(1) and expect(list.head.value).toBe('1') with matching expected values 1 and '1' and a failing observed value; sourceEvidenceStatus is VERIFIED and the unresolved conflict gate stays closed", "expectedEvidence": "ACCEPTED", "expectedResult": "FAIL" },
+    { "name": "SOURCE_CONFLICT unrelated assertions", "mutation": "sourceValidity = SOURCE_CONFLICT quoting expect(foo).toEqual(1) from one source and expect(bar).toEqual(3) from another; unequal expectations about different subjects", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
     { "name": "malformed input", "mutation": "requirements is not an array", "expectedResult": "FAIL" },
     { "name": "unexpected input", "mutation": "one unexpectedInput check is FAIL", "expectedResult": "FAIL" },
     { "name": "error handling", "mutation": "one errorHandling check is SKIPPED", "expectedResult": "FAIL" },

package/plugins/harness-everything/skills/tdd/references/quality-model.md

@@ -2,6 +2,8 @@
 
 This model turns each source requirement into auditable evidence. The authoritative source remains the contract even when it appears wrong. Record a suspected documentation problem as `SOURCE_DEFECT`; record disagreement between traced sources as `SOURCE_CONFLICT`. Never reinterpret either into a passing implementation. If no authoritative source exists, use `INFERRED` and state the inference basis.
 
+Before declaring either outcome, `sourceEvidence` quotes the exact conflicting assertions, names each declared source by path, and compares each `expected` value with the `observed` value from the run. A `SOURCE_CONFLICT` needs at least two distinct sources whose expected values disagree about the same `subject` — the contract target each quoted assertion constrains, written as it appears in the quote (`list.head` is not `list.head.value`). Unequal expectations about unrelated targets are not a conflict. If the quoted expectations agree and the run still fails, the mismatch is in the implementation: record `NON_CONFORMANT`, not `SOURCE_CONFLICT`. Each `expected` must match a mechanically readable literal in the quoted assertion (`toEqual`, `toStrictEqual`, `toBe`, or `assert.*` with a JSON or single-quoted literal); otherwise the claim is marked `REVIEWER_REQUIRED` and the gate does not treat it as an established disagreement. `VERIFIED` applies only to a `SOURCE_CONFLICT` whose two quoted literals disagree. A `SOURCE_DEFECT` is always `REVIEWER_REQUIRED`, because a mismatch with the observed value does not prove the source is wrong; the implementation may be the one at fault.
+
 ## Profile contract
 
 Every requirement declares exactly one `testProfile`: `unit` or `integration`. The selected profile defines which case classes are applicable; the common scoring and source rules do not make every test use identical evidence.

package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json

@@ -149,6 +149,18 @@
         }
       ]
     },
+    "sourceEvidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["source", "assertion", "expected", "observed"],
+      "properties": {
+        "source": { "type": "string", "minLength": 1 },
+        "subject": { "type": "string", "minLength": 1 },
+        "assertion": { "type": "string", "minLength": 1 },
+        "expected": {},
+        "observed": {}
+      }
+    },
     "requirement": {
       "type": "object",
       "additionalProperties": false,
@@ -161,6 +173,7 @@
         "sourceConformance": { "enum": ["PASS", "NON_CONFORMANT"] },
         "sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT"] },
         "sourceNotes": { "type": "string", "minLength": 1 },
+        "sourceEvidence": { "type": "array", "items": { "$ref": "#/$defs/sourceEvidence" } },
         "skippedTests": { "type": "array" },
         "flakyTests": { "type": "array" },
         "naDeclarations": { "type": "array" },
@@ -186,6 +199,14 @@
         }
       },
       "allOf": [
+        {
+          "if": { "required": ["sourceValidity"], "properties": { "sourceValidity": { "enum": ["SOURCE_DEFECT", "SOURCE_CONFLICT"] } } },
+          "then": { "required": ["sourceEvidence"], "properties": { "sourceEvidence": { "minItems": 1 } } }
+        },
+        {
+          "if": { "required": ["sourceValidity"], "properties": { "sourceValidity": { "const": "SOURCE_CONFLICT" } } },
+          "then": { "properties": { "sourceEvidence": { "items": { "required": ["subject"] } } } }
+        },
         {
           "if": {
             "required": ["testProfile"],

package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json

@@ -96,6 +96,8 @@
         "sourceConformance": { "enum": ["PASS", "NON_CONFORMANT", null] },
         "sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT", null] },
         "sourceNotes": { "type": "string" },
+        "sourceEvidence": { "type": "array" },
+        "sourceEvidenceStatus": { "enum": ["VERIFIED", "REVIEWER_REQUIRED"], "description": "VERIFIED only for a SOURCE_CONFLICT whose two quoted literals disagree; SOURCE_DEFECT is always REVIEWER_REQUIRED." },
         "dimensions": { "$ref": "#/$defs/dimensions" },
         "skippedTests": { "type": "array" },
         "flakyTests": { "type": "array" },