harness-everything 0.47.3 → 0.47.5

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "harness-everything",
       "source": "./",
       "description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
-      "version": "0.47.3",
+      "version": "0.47.5",
       "author": {
         "name": "Miya Daniel"
       },

package/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "harness-everything",
   "description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
-  "version": "0.47.3",
+  "version": "0.47.5",
   "author": {
     "name": "Miya Daniel"
   },

package/CHANGELOG.md

@@ -2,6 +2,28 @@
 
 All notable changes to this project are documented in this file.
 
+## [0.47.5](https://github.com/dyphn1/Harness-everything/compare/v0.47.4...v0.47.5) (2026-11-03)
+
+### Bug Fixes
+
+* **behavioral-evals:** detect npm ERR! lines in quality gate output ([b599224](https://github.com/dyphn1/Harness-everything/commit/b599224ba09f27dc1f5e5c1156c40c20e57f6f52))
+* **behavioral-evals:** grade tdd quality gate on independently re-evaluated evidence ([f9433ed](https://github.com/dyphn1/Harness-everything/commit/f9433ed535ba4b1882029622a31325ae564cdd50)), closes [#311](https://github.com/dyphn1/Harness-everything/issues/311)
+* **behavioral-evals:** grade the last quality gate attempt, not the last completed run ([8065345](https://github.com/dyphn1/Harness-everything/commit/806534592dbc47665868e74a36b4095d44a2647b))
+* **behavioral-evals:** order gate workspace checks before outcome and compare printed report ([ded48fd](https://github.com/dyphn1/Harness-everything/commit/ded48fd7221cb66c61cc58a70c1f4924f0c0fc82))
+* **behavioral-evals:** require observed gate success and full retained report ([47d35d7](https://github.com/dyphn1/Harness-everything/commit/47d35d725623a4355dcc8a054347ec68832edaa2))
+* **tdd:** keep source defect evidence reviewer-required ([443e966](https://github.com/dyphn1/Harness-everything/commit/443e96635c972713acd9fa76e974cb79bfa310ff))
+* **tdd:** read assert.* expected from second argument; tolerate trailing semicolon ([1745ea2](https://github.com/dyphn1/Harness-everything/commit/1745ea2860a2566703af93ec23fc1ff1b00d7eef))
+* **tdd:** require quoted assertion evidence for source conflict claims ([e9e1ab9](https://github.com/dyphn1/Harness-everything/commit/e9e1ab9b8170054a1ecf57b440b18124b4f99108)), closes [#312](https://github.com/dyphn1/Harness-everything/issues/312)
+* **tdd:** require same-subject assertions for SOURCE_CONFLICT evidence ([d0ab25f](https://github.com/dyphn1/Harness-everything/commit/d0ab25fb810f390921acc9f38584eead95f18214))
+* **tdd:** verify sourceEvidence expected values against quoted literals ([77a8ba6](https://github.com/dyphn1/Harness-everything/commit/77a8ba690de69db9852371270e86c063d30b77ed))
+
+## [0.47.4](https://github.com/dyphn1/Harness-everything/compare/v0.47.3...v0.47.4) (2026-11-03)
+
+### Bug Fixes
+
+* **behavioral-evals:** grade the whole final assistant message for json-only ([ebbe33c](https://github.com/dyphn1/Harness-everything/commit/ebbe33c7ac8beb3d2b104f81666f06a0ebfcd77d)), closes [#308](https://github.com/dyphn1/Harness-everything/issues/308)
+* **behavioral-evals:** recombine final text parts exactly for json-only grading ([de8341c](https://github.com/dyphn1/Harness-everything/commit/de8341c0621c6aacf319467a9e3c9b1420f51f37)), closes [#308](https://github.com/dyphn1/Harness-everything/issues/308)
+
 ## [0.47.3](https://github.com/dyphn1/Harness-everything/compare/v0.47.2...v0.47.3) (2026-10-31)
 
 ### Bug Fixes
@@ -46,6 +68,8 @@ All notable changes to this project are documented in this file.
 
 ### Bug Fixes
 
+* **code-review:** add evidence-gate counterexamples (claimed vs observed, select then judge, one witness per rule, comparable subjects and honest labels) for reviewers and author self-review.
+* **zoom-out:** the author applying review fixes counts repair-loop rounds too; a decision class repeated from the previous round triggers Phase 2 before editing.
 * **doc-reconciliation:** pin the owner-approved VERI-to-EVID row mapping and exclude ordinary SubagentStop completions from quality-review candidates.
 * **doc-reconciliation:** allow uniquely identified same-root reviewers that start after the correlated draft worker stops.
 

package/behavioral-evals/README.md

@@ -168,9 +168,36 @@ command ran: final prose and attempted tool inputs can mention an action that
 never completed. Use structured execution expectations instead, for example
 `tool_completed` with `command: "npm test"` (command prefixes may include
 additional arguments) or `tool_denied` when denial is the behavior under test.
+When one completion must accept several explicit command shapes, use
+`value: {command_pattern: "<regex>"}`; the regex is matched against the whole
+command, so shell control operators must be excluded explicitly.
+`tdd_quality_gate` takes the same `command_pattern` but does not trust the tool
+call's completed status. For the last matching run the grader first runs the
+workspace checks: the evidence file must be retained, valid JSON and re-evaluate
+to `PASS` with this repository's `tdd/scripts/quality-gate.js`; a `--output`
+report must be present and deep-equal the fresh re-evaluation over every
+top-level key (key order ignored); and a full report printed to stdout must do
+the same. Missing, invalid or disagreeing artifacts are definitive `fail`s, and
+a missing required key or a differing key is named in the reason. Then it reads
+the observed command outcome: the call fails when it reports an error, exits
+non-zero, prints an npm error line (`npm ERR!`, `npm error`, or `Missing script:`)
+or prints `TDD Quality Gate: FAIL`. It passes only when the output contains the
+gate's own `TDD Quality Gate: PASS` line and none of those signals. When the
+transcript shows neither output nor an exit code, the result is `inconclusive`
+rather than a pass. The last matching attempt counts, in transcript order: a later failed or denied run fails the gate, and a later unresolved run is inconclusive, even when an earlier run passed.
 Legacy one-object Claude JSON and incomplete streams are graded inconclusive
 for execution evidence.
 
+Use `final_output_json_only` when the final message must be only a JSON object
+(raw, or inside one fenced block) with no prose outside it. Hidden
+`<!-- harness-label -->` lines are ignored. Optional `required_keys` lists keys
+that must be present. Extracting the selection fields from prose does not pass.
+The final message is every assistant text part after the last tool call, joined
+by exact concatenation (so JSON streamed in chunks reassembles exactly), with
+newline-separated blocks as an alternative; the case passes if either reading is
+a single JSON output. Claude's repeated `result` is used only when no parts exist.
+Prose streamed in a separate part still fails under both readings.
+
 Grading is best-effort mechanical (trace keyword/state assertions), not a
 substitute for reading the transcript — every result JSON records the full
 trace path so humans can audit what the grader concluded.

package/behavioral-evals/case-validator.js

@@ -25,6 +25,8 @@ const EXPECTATION_TYPES = new Set([
   'tool_completed',
   'tool_denied',
   'execution_evidence',
+  'tdd_quality_gate',
+  'final_output_json_only',
 ]);
 
 const PRESSURE_CATEGORIES = new Set([
@@ -118,13 +120,30 @@ function validateExpectation(expectation, index) {
     errors.push(`expectations[${index}] ${type} requires a string value`);
   }
   if (type === 'command_exit_0' && (typeof expectation.command !== 'string' || !expectation.command.trim())) errors.push(`expectations[${index}] command_exit_0 requires a non-empty command`);
-  if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence'].includes(type)) {
+  if (type === 'tdd_quality_gate' && !(isPlainObject(expectation.value) && expectation.value.command_pattern !== undefined)) {
+    errors.push(`expectations[${index}] tdd_quality_gate requires value.command_pattern`);
+  }
+  if (type === 'final_output_json_only' && expectation.required_keys !== undefined) {
+    const keys = expectation.required_keys;
+    if (!Array.isArray(keys) || keys.some(key => typeof key !== 'string' || !key.trim())) {
+      errors.push(`expectations[${index}] required_keys must be an array of non-empty strings`);
+    }
+  }
+  if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence', 'tdd_quality_gate'].includes(type)) {
     if (expectation.value === undefined && expectation.tool === undefined && expectation.name === undefined && expectation.command === undefined) {
       errors.push(`expectations[${index}] ${type} requires a tool/value target`);
     }
     if (expectation.value !== undefined && !['string', 'number'].includes(typeof expectation.value) && !isPlainObject(expectation.value)) {
       errors.push(`expectations[${index}] execution target must be scalar or mapping`);
     }
+    if (isPlainObject(expectation.value) && expectation.value.command_pattern !== undefined) {
+      if (typeof expectation.value.command_pattern !== 'string' || !expectation.value.command_pattern) {
+        errors.push(`expectations[${index}] command_pattern must be a non-empty string`);
+      } else {
+        try { new RegExp(expectation.value.command_pattern); }
+        catch { errors.push(`expectations[${index}] command_pattern must be a valid regular expression`); }
+      }
+    }
   }
   return errors;
 }
@@ -203,6 +222,8 @@ function negativeControls() {
     { id: 'bad-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: '../escape.js', content: 'x' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
     { id: 'duplicate-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }, { path: 'x.js', content: '' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
     { id: 'bad-execution', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tool_executed' }] },
+    { id: 'quality-gate-no-pattern', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tdd_quality_gate', value: { name: 'Bash' } }] },
+    { id: 'bad-command-pattern', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tool_executed', value: { command_pattern: '(' } }] },
   ];
   return controls.map(control => ({ control, errors: validateCase(control) })).filter(result => result.errors.length === 0);
 }

package/behavioral-evals/cases/baseline-profile-lookup-json-only.yaml

@@ -0,0 +1,23 @@
+# Baseline case: Profile lookup returns only the JSON selection record
+id: baseline-profile-lookup-json-only
+name: Profile lookup answers with a JSON-only selection record
+discipline: fable-mode
+loaded_skills:
+  - fable-mode
+fixture:
+  files:
+    - path: README.md
+      content: |
+        Profile lookup fixture. No orchestration or delegation is needed.
+prompt: "Explain what 'fable on sonnect' means; return only a small JSON selection record with requestedProfile, effectiveProfile, profileAlias, assignedRole, runtimeModel, runtimeEffort. No orchestration or delegation."
+max_turns: 4
+expectations:
+  - type: final_output_json_only
+    required_keys:
+      - requestedProfile
+      - effectiveProfile
+      - profileAlias
+      - assignedRole
+      - runtimeModel
+      - runtimeEffort
+    description: final message is exactly one JSON selection record (raw or fenced) with no prose outside it, and carries all six selection fields; selection-field extraction alone is not completion

package/behavioral-evals/cases/baseline-tdd-workflow.yaml

@@ -26,3 +26,7 @@ expectations:
     path: "calculator.js"
     value: "divide"
     description: divide function exists
+  - type: tdd_quality_gate
+    value:
+      command_pattern: '^(?:npm (?:run|run-script) (?:(?:--silent|-s|--quiet) )*(?:tdd:quality|test:tdd-quality)|node (?:--no-warnings )?(?:[^\s;&|<>`$()]*/)?tdd/scripts/quality-gate\.js)(?: [^;&|<>`$()\n\r]*)?$'
+    description: TDD quality gate passed on retained evidence (independently re-evaluated; skill-core completion, not artifact pass)

package/behavioral-evals/run.js

@@ -28,7 +28,7 @@ const {
   parseTranscriptFile,
 } = require('./transcript-parser');
 const { commandMatches, normalizeRepoPath } = require('../scripts/lib/execution-contract');
-const { resolveFixtureSetup, validateFixture } = require('./case-validator');
+const { resolveFixtureSetup, validateFixture, validateExpectation } = require('./case-validator');
 
 const ROOT = path.resolve(__dirname, '..');
 const CASES_DIR = path.join(__dirname, 'cases');
@@ -150,6 +150,8 @@ function validate(cases) {
   const EXPECT_TYPES = new Set([
     'trace_contains', 'trace_not_contains', 'file_contains', 'file_not_exists', 'command_exit_0',
     'tool_call', 'tool_attempted', 'tool_completed', 'tool_executed', 'tool_denied', 'execution_evidence',
+    'tdd_quality_gate',
+    'final_output_json_only',
   ]);
   for (const c of cases) {
     const problems = [];
@@ -165,7 +167,9 @@ function validate(cases) {
     if (!resolveTreatmentSkills(c).length) problems.push('treatment must load at least one named Harness skill');
     for (const e of c.expectations || []) {
       if (!EXPECT_TYPES.has(e.type)) problems.push(`unknown expectation type "${e.type}"`);
-      if (e.value === undefined && e.command === undefined && e.tool === undefined && e.name === undefined && e.count === undefined) {
+      if (e.type === 'final_output_json_only') {
+        problems.push(...validateExpectation(e, c.expectations.indexOf(e)));
+      } else if (e.value === undefined && e.command === undefined && e.tool === undefined && e.name === undefined && e.count === undefined) {
         problems.push(`expectation ${e.type} needs value, tool, name, count, or command`);
       }
     }
@@ -492,8 +496,10 @@ function parseTranscriptEvents(transcriptPath) {
   const events = [];
   const pendingById = new Map();
   let sequence = 0;
-  const addText = text => {
-    if (typeof text === 'string' && text) events.push({ kind: 'text', text, sequence: sequence++ });
+  // `source` separates assistant content parts from Claude's terminal `result`
+  // field, which repeats the final message and must not be joined with it.
+  const addText = (text, source = 'content') => {
+    if (typeof text === 'string' && text) events.push({ kind: 'text', text, source, sequence: sequence++ });
   };
   const addTool = (event, part) => {
     const type = eventType(event);
@@ -533,7 +539,7 @@ function parseTranscriptEvents(transcriptPath) {
     }
     if (value.message && value.message.content) visit(value.message.content, value.type !== 'user');
     if (value.content && value.content !== value.message) visit(value.content, includeText);
-    if (!value.part && typeof value.result === 'string' && includeText) addText(value.result);
+    if (!value.part && typeof value.result === 'string' && includeText) addText(value.result, 'result');
   };
   let structured = false;
   for (const line of raw.split('\n')) {
@@ -548,7 +554,7 @@ function parseTranscriptEvents(transcriptPath) {
   if (!structured) {
     try {
       const value = JSON.parse(raw);
-      if (value && typeof value.result === 'string') addText(value.result);
+      if (value && typeof value.result === 'string') addText(value.result, 'result');
       else visit(value);
     } catch { /* raw fallback handled by renderAgentTrace */ }
   }
@@ -644,6 +650,227 @@ function isExecutionLikeTraceValue(value) {
     || /(?:^|[/\\])[\w.-]+\.(?:c?js|mjs|py|sh|ps1|bat|cmd)\b/i.test(text);
 }
 
+// Harness turn-label lines are hidden bookkeeping (AGENTS.md rule 14) and are
+// ignored when judging whether the final message is JSON-only.
+const HARNESS_LABEL_LINE_RE = /^[ \t]*<!--\s*harness-label\b.*?-->[ \t\r]*$/gm;
+
+// Pure check for `final_output_json_only`: the final assistant text must be
+// exactly one JSON object, raw or inside a single fenced block, with no prose.
+function checkJsonOnlyOutput(text, requiredKeys = []) {
+  const fail = reason => ({ pass: false, reason });
+  if (typeof text !== 'string') return fail('final assistant text is missing');
+  const body = text.replace(HARNESS_LABEL_LINE_RE, '').trim();
+  if (!body) return fail('final assistant text is empty');
+  let json;
+  if (body.startsWith('```')) {
+    const fence = /^```(?:json)?[ \t]*\r?\n([\s\S]*)\r?\n```$/i.exec(body);
+    if (!fence) return fail('final output has text outside the single JSON code fence');
+    if (fence[1].includes('```')) return fail('final output contains more than one code fence');
+    json = fence[1];
+  } else if (body.startsWith('{') || body.startsWith('[')) {
+    json = body;
+  } else {
+    try { JSON.parse(body); }
+    catch { return fail('final output has prose before the JSON object'); }
+    return fail('final JSON value must be an object');
+  }
+  let value;
+  try { value = JSON.parse(json); }
+  catch (error) { return fail(`final output is not a single valid JSON value (${error.message}); prose after the JSON also fails`); }
+  if (value === null || typeof value !== 'object' || Array.isArray(value)) return fail('final JSON value must be an object');
+  const missing = requiredKeys.filter(key => !Object.prototype.hasOwnProperty.call(value, key));
+  if (missing.length) return fail(`missing required JSON keys: ${missing.join(', ')}`);
+  return { pass: true, reason: null };
+}
+
+// The final assistant message is every assistant text part after the last tool
+// event. A final turn may stream prose and JSON as separate parts, so all parts
+// count (#308). Parts can be transport chunks of one message (exact
+// concatenation) or separate complete blocks (newline-separated). Candidates
+// are returned exact-concatenation first. Claude's terminal `result` repeats
+// that message, so it is used only when the transcript has no content parts.
+function finalAssistantTextCandidates(events) {
+  let lastToolIndex = -1;
+  events.forEach((event, index) => { if (event.kind === 'tool') lastToolIndex = index; });
+  const texts = events.slice(lastToolIndex + 1).filter(event => event.kind === 'text');
+  const parts = texts.filter(event => event.source !== 'result');
+  if (!parts.length) return [texts.length ? texts[texts.length - 1].text : ''];
+  const candidates = [parts.map(event => event.text).join(''), parts.map(event => event.text).join('\n')];
+  return [...new Set(candidates)];
+}
+
+// Exact text of the final message (first candidate).
+function finalAssistantText(events) {
+  return finalAssistantTextCandidates(events)[0];
+}
+
+// --- tdd_quality_gate -------------------------------------------------------
+// A completed tool call only proves the shell returned; it does not prove the
+// gate passed (exit 1 still "completes") or that auditable evidence exists.
+// The grader therefore re-evaluates the evidence the agent pointed the gate at,
+// independently and after the run, with this repository's own evaluator.
+
+function splitCommandArgs(command) {
+  const args = [];
+  const re = /"([^"]*)"|'([^']*)'|(\S+)/g;
+  let match;
+  while ((match = re.exec(command))) args.push(match[1] ?? match[2] ?? match[3]);
+  return args;
+}
+
+// Returns { evidence, output } paths named by a quality-gate command, or null
+// when the command names no evidence file.
+function qualityGateArgs(command) {
+  const args = splitCommandArgs(String(command || ''));
+  const scriptIndex = args.findIndex(arg => /(?:^|[/\\])quality-gate\.js$/.test(arg) || arg === 'tdd:quality' || arg === 'test:tdd-quality');
+  if (scriptIndex < 0) return null;
+  const rest = args.slice(scriptIndex + 1).filter(arg => arg !== '--');
+  let evidence = null;
+  let output = null;
+  for (let index = 0; index < rest.length; index++) {
+    if (rest[index] === '--output') { output = rest[index + 1] || null; index++; continue; }
+    if (rest[index].startsWith('-')) continue;
+    if (evidence === null) evidence = rest[index];
+  }
+  return { script: args[scriptIndex], evidence, output };
+}
+
+function resolveInWorkspace(ws, file) {
+  const root = path.resolve(ws);
+  const resolved = path.resolve(root, file);
+  return resolved === root || resolved.startsWith(root + path.sep) ? resolved : null;
+}
+
+// Judges what the transcript shows the gate command did. A completed status
+// alone is not enough: OpenCode reports "completed" for a gate that exited 1.
+// Returns { status: 'pass' | 'fail' | 'inconclusive', reason }.
+// Line-anchored so prose such as "avoid npm error handling" is not an error.
+const GATE_COMMAND_ERROR_RE = /^(?:npm (?:ERR!|error)(?=\s|$)|.*Missing script:)/m;
+
+function isObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function observedGateOutcome(call) {
+  const output = typeof call.output === 'string' ? call.output : '';
+  const exitCode = typeof call.exit_code === 'number' ? call.exit_code : null;
+  if (call.is_error === true) return { status: 'fail', reason: 'the tool call reported an error' };
+  if (exitCode !== null && exitCode !== 0) return { status: 'fail', reason: `the command exited with code ${exitCode}` };
+  if (GATE_COMMAND_ERROR_RE.test(output)) return { status: 'fail', reason: 'the command output reports an npm error' };
+  if (output.includes('TDD Quality Gate: FAIL')) return { status: 'fail', reason: 'the gate printed TDD Quality Gate: FAIL' };
+  if (output.includes('TDD Quality Gate: PASS')) return { status: 'pass', reason: 'the gate printed TDD Quality Gate: PASS' };
+  if (!output && exitCode === null) return { status: 'inconclusive', reason: 'quality gate command outcome is not observable in the transcript' };
+  return { status: 'inconclusive', reason: 'the gate success line TDD Quality Gate: PASS was not observed in the command output' };
+}
+
+// Key-sorted serialisation, so two reports compare equal regardless of key order.
+function stableStringify(value) {
+  if (value === undefined) return 'undefined';
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(',')}}`;
+}
+
+// Key-sorted comparison of two reports. Returns the first top-level key that
+// differs, or undefined when they are equal. Shared by the --output file and
+// the report the gate prints to stdout.
+function firstDifferingKey(actual, expected) {
+  return [...new Set([...Object.keys(actual), ...Object.keys(expected)])].sort()
+    .find(key => stableStringify(actual[key]) !== stableStringify(expected[key]));
+}
+
+// The gate CLI prints a summary, then (without --output) the full report JSON.
+// Returns that object when the output carries a parseable one, else null.
+function embeddedReport(output) {
+  const lines = String(output || '').split(/\r?\n/);
+  const start = lines.findIndex(line => line.startsWith('{'));
+  if (start < 0) return null;
+  try {
+    const value = JSON.parse(lines.slice(start).join('\n'));
+    return isObject(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+
+// Workspace checks: the evidence the command names must be retained, valid and
+// re-evaluate to PASS, and any report the gate wrote or printed must equal the
+// fresh re-evaluation. Definitive failures here win over the observed outcome.
+function checkRetainedGateArtifacts(call, args, ws) {
+  const fail = reason => ({ pass: false, status: 'fail', reason });
+  const evidencePath = resolveInWorkspace(ws, args.evidence);
+  if (!evidencePath) return fail(`evidence file ${args.evidence} is outside the workspace`);
+  if (!fs.existsSync(evidencePath)) return fail(`evidence file ${args.evidence} was not retained in the workspace`);
+  let evidence;
+  try { evidence = JSON.parse(fs.readFileSync(evidencePath, 'utf8')); }
+  catch (error) { return fail(`evidence file ${args.evidence} is not valid JSON (${error.message})`); }
+  const { evaluateEvidence } = require(path.join(ROOT, 'tdd', 'scripts', 'quality-gate.js'));
+  const report = evaluateEvidence(evidence);
+  if (report.result !== 'PASS') {
+    const failed = Object.entries(report.mandatoryGates || {}).filter(([, ok]) => !ok).map(([name]) => name);
+    return fail(`independent re-evaluation of ${args.evidence} is ${report.result} (gate would exit 1)${failed.length ? `; failed gates: ${failed.join(', ')}` : ''}`);
+  }
+  // Compare through JSON so the fresh report has the same shape as a retained one.
+  const expected = JSON.parse(JSON.stringify(report));
+  if (args.output) {
+    const outputPath = resolveInWorkspace(ws, args.output);
+    if (!outputPath || !fs.existsSync(outputPath)) return fail(`quality report ${args.output} was not retained in the workspace`);
+    let retained;
+    try { retained = JSON.parse(fs.readFileSync(outputPath, 'utf8')); }
+    catch (error) { return fail(`quality report ${args.output} is not valid JSON (${error.message})`); }
+    if (!isObject(retained)) return fail(`retained report ${args.output} is not a JSON object`);
+    const { required } = require(path.join(ROOT, 'tdd', 'schemas', 'quality-report.schema.json'));
+    const missing = required.filter(key => !Object.prototype.hasOwnProperty.call(retained, key));
+    if (missing.length) return fail(`retained report ${args.output} is missing required keys: ${missing.join(', ')}`);
+    // The retained file must be the full report, not a marker.
+    const differing = firstDifferingKey(retained, expected);
+    if (differing !== undefined) return fail(`retained report ${args.output} differs from re-evaluation at "${differing}"`);
+  }
+  const printed = embeddedReport(call.output);
+  if (printed) {
+    const differing = firstDifferingKey(printed, expected);
+    if (differing !== undefined) return fail(`gate output report differs from re-evaluation at "${differing}"`);
+  }
+  return { pass: true, status: 'pass', reason: `quality gate re-evaluated ${args.evidence}: PASS` };
+}
+
+function verifyQualityGateRun(call, ws) {
+  const fail = reason => ({ pass: false, status: 'fail', reason });
+  const command = call && call.input && typeof call.input === 'object' ? call.input.command : undefined;
+  const args = qualityGateArgs(command);
+  if (!args) return fail('matched command is not a TDD quality-gate invocation');
+  if (args.script === 'test:tdd-quality') return fail('test:tdd-quality runs the repository self-test, not the gate on task evidence');
+  if (!args.evidence) return fail('quality gate was run without an evidence file, so no auditable report exists');
+  const artifacts = checkRetainedGateArtifacts(call, args, ws);
+  if (!artifacts.pass) return artifacts;
+  const outcome = observedGateOutcome(call);
+  if (outcome.status === 'fail') return fail(`quality gate command failed: ${outcome.reason}`);
+  if (outcome.status === 'inconclusive') return { pass: false, status: 'inconclusive', reason: outcome.reason };
+  return artifacts;
+}
+
+// The last matching attempt is the one that counts: an agent may fix evidence and
+// re-run. Attempts are graded in transcript order, including failed, denied and
+// unresolved ones, so an earlier PASS cannot satisfy the gate once a later run
+// did not complete. A completed last run is re-verified from retained evidence.
+function gradeQualityGate(expectation, parsed, ws) {
+  const attempted = gradeExecutionEvidence({ ...expectation, type: 'tool_attempted' }, parsed);
+  if (attempted.status === 'inconclusive') return attempted;
+  if (!attempted.pass) return { pass: false, status: 'fail', reason: `quality gate not executed: ${attempted.reason}` };
+  const last = attempted.observed[attempted.observed.length - 1];
+  if (last.denied === true || last.status === 'denied') {
+    return { pass: false, status: 'fail', reason: `the last quality gate run (${last.id}) was denied; an earlier run cannot satisfy the gate` };
+  }
+  if (last.status === 'failed') {
+    return { pass: false, status: 'fail', reason: `the last quality gate run (${last.id}) failed; an earlier run cannot satisfy the gate` };
+  }
+  if (last.status === 'completed') {
+    const verdict = verifyQualityGateRun(last, ws);
+    return { pass: verdict.pass, status: verdict.status, reason: verdict.reason };
+  }
+  return { pass: false, status: 'inconclusive', reason: `the last quality gate run (${last.id}) has no observable result` };
+}
+
 function grade(c, ws, transcriptPath, engine = 'auto') {
   const parsed = parseTranscriptFile(transcriptPath, engine);
   const ordered = parseTranscriptEvents(transcriptPath);
@@ -664,6 +891,8 @@ function grade(c, ws, transcriptPath, engine = 'auto') {
           status = 'fail';
           reason = 'expected a completed edit before the matching tool execution';
         }
+      } else if (e.type === 'tdd_quality_gate') {
+        ({ pass, status, reason } = gradeQualityGate(e, parsed, ws));
       } else if (e.type === 'trace_contains' || e.type === 'trace_not_contains') {
         if (e.type === 'trace_contains' && isExecutionLikeTraceValue(e.value)) {
           status = 'inconclusive';
@@ -675,6 +904,23 @@ function grade(c, ws, transcriptPath, engine = 'auto') {
           pass = e.type === 'trace_contains' ? trace.includes(e.value) : !trace.includes(e.value);
           status = pass ? 'pass' : 'fail';
         }
+      } else if (e.type === 'final_output_json_only') {
+        if (parsed.parseStatus !== 'parsed') {
+          status = 'inconclusive';
+          reason = parsed.parseError || 'transcript parser did not produce a complete structured trace';
+        } else {
+          // Neither join can hide prose: prose stays outside the JSON either way.
+          // Exact concatenation covers transport chunks of one JSON message;
+          // newline join covers separate complete blocks such as a fenced JSON
+          // block followed by a separate `<!-- harness-label ... -->` part.
+          // PASS if any candidate passes; otherwise report the exact-text reason.
+          const verdicts = finalAssistantTextCandidates(ordered.events)
+            .map(candidate => checkJsonOnlyOutput(candidate, e.required_keys || []));
+          const verdict = verdicts.find(v => v.pass) || verdicts[0];
+          pass = verdict.pass;
+          status = pass ? 'pass' : 'fail';
+          reason = verdict.reason;
+        }
       } else if (e.type === 'file_contains') {
         pass = fs.readFileSync(path.join(ws, e.path), 'utf8').includes(e.value);
         status = pass ? 'pass' : 'fail';
@@ -998,7 +1244,12 @@ module.exports = {
   extractAgentEvents,
   extractAgentTrace,
   extractSessionMetadata,
+  checkJsonOnlyOutput,
+  finalAssistantText,
+  finalAssistantTextCandidates,
   grade,
+  GATE_COMMAND_ERROR_RE,
+  qualityGateArgs,
   parseTranscriptEvents,
   pairVerdict,
   parseSimpleYaml,

package/behavioral-evals/transcript-parser.js

@@ -28,6 +28,32 @@ function clone(value) {
   return JSON.parse(JSON.stringify(value));
 }
 
+// Observable outcome of a tool result: the text the tool printed, whether the
+// host flagged it as an error, and an exit code when the engine reports one.
+// Claude tool_result blocks carry content and is_error; OpenCode carries
+// state.output and state.metadata (merged into the result as `metadata`).
+function normalizeOutput(content) {
+  if (content === null || content === undefined) return null;
+  if (typeof content === 'string') return content;
+  if (Array.isArray(content)) {
+    const texts = content
+      .map(block => {
+        if (typeof block === 'string') return block;
+        return isObject(block) && typeof block.text === 'string' ? block.text : null;
+      })
+      .filter(text => text !== null);
+    return texts.length ? texts.join('\n') : null;
+  }
+  if (isObject(content)) return JSON.stringify(content);
+  return String(content);
+}
+
+function exitCodeOf(result) {
+  const metadata = isObject(result.metadata) ? result.metadata : {};
+  const value = [result.exit_code, result.exitCode, metadata.exit].find(item => typeof item === 'number');
+  return value;
+}
+
 function makeMetadata() {
   return {
     cost: null,
@@ -144,6 +170,9 @@ function ensureCall(context, details = {}) {
     denied: false,
     status: 'attempted',
     result_count: 0,
+    output: null,
+    is_error: null,
+    exit_code: null,
     explicitId: Boolean(suppliedId),
     anonymousKey: suppliedId ? null : anonymousKey,
   };
@@ -194,6 +223,13 @@ function addToolResult(context, details = {}, result = {}, forcedStatus) {
     call.result_count += 1;
   }
   applyStatus(call, status);
+  // Keep the latest observable outcome. A result that does not report a field
+  // leaves the earlier value alone rather than erasing it.
+  const output = normalizeOutput(content);
+  if (output !== null) call.output = output;
+  if (typeof result.is_error === 'boolean') call.is_error = result.is_error;
+  const exitCode = exitCodeOf(result);
+  if (exitCode !== undefined) call.exit_code = exitCode;
   return call;
 }
 
@@ -417,6 +453,9 @@ function publicCall(call) {
     completed: call.completed,
     denied: call.denied,
     status: call.status,
+    output: call.output ?? null,
+    is_error: call.is_error ?? null,
+    exit_code: call.exit_code ?? null,
   };
 }
 
@@ -540,6 +579,10 @@ function targetMatches(call, target) {
     if (target.id !== undefined && call.id !== target.id) return false;
     if (target.name !== undefined && call.name !== target.name) return false;
     if (target.command !== undefined && (!isObject(call.input) || !commandMatches(call.input.command, target.command))) return false;
+    // command_pattern lets one expectation accept several explicit command shapes
+    // (for example npm run tdd:quality or node tdd/scripts/quality-gate.js) with
+    // arbitrary evidence arguments; shell control operators are rejected.
+    if (target.command_pattern !== undefined && (!isObject(call.input) || typeof call.input.command !== 'string' || !new RegExp(target.command_pattern).test(call.input.command))) return false;
     if (target.input_contains !== undefined && !stableJson(call.input).includes(String(target.input_contains))) return false;
     return true;
   }