harness-everything 0.47.4 → 0.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/.agents/harness-everything/project-docs.json +2 -1
  2. package/.claude-plugin/marketplace.json +1 -1
  3. package/.claude-plugin/plugin.json +1 -1
  4. package/CHANGELOG.md +33 -0
  5. package/behavioral-evals/README.md +17 -0
  6. package/behavioral-evals/case-validator.js +15 -1
  7. package/behavioral-evals/cases/baseline-tdd-workflow.yaml +4 -0
  8. package/behavioral-evals/run.js +172 -0
  9. package/behavioral-evals/transcript-parser.js +43 -0
  10. package/ci/fixtures/doc-obligations/reconciliation-s2a/run13-workflow-requirements.json +42 -0
  11. package/ci/mechanism-2s-tdd-quality.test.js +149 -6
  12. package/ci/mechanism-e1-dependency-artifact-scope.test.js +657 -0
  13. package/ci/mechanism-prb-outline-review.test.js +1 -1
  14. package/ci/test-behavioral-case-validation.js +215 -1
  15. package/code-review/SKILL.md +1 -1
  16. package/code-review/references/review-playbook.md +9 -0
  17. package/docs/doc-workflow-reconciliation.md +3 -1
  18. package/harness-everything/SKILL.md +1 -1
  19. package/harness-everything/scripts/doc-lifecycle.js +11 -3
  20. package/harness-everything/scripts/project-docs-resolver.js +12 -0
  21. package/hooks/scripts/lib/doc-reconciliation-core.js +56 -0
  22. package/hooks/scripts/lib/doc-reconciliation-producer.js +274 -12
  23. package/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
  24. package/multi-agent-workspace/SKILL.md +1 -1
  25. package/multi-agent-workspace/scripts/project-docs-resolver.js +12 -0
  26. package/opencode-plugin/plugin.json +1 -1
  27. package/package.json +1 -1
  28. package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
  29. package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-core.js +56 -0
  30. package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-producer.js +274 -12
  31. package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
  32. package/plugins/harness-everything/plugin.json +1 -1
  33. package/plugins/harness-everything/skills/code-review/SKILL.md +1 -1
  34. package/plugins/harness-everything/skills/code-review/references/review-playbook.md +9 -0
  35. package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
  36. package/plugins/harness-everything/skills/harness-everything/scripts/doc-lifecycle.js +11 -3
  37. package/plugins/harness-everything/skills/harness-everything/scripts/project-docs-resolver.js +12 -0
  38. package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
  39. package/plugins/harness-everything/skills/multi-agent-workspace/scripts/project-docs-resolver.js +12 -0
  40. package/plugins/harness-everything/skills/tdd/SKILL.md +1 -1
  41. package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json +10 -2
  42. package/plugins/harness-everything/skills/tdd/references/quality-model.md +2 -0
  43. package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json +21 -0
  44. package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json +2 -0
  45. package/plugins/harness-everything/skills/tdd/scripts/quality-gate.js +165 -1
  46. package/plugins/harness-everything/skills/to-spec/SKILL.md +1 -1
  47. package/plugins/harness-everything/skills/to-spec/scripts/project-docs-resolver.js +12 -0
  48. package/plugins/harness-everything/skills/zoom-out/SKILL.md +1 -1
  49. package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +1 -0
  50. package/sbom.cdx.json +7 -7
  51. package/tdd/SKILL.md +1 -1
  52. package/tdd/fixtures/quality-scenarios.json +10 -2
  53. package/tdd/references/quality-model.md +2 -0
  54. package/tdd/schemas/quality-evidence.schema.json +21 -0
  55. package/tdd/schemas/quality-report.schema.json +2 -0
  56. package/tdd/scripts/quality-gate.js +165 -1
  57. package/to-spec/SKILL.md +1 -1
  58. package/to-spec/scripts/project-docs-resolver.js +12 -0
  59. package/zoom-out/SKILL.md +1 -1
  60. package/zoom-out/references/circuit-breaker.md +1 -0
@@ -7,5 +7,6 @@
7
7
  "domains": ["ROUTE", "GATE", "EVID", "SONE", "DOCS", "DIST"],
8
8
  "derived": [
9
9
  { "path": "docs/governed-index.md", "generator": "node harness-everything/scripts/doc-lifecycle.js index --workspace . --config .agents/harness-everything/project-docs.json --all" }
10
- ]
10
+ ],
11
+ "inventory": { "dependencyArtifactRoots": ["node_modules", ".husky/_"] }
11
12
  }
@@ -10,7 +10,7 @@
10
10
  "name": "harness-everything",
11
11
  "source": "./",
12
12
  "description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
13
- "version": "0.47.4",
13
+ "version": "0.48.0",
14
14
  "author": {
15
15
  "name": "Miya Daniel"
16
16
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "harness-everything",
3
3
  "description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
4
- "version": "0.47.4",
4
+ "version": "0.48.0",
5
5
  "author": {
6
6
  "name": "Miya Daniel"
7
7
  },
package/CHANGELOG.md CHANGED
@@ -2,6 +2,33 @@
2
2
 
3
3
  All notable changes to this project are documented in this file.
4
4
 
5
+ ## [0.48.0](https://github.com/dyphn1/Harness-everything/compare/v0.47.5...v0.48.0) (2026-11-07)
6
+
7
+ ### Features
8
+
9
+ * **doc-reconciliation:** owner-approved dependency-artifact inventory scope ([#396](https://github.com/dyphn1/Harness-everything/issues/396)) ([f571ec6](https://github.com/dyphn1/Harness-everything/commit/f571ec68519a6af46a7bef6230cbb639152ff950)), closes [#325](https://github.com/dyphn1/Harness-everything/issues/325) [#341](https://github.com/dyphn1/Harness-everything/issues/341)
10
+
11
+ ## [Unreleased]
12
+
13
+ ### Bug Fixes
14
+
15
+ * **doc-reconciliation:** keep outline approvals durable and detect dependency-root selector overlaps conservatively, including unsupported glob syntax and parent traversal
16
+
17
+ ## [0.47.5](https://github.com/dyphn1/Harness-everything/compare/v0.47.4...v0.47.5) (2026-11-03)
18
+
19
+ ### Bug Fixes
20
+
21
+ * **behavioral-evals:** detect npm ERR! lines in quality gate output ([b599224](https://github.com/dyphn1/Harness-everything/commit/b599224ba09f27dc1f5e5c1156c40c20e57f6f52))
22
+ * **behavioral-evals:** grade tdd quality gate on independently re-evaluated evidence ([f9433ed](https://github.com/dyphn1/Harness-everything/commit/f9433ed535ba4b1882029622a31325ae564cdd50)), closes [#311](https://github.com/dyphn1/Harness-everything/issues/311)
23
+ * **behavioral-evals:** grade the last quality gate attempt, not the last completed run ([8065345](https://github.com/dyphn1/Harness-everything/commit/806534592dbc47665868e74a36b4095d44a2647b))
24
+ * **behavioral-evals:** order gate workspace checks before outcome and compare printed report ([ded48fd](https://github.com/dyphn1/Harness-everything/commit/ded48fd7221cb66c61cc58a70c1f4924f0c0fc82))
25
+ * **behavioral-evals:** require observed gate success and full retained report ([47d35d7](https://github.com/dyphn1/Harness-everything/commit/47d35d725623a4355dcc8a054347ec68832edaa2))
26
+ * **tdd:** keep source defect evidence reviewer-required ([443e966](https://github.com/dyphn1/Harness-everything/commit/443e96635c972713acd9fa76e974cb79bfa310ff))
27
+ * **tdd:** read assert.* expected from second argument; tolerate trailing semicolon ([1745ea2](https://github.com/dyphn1/Harness-everything/commit/1745ea2860a2566703af93ec23fc1ff1b00d7eef))
28
+ * **tdd:** require quoted assertion evidence for source conflict claims ([e9e1ab9](https://github.com/dyphn1/Harness-everything/commit/e9e1ab9b8170054a1ecf57b440b18124b4f99108)), closes [#312](https://github.com/dyphn1/Harness-everything/issues/312)
29
+ * **tdd:** require same-subject assertions for SOURCE_CONFLICT evidence ([d0ab25f](https://github.com/dyphn1/Harness-everything/commit/d0ab25fb810f390921acc9f38584eead95f18214))
30
+ * **tdd:** verify sourceEvidence expected values against quoted literals ([77a8ba6](https://github.com/dyphn1/Harness-everything/commit/77a8ba690de69db9852371270e86c063d30b77ed))
31
+
5
32
  ## [0.47.4](https://github.com/dyphn1/Harness-everything/compare/v0.47.3...v0.47.4) (2026-11-03)
6
33
 
7
34
  ### Bug Fixes
@@ -51,8 +78,14 @@ All notable changes to this project are documented in this file.
51
78
 
52
79
  ## [Unreleased]
53
80
 
81
+ ### Added
82
+
83
+ * **doc-reconciliation:** support a baseline-pinned, opt-in inventory exclusion for untracked artifacts under `node_modules` and `.husky/_`, with overlap guards that resolve absolute selectors against the workspace while ignoring task-pin identity metadata.
84
+
54
85
  ### Bug Fixes
55
86
 
87
+ * **code-review:** add evidence-gate counterexamples (claimed vs observed, select then judge, one witness per rule, comparable subjects and honest labels) for reviewers and author self-review.
88
+ * **zoom-out:** the author applying review fixes counts repair-loop rounds too; a decision class repeated from the previous round triggers Phase 2 before editing.
56
89
  * **doc-reconciliation:** pin the owner-approved VERI-to-EVID row mapping and exclude ordinary SubagentStop completions from quality-review candidates.
57
90
  * **doc-reconciliation:** allow uniquely identified same-root reviewers that start after the correlated draft worker stops.
58
91
 
@@ -168,6 +168,23 @@ command ran: final prose and attempted tool inputs can mention an action that
168
168
  never completed. Use structured execution expectations instead, for example
169
169
  `tool_completed` with `command: "npm test"` (command prefixes may include
170
170
  additional arguments) or `tool_denied` when denial is the behavior under test.
171
+ When one completion must accept several explicit command shapes, use
172
+ `value: {command_pattern: "<regex>"}`; the regex is matched against the whole
173
+ command, so shell control operators must be excluded explicitly.
174
+ `tdd_quality_gate` takes the same `command_pattern` but does not trust the tool
175
+ call's completed status. For the last matching run the grader first runs the
176
+ workspace checks: the evidence file must be retained, valid JSON and re-evaluate
177
+ to `PASS` with this repository's `tdd/scripts/quality-gate.js`; a `--output`
178
+ report must be present and deep-equal the fresh re-evaluation over every
179
+ top-level key (key order ignored); and a full report printed to stdout must do
180
+ the same. Missing, invalid or disagreeing artifacts are definitive `fail`s, and
181
+ a missing required key or a differing key is named in the reason. Then it reads
182
+ the observed command outcome: the call fails when it reports an error, exits
183
+ non-zero, prints an npm error line (`npm ERR!`, `npm error`, or `Missing script:`)
184
+ or prints `TDD Quality Gate: FAIL`. It passes only when the output contains the
185
+ gate's own `TDD Quality Gate: PASS` line and none of those signals. When the
186
+ transcript shows neither output nor an exit code, the result is `inconclusive`
187
+ rather than a pass. The last matching attempt counts, in transcript order: a later failed or denied run fails the gate, and a later unresolved run is inconclusive, even when an earlier run passed.
171
188
  Legacy one-object Claude JSON and incomplete streams are graded inconclusive
172
189
  for execution evidence.
173
190
 
@@ -25,6 +25,7 @@ const EXPECTATION_TYPES = new Set([
25
25
  'tool_completed',
26
26
  'tool_denied',
27
27
  'execution_evidence',
28
+ 'tdd_quality_gate',
28
29
  'final_output_json_only',
29
30
  ]);
30
31
 
@@ -119,19 +120,30 @@ function validateExpectation(expectation, index) {
119
120
  errors.push(`expectations[${index}] ${type} requires a string value`);
120
121
  }
121
122
  if (type === 'command_exit_0' && (typeof expectation.command !== 'string' || !expectation.command.trim())) errors.push(`expectations[${index}] command_exit_0 requires a non-empty command`);
123
+ if (type === 'tdd_quality_gate' && !(isPlainObject(expectation.value) && expectation.value.command_pattern !== undefined)) {
124
+ errors.push(`expectations[${index}] tdd_quality_gate requires value.command_pattern`);
125
+ }
122
126
  if (type === 'final_output_json_only' && expectation.required_keys !== undefined) {
123
127
  const keys = expectation.required_keys;
124
128
  if (!Array.isArray(keys) || keys.some(key => typeof key !== 'string' || !key.trim())) {
125
129
  errors.push(`expectations[${index}] required_keys must be an array of non-empty strings`);
126
130
  }
127
131
  }
128
- if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence'].includes(type)) {
132
+ if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence', 'tdd_quality_gate'].includes(type)) {
129
133
  if (expectation.value === undefined && expectation.tool === undefined && expectation.name === undefined && expectation.command === undefined) {
130
134
  errors.push(`expectations[${index}] ${type} requires a tool/value target`);
131
135
  }
132
136
  if (expectation.value !== undefined && !['string', 'number'].includes(typeof expectation.value) && !isPlainObject(expectation.value)) {
133
137
  errors.push(`expectations[${index}] execution target must be scalar or mapping`);
134
138
  }
139
+ if (isPlainObject(expectation.value) && expectation.value.command_pattern !== undefined) {
140
+ if (typeof expectation.value.command_pattern !== 'string' || !expectation.value.command_pattern) {
141
+ errors.push(`expectations[${index}] command_pattern must be a non-empty string`);
142
+ } else {
143
+ try { new RegExp(expectation.value.command_pattern); }
144
+ catch { errors.push(`expectations[${index}] command_pattern must be a valid regular expression`); }
145
+ }
146
+ }
135
147
  }
136
148
  return errors;
137
149
  }
@@ -210,6 +222,8 @@ function negativeControls() {
210
222
  { id: 'bad-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: '../escape.js', content: 'x' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
211
223
  { id: 'duplicate-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }, { path: 'x.js', content: '' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
212
224
  { id: 'bad-execution', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tool_executed' }] },
225
+ { id: 'quality-gate-no-pattern', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tdd_quality_gate', value: { name: 'Bash' } }] },
226
+ { id: 'bad-command-pattern', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tool_executed', value: { command_pattern: '(' } }] },
213
227
  ];
214
228
  return controls.map(control => ({ control, errors: validateCase(control) })).filter(result => result.errors.length === 0);
215
229
  }
@@ -26,3 +26,7 @@ expectations:
26
26
  path: "calculator.js"
27
27
  value: "divide"
28
28
  description: divide function exists
29
+ - type: tdd_quality_gate
30
+ value:
31
+ command_pattern: '^(?:npm (?:run|run-script) (?:(?:--silent|-s|--quiet) )*(?:tdd:quality|test:tdd-quality)|node (?:--no-warnings )?(?:[^\s;&|<>`$()]*/)?tdd/scripts/quality-gate\.js)(?: [^;&|<>`$()\n\r]*)?$'
32
+ description: TDD quality gate passed on retained evidence (independently re-evaluated; skill-core completion, not artifact pass)
@@ -150,6 +150,7 @@ function validate(cases) {
150
150
  const EXPECT_TYPES = new Set([
151
151
  'trace_contains', 'trace_not_contains', 'file_contains', 'file_not_exists', 'command_exit_0',
152
152
  'tool_call', 'tool_attempted', 'tool_completed', 'tool_executed', 'tool_denied', 'execution_evidence',
153
+ 'tdd_quality_gate',
153
154
  'final_output_json_only',
154
155
  ]);
155
156
  for (const c of cases) {
@@ -703,6 +704,173 @@ function finalAssistantText(events) {
703
704
  return finalAssistantTextCandidates(events)[0];
704
705
  }
705
706
 
707
+ // --- tdd_quality_gate -------------------------------------------------------
708
+ // A completed tool call only proves the shell returned; it does not prove the
709
+ // gate passed (exit 1 still "completes") or that auditable evidence exists.
710
+ // The grader therefore re-evaluates the evidence the agent pointed the gate at,
711
+ // independently and after the run, with this repository's own evaluator.
712
+
713
+ function splitCommandArgs(command) {
714
+ const args = [];
715
+ const re = /"([^"]*)"|'([^']*)'|(\S+)/g;
716
+ let match;
717
+ while ((match = re.exec(command))) args.push(match[1] ?? match[2] ?? match[3]);
718
+ return args;
719
+ }
720
+
721
+ // Returns { evidence, output } paths named by a quality-gate command, or null
722
+ // when the command names no evidence file.
723
+ function qualityGateArgs(command) {
724
+ const args = splitCommandArgs(String(command || ''));
725
+ const scriptIndex = args.findIndex(arg => /(?:^|[/\\])quality-gate\.js$/.test(arg) || arg === 'tdd:quality' || arg === 'test:tdd-quality');
726
+ if (scriptIndex < 0) return null;
727
+ const rest = args.slice(scriptIndex + 1).filter(arg => arg !== '--');
728
+ let evidence = null;
729
+ let output = null;
730
+ for (let index = 0; index < rest.length; index++) {
731
+ if (rest[index] === '--output') { output = rest[index + 1] || null; index++; continue; }
732
+ if (rest[index].startsWith('-')) continue;
733
+ if (evidence === null) evidence = rest[index];
734
+ }
735
+ return { script: args[scriptIndex], evidence, output };
736
+ }
737
+
738
+ function resolveInWorkspace(ws, file) {
739
+ const root = path.resolve(ws);
740
+ const resolved = path.resolve(root, file);
741
+ return resolved === root || resolved.startsWith(root + path.sep) ? resolved : null;
742
+ }
743
+
744
+ // Judges what the transcript shows the gate command did. A completed status
745
+ // alone is not enough: OpenCode reports "completed" for a gate that exited 1.
746
+ // Returns { status: 'pass' | 'fail' | 'inconclusive', reason }.
747
+ // Line-anchored so prose such as "avoid npm error handling" is not an error.
748
+ const GATE_COMMAND_ERROR_RE = /^(?:npm (?:ERR!|error)(?=\s|$)|.*Missing script:)/m;
749
+
750
+ function isObject(value) {
751
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
752
+ }
753
+
754
+ function observedGateOutcome(call) {
755
+ const output = typeof call.output === 'string' ? call.output : '';
756
+ const exitCode = typeof call.exit_code === 'number' ? call.exit_code : null;
757
+ if (call.is_error === true) return { status: 'fail', reason: 'the tool call reported an error' };
758
+ if (exitCode !== null && exitCode !== 0) return { status: 'fail', reason: `the command exited with code ${exitCode}` };
759
+ if (GATE_COMMAND_ERROR_RE.test(output)) return { status: 'fail', reason: 'the command output reports an npm error' };
760
+ if (output.includes('TDD Quality Gate: FAIL')) return { status: 'fail', reason: 'the gate printed TDD Quality Gate: FAIL' };
761
+ if (output.includes('TDD Quality Gate: PASS')) return { status: 'pass', reason: 'the gate printed TDD Quality Gate: PASS' };
762
+ if (!output && exitCode === null) return { status: 'inconclusive', reason: 'quality gate command outcome is not observable in the transcript' };
763
+ return { status: 'inconclusive', reason: 'the gate success line TDD Quality Gate: PASS was not observed in the command output' };
764
+ }
765
+
766
+ // Key-sorted serialisation, so two reports compare equal regardless of key order.
767
+ function stableStringify(value) {
768
+ if (value === undefined) return 'undefined';
769
+ if (value === null || typeof value !== 'object') return JSON.stringify(value);
770
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
771
+ return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(',')}}`;
772
+ }
773
+
774
+ // Key-sorted comparison of two reports. Returns the first top-level key that
775
+ // differs, or undefined when they are equal. Shared by the --output file and
776
+ // the report the gate prints to stdout.
777
+ function firstDifferingKey(actual, expected) {
778
+ return [...new Set([...Object.keys(actual), ...Object.keys(expected)])].sort()
779
+ .find(key => stableStringify(actual[key]) !== stableStringify(expected[key]));
780
+ }
781
+
782
+ // The gate CLI prints a summary, then (without --output) the full report JSON.
783
+ // Returns that object when the output carries a parseable one, else null.
784
+ function embeddedReport(output) {
785
+ const lines = String(output || '').split(/\r?\n/);
786
+ const start = lines.findIndex(line => line.startsWith('{'));
787
+ if (start < 0) return null;
788
+ try {
789
+ const value = JSON.parse(lines.slice(start).join('\n'));
790
+ return isObject(value) ? value : null;
791
+ } catch {
792
+ return null;
793
+ }
794
+ }
795
+
796
+ // Workspace checks: the evidence the command names must be retained, valid and
797
+ // re-evaluate to PASS, and any report the gate wrote or printed must equal the
798
+ // fresh re-evaluation. Definitive failures here win over the observed outcome.
799
+ function checkRetainedGateArtifacts(call, args, ws) {
800
+ const fail = reason => ({ pass: false, status: 'fail', reason });
801
+ const evidencePath = resolveInWorkspace(ws, args.evidence);
802
+ if (!evidencePath) return fail(`evidence file ${args.evidence} is outside the workspace`);
803
+ if (!fs.existsSync(evidencePath)) return fail(`evidence file ${args.evidence} was not retained in the workspace`);
804
+ let evidence;
805
+ try { evidence = JSON.parse(fs.readFileSync(evidencePath, 'utf8')); }
806
+ catch (error) { return fail(`evidence file ${args.evidence} is not valid JSON (${error.message})`); }
807
+ const { evaluateEvidence } = require(path.join(ROOT, 'tdd', 'scripts', 'quality-gate.js'));
808
+ const report = evaluateEvidence(evidence);
809
+ if (report.result !== 'PASS') {
810
+ const failed = Object.entries(report.mandatoryGates || {}).filter(([, ok]) => !ok).map(([name]) => name);
811
+ return fail(`independent re-evaluation of ${args.evidence} is ${report.result} (gate would exit 1)${failed.length ? `; failed gates: ${failed.join(', ')}` : ''}`);
812
+ }
813
+ // Compare through JSON so the fresh report has the same shape as a retained one.
814
+ const expected = JSON.parse(JSON.stringify(report));
815
+ if (args.output) {
816
+ const outputPath = resolveInWorkspace(ws, args.output);
817
+ if (!outputPath || !fs.existsSync(outputPath)) return fail(`quality report ${args.output} was not retained in the workspace`);
818
+ let retained;
819
+ try { retained = JSON.parse(fs.readFileSync(outputPath, 'utf8')); }
820
+ catch (error) { return fail(`quality report ${args.output} is not valid JSON (${error.message})`); }
821
+ if (!isObject(retained)) return fail(`retained report ${args.output} is not a JSON object`);
822
+ const { required } = require(path.join(ROOT, 'tdd', 'schemas', 'quality-report.schema.json'));
823
+ const missing = required.filter(key => !Object.prototype.hasOwnProperty.call(retained, key));
824
+ if (missing.length) return fail(`retained report ${args.output} is missing required keys: ${missing.join(', ')}`);
825
+ // The retained file must be the full report, not a marker.
826
+ const differing = firstDifferingKey(retained, expected);
827
+ if (differing !== undefined) return fail(`retained report ${args.output} differs from re-evaluation at "${differing}"`);
828
+ }
829
+ const printed = embeddedReport(call.output);
830
+ if (printed) {
831
+ const differing = firstDifferingKey(printed, expected);
832
+ if (differing !== undefined) return fail(`gate output report differs from re-evaluation at "${differing}"`);
833
+ }
834
+ return { pass: true, status: 'pass', reason: `quality gate re-evaluated ${args.evidence}: PASS` };
835
+ }
836
+
837
+ function verifyQualityGateRun(call, ws) {
838
+ const fail = reason => ({ pass: false, status: 'fail', reason });
839
+ const command = call && call.input && typeof call.input === 'object' ? call.input.command : undefined;
840
+ const args = qualityGateArgs(command);
841
+ if (!args) return fail('matched command is not a TDD quality-gate invocation');
842
+ if (args.script === 'test:tdd-quality') return fail('test:tdd-quality runs the repository self-test, not the gate on task evidence');
843
+ if (!args.evidence) return fail('quality gate was run without an evidence file, so no auditable report exists');
844
+ const artifacts = checkRetainedGateArtifacts(call, args, ws);
845
+ if (!artifacts.pass) return artifacts;
846
+ const outcome = observedGateOutcome(call);
847
+ if (outcome.status === 'fail') return fail(`quality gate command failed: ${outcome.reason}`);
848
+ if (outcome.status === 'inconclusive') return { pass: false, status: 'inconclusive', reason: outcome.reason };
849
+ return artifacts;
850
+ }
851
+
852
+ // The last matching attempt is the one that counts: an agent may fix evidence and
853
+ // re-run. Attempts are graded in transcript order, including failed, denied and
854
+ // unresolved ones, so an earlier PASS cannot satisfy the gate once a later run
855
+ // did not complete. A completed last run is re-verified from retained evidence.
856
+ function gradeQualityGate(expectation, parsed, ws) {
857
+ const attempted = gradeExecutionEvidence({ ...expectation, type: 'tool_attempted' }, parsed);
858
+ if (attempted.status === 'inconclusive') return attempted;
859
+ if (!attempted.pass) return { pass: false, status: 'fail', reason: `quality gate not executed: ${attempted.reason}` };
860
+ const last = attempted.observed[attempted.observed.length - 1];
861
+ if (last.denied === true || last.status === 'denied') {
862
+ return { pass: false, status: 'fail', reason: `the last quality gate run (${last.id}) was denied; an earlier run cannot satisfy the gate` };
863
+ }
864
+ if (last.status === 'failed') {
865
+ return { pass: false, status: 'fail', reason: `the last quality gate run (${last.id}) failed; an earlier run cannot satisfy the gate` };
866
+ }
867
+ if (last.status === 'completed') {
868
+ const verdict = verifyQualityGateRun(last, ws);
869
+ return { pass: verdict.pass, status: verdict.status, reason: verdict.reason };
870
+ }
871
+ return { pass: false, status: 'inconclusive', reason: `the last quality gate run (${last.id}) has no observable result` };
872
+ }
873
+
706
874
  function grade(c, ws, transcriptPath, engine = 'auto') {
707
875
  const parsed = parseTranscriptFile(transcriptPath, engine);
708
876
  const ordered = parseTranscriptEvents(transcriptPath);
@@ -723,6 +891,8 @@ function grade(c, ws, transcriptPath, engine = 'auto') {
723
891
  status = 'fail';
724
892
  reason = 'expected a completed edit before the matching tool execution';
725
893
  }
894
+ } else if (e.type === 'tdd_quality_gate') {
895
+ ({ pass, status, reason } = gradeQualityGate(e, parsed, ws));
726
896
  } else if (e.type === 'trace_contains' || e.type === 'trace_not_contains') {
727
897
  if (e.type === 'trace_contains' && isExecutionLikeTraceValue(e.value)) {
728
898
  status = 'inconclusive';
@@ -1078,6 +1248,8 @@ module.exports = {
1078
1248
  finalAssistantText,
1079
1249
  finalAssistantTextCandidates,
1080
1250
  grade,
1251
+ GATE_COMMAND_ERROR_RE,
1252
+ qualityGateArgs,
1081
1253
  parseTranscriptEvents,
1082
1254
  pairVerdict,
1083
1255
  parseSimpleYaml,
@@ -28,6 +28,32 @@ function clone(value) {
28
28
  return JSON.parse(JSON.stringify(value));
29
29
  }
30
30
 
31
+ // Observable outcome of a tool result: the text the tool printed, whether the
32
+ // host flagged it as an error, and an exit code when the engine reports one.
33
+ // Claude tool_result blocks carry content and is_error; OpenCode carries
34
+ // state.output and state.metadata (merged into the result as `metadata`).
35
+ function normalizeOutput(content) {
36
+ if (content === null || content === undefined) return null;
37
+ if (typeof content === 'string') return content;
38
+ if (Array.isArray(content)) {
39
+ const texts = content
40
+ .map(block => {
41
+ if (typeof block === 'string') return block;
42
+ return isObject(block) && typeof block.text === 'string' ? block.text : null;
43
+ })
44
+ .filter(text => text !== null);
45
+ return texts.length ? texts.join('\n') : null;
46
+ }
47
+ if (isObject(content)) return JSON.stringify(content);
48
+ return String(content);
49
+ }
50
+
51
+ function exitCodeOf(result) {
52
+ const metadata = isObject(result.metadata) ? result.metadata : {};
53
+ const value = [result.exit_code, result.exitCode, metadata.exit].find(item => typeof item === 'number');
54
+ return value;
55
+ }
56
+
31
57
  function makeMetadata() {
32
58
  return {
33
59
  cost: null,
@@ -144,6 +170,9 @@ function ensureCall(context, details = {}) {
144
170
  denied: false,
145
171
  status: 'attempted',
146
172
  result_count: 0,
173
+ output: null,
174
+ is_error: null,
175
+ exit_code: null,
147
176
  explicitId: Boolean(suppliedId),
148
177
  anonymousKey: suppliedId ? null : anonymousKey,
149
178
  };
@@ -194,6 +223,13 @@ function addToolResult(context, details = {}, result = {}, forcedStatus) {
194
223
  call.result_count += 1;
195
224
  }
196
225
  applyStatus(call, status);
226
+ // Keep the latest observable outcome. A result that does not report a field
227
+ // leaves the earlier value alone rather than erasing it.
228
+ const output = normalizeOutput(content);
229
+ if (output !== null) call.output = output;
230
+ if (typeof result.is_error === 'boolean') call.is_error = result.is_error;
231
+ const exitCode = exitCodeOf(result);
232
+ if (exitCode !== undefined) call.exit_code = exitCode;
197
233
  return call;
198
234
  }
199
235
 
@@ -417,6 +453,9 @@ function publicCall(call) {
417
453
  completed: call.completed,
418
454
  denied: call.denied,
419
455
  status: call.status,
456
+ output: call.output ?? null,
457
+ is_error: call.is_error ?? null,
458
+ exit_code: call.exit_code ?? null,
420
459
  };
421
460
  }
422
461
 
@@ -540,6 +579,10 @@ function targetMatches(call, target) {
540
579
  if (target.id !== undefined && call.id !== target.id) return false;
541
580
  if (target.name !== undefined && call.name !== target.name) return false;
542
581
  if (target.command !== undefined && (!isObject(call.input) || !commandMatches(call.input.command, target.command))) return false;
582
+ // command_pattern lets one expectation accept several explicit command shapes
583
+ // (for example npm run tdd:quality or node tdd/scripts/quality-gate.js) with
584
+ // arbitrary evidence arguments; shell control operators are rejected.
585
+ if (target.command_pattern !== undefined && (!isObject(call.input) || typeof call.input.command !== 'string' || !new RegExp(target.command_pattern).test(call.input.command))) return false;
543
586
  if (target.input_contains !== undefined && !stableJson(call.input).includes(String(target.input_contains))) return false;
544
587
  return true;
545
588
  }
@@ -0,0 +1,42 @@
1
+ [
2
+ {
3
+ "id": "adr-evid",
4
+ "order": 1,
5
+ "summary": "Draft the row-17 EVID-001 verification-evidence ADR.",
6
+ "acceptance": "A real proposed O is independently reviewed before the draft D; D is a strict descendant of O and changes only the ADR.",
7
+ "suggestedSkills": [],
8
+ "stepType": "doc-change",
9
+ "status": "pending",
10
+ "evidence": null,
11
+ "reasonCode": null,
12
+ "updatedAt": null,
13
+ "requiredBindings": [],
14
+ "optionalBindings": []
15
+ },
16
+ {
17
+ "id": "verify-row17",
18
+ "order": 2,
19
+ "summary": "Generate the governed index and verify the final ADR through one combined registered check and independent review.",
20
+ "acceptance": "All four combined-check capabilities and the ADR-only final review are captured before normal Stop reconciliation.",
21
+ "suggestedSkills": [],
22
+ "stepType": "verification",
23
+ "status": "pending",
24
+ "evidence": null,
25
+ "reasonCode": null,
26
+ "updatedAt": null,
27
+ "requiredBindings": [
28
+ {
29
+ "id": "verification-loop",
30
+ "required": true,
31
+ "status": "pending",
32
+ "path": "verification-loop/SKILL.md",
33
+ "availability": "available",
34
+ "resolvedPath": "/private/tmp/row17/plugin-f/verification-loop/SKILL.md",
35
+ "evidence": null,
36
+ "reasonCode": null,
37
+ "updatedAt": null
38
+ }
39
+ ],
40
+ "optionalBindings": []
41
+ }
42
+ ]