harness-everything 0.47.4 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/harness-everything/project-docs.json +2 -1
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +33 -0
- package/behavioral-evals/README.md +17 -0
- package/behavioral-evals/case-validator.js +15 -1
- package/behavioral-evals/cases/baseline-tdd-workflow.yaml +4 -0
- package/behavioral-evals/run.js +172 -0
- package/behavioral-evals/transcript-parser.js +43 -0
- package/ci/fixtures/doc-obligations/reconciliation-s2a/run13-workflow-requirements.json +42 -0
- package/ci/mechanism-2s-tdd-quality.test.js +149 -6
- package/ci/mechanism-e1-dependency-artifact-scope.test.js +657 -0
- package/ci/mechanism-prb-outline-review.test.js +1 -1
- package/ci/test-behavioral-case-validation.js +215 -1
- package/code-review/SKILL.md +1 -1
- package/code-review/references/review-playbook.md +9 -0
- package/docs/doc-workflow-reconciliation.md +3 -1
- package/harness-everything/SKILL.md +1 -1
- package/harness-everything/scripts/doc-lifecycle.js +11 -3
- package/harness-everything/scripts/project-docs-resolver.js +12 -0
- package/hooks/scripts/lib/doc-reconciliation-core.js +56 -0
- package/hooks/scripts/lib/doc-reconciliation-producer.js +274 -12
- package/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
- package/multi-agent-workspace/SKILL.md +1 -1
- package/multi-agent-workspace/scripts/project-docs-resolver.js +12 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-core.js +56 -0
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-producer.js +274 -12
- package/plugins/harness-everything/hooks/scripts/lib/doc-reconciliation-provider.js +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/code-review/SKILL.md +1 -1
- package/plugins/harness-everything/skills/code-review/references/review-playbook.md +9 -0
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +1 -1
- package/plugins/harness-everything/skills/harness-everything/scripts/doc-lifecycle.js +11 -3
- package/plugins/harness-everything/skills/harness-everything/scripts/project-docs-resolver.js +12 -0
- package/plugins/harness-everything/skills/multi-agent-workspace/SKILL.md +1 -1
- package/plugins/harness-everything/skills/multi-agent-workspace/scripts/project-docs-resolver.js +12 -0
- package/plugins/harness-everything/skills/tdd/SKILL.md +1 -1
- package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json +10 -2
- package/plugins/harness-everything/skills/tdd/references/quality-model.md +2 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json +21 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json +2 -0
- package/plugins/harness-everything/skills/tdd/scripts/quality-gate.js +165 -1
- package/plugins/harness-everything/skills/to-spec/SKILL.md +1 -1
- package/plugins/harness-everything/skills/to-spec/scripts/project-docs-resolver.js +12 -0
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +1 -1
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +1 -0
- package/sbom.cdx.json +7 -7
- package/tdd/SKILL.md +1 -1
- package/tdd/fixtures/quality-scenarios.json +10 -2
- package/tdd/references/quality-model.md +2 -0
- package/tdd/schemas/quality-evidence.schema.json +21 -0
- package/tdd/schemas/quality-report.schema.json +2 -0
- package/tdd/scripts/quality-gate.js +165 -1
- package/to-spec/SKILL.md +1 -1
- package/to-spec/scripts/project-docs-resolver.js +12 -0
- package/zoom-out/SKILL.md +1 -1
- package/zoom-out/references/circuit-breaker.md +1 -0
|
@@ -7,5 +7,6 @@
|
|
|
7
7
|
"domains": ["ROUTE", "GATE", "EVID", "SONE", "DOCS", "DIST"],
|
|
8
8
|
"derived": [
|
|
9
9
|
{ "path": "docs/governed-index.md", "generator": "node harness-everything/scripts/doc-lifecycle.js index --workspace . --config .agents/harness-everything/project-docs.json --all" }
|
|
10
|
-
]
|
|
10
|
+
],
|
|
11
|
+
"inventory": { "dependencyArtifactRoots": ["node_modules", ".husky/_"] }
|
|
11
12
|
}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "harness-everything",
|
|
11
11
|
"source": "./",
|
|
12
12
|
"description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
|
|
13
|
-
"version": "0.
|
|
13
|
+
"version": "0.48.0",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Miya Daniel"
|
|
16
16
|
},
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "harness-everything",
|
|
3
3
|
"description": "Harness OS: an orchestrated agent operating system — tier routing, circuit breaker guardrails, verification gates, WAL session memory, and self-evolving workspace rules.",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.48.0",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Miya Daniel"
|
|
7
7
|
},
|
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,33 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.48.0](https://github.com/dyphn1/Harness-everything/compare/v0.47.5...v0.48.0) (2026-11-07)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **doc-reconciliation:** owner-approved dependency-artifact inventory scope ([#396](https://github.com/dyphn1/Harness-everything/issues/396)) ([f571ec6](https://github.com/dyphn1/Harness-everything/commit/f571ec68519a6af46a7bef6230cbb639152ff950)), closes [#325](https://github.com/dyphn1/Harness-everything/issues/325) [#341](https://github.com/dyphn1/Harness-everything/issues/341)
|
|
10
|
+
|
|
11
|
+
## [Unreleased]
|
|
12
|
+
|
|
13
|
+
### Bug Fixes
|
|
14
|
+
|
|
15
|
+
* **doc-reconciliation:** keep outline approvals durable and detect dependency-root selector overlaps conservatively, including unsupported glob syntax and parent traversal
|
|
16
|
+
|
|
17
|
+
## [0.47.5](https://github.com/dyphn1/Harness-everything/compare/v0.47.4...v0.47.5) (2026-11-03)
|
|
18
|
+
|
|
19
|
+
### Bug Fixes
|
|
20
|
+
|
|
21
|
+
* **behavioral-evals:** detect npm ERR! lines in quality gate output ([b599224](https://github.com/dyphn1/Harness-everything/commit/b599224ba09f27dc1f5e5c1156c40c20e57f6f52))
|
|
22
|
+
* **behavioral-evals:** grade tdd quality gate on independently re-evaluated evidence ([f9433ed](https://github.com/dyphn1/Harness-everything/commit/f9433ed535ba4b1882029622a31325ae564cdd50)), closes [#311](https://github.com/dyphn1/Harness-everything/issues/311)
|
|
23
|
+
* **behavioral-evals:** grade the last quality gate attempt, not the last completed run ([8065345](https://github.com/dyphn1/Harness-everything/commit/806534592dbc47665868e74a36b4095d44a2647b))
|
|
24
|
+
* **behavioral-evals:** order gate workspace checks before outcome and compare printed report ([ded48fd](https://github.com/dyphn1/Harness-everything/commit/ded48fd7221cb66c61cc58a70c1f4924f0c0fc82))
|
|
25
|
+
* **behavioral-evals:** require observed gate success and full retained report ([47d35d7](https://github.com/dyphn1/Harness-everything/commit/47d35d725623a4355dcc8a054347ec68832edaa2))
|
|
26
|
+
* **tdd:** keep source defect evidence reviewer-required ([443e966](https://github.com/dyphn1/Harness-everything/commit/443e96635c972713acd9fa76e974cb79bfa310ff))
|
|
27
|
+
* **tdd:** read assert.* expected from second argument; tolerate trailing semicolon ([1745ea2](https://github.com/dyphn1/Harness-everything/commit/1745ea2860a2566703af93ec23fc1ff1b00d7eef))
|
|
28
|
+
* **tdd:** require quoted assertion evidence for source conflict claims ([e9e1ab9](https://github.com/dyphn1/Harness-everything/commit/e9e1ab9b8170054a1ecf57b440b18124b4f99108)), closes [#312](https://github.com/dyphn1/Harness-everything/issues/312)
|
|
29
|
+
* **tdd:** require same-subject assertions for SOURCE_CONFLICT evidence ([d0ab25f](https://github.com/dyphn1/Harness-everything/commit/d0ab25fb810f390921acc9f38584eead95f18214))
|
|
30
|
+
* **tdd:** verify sourceEvidence expected values against quoted literals ([77a8ba6](https://github.com/dyphn1/Harness-everything/commit/77a8ba690de69db9852371270e86c063d30b77ed))
|
|
31
|
+
|
|
5
32
|
## [0.47.4](https://github.com/dyphn1/Harness-everything/compare/v0.47.3...v0.47.4) (2026-11-03)
|
|
6
33
|
|
|
7
34
|
### Bug Fixes
|
|
@@ -51,8 +78,14 @@ All notable changes to this project are documented in this file.
|
|
|
51
78
|
|
|
52
79
|
## [Unreleased]
|
|
53
80
|
|
|
81
|
+
### Added
|
|
82
|
+
|
|
83
|
+
* **doc-reconciliation:** support a baseline-pinned, opt-in inventory exclusion for untracked artifacts under `node_modules` and `.husky/_`, with overlap guards that resolve absolute selectors against the workspace while ignoring task-pin identity metadata.
|
|
84
|
+
|
|
54
85
|
### Bug Fixes
|
|
55
86
|
|
|
87
|
+
* **code-review:** add evidence-gate counterexamples (claimed vs observed, select then judge, one witness per rule, comparable subjects and honest labels) for reviewers and author self-review.
|
|
88
|
+
* **zoom-out:** the author applying review fixes counts repair-loop rounds too; a decision class repeated from the previous round triggers Phase 2 before editing.
|
|
56
89
|
* **doc-reconciliation:** pin the owner-approved VERI-to-EVID row mapping and exclude ordinary SubagentStop completions from quality-review candidates.
|
|
57
90
|
* **doc-reconciliation:** allow uniquely identified same-root reviewers that start after the correlated draft worker stops.
|
|
58
91
|
|
|
@@ -168,6 +168,23 @@ command ran: final prose and attempted tool inputs can mention an action that
|
|
|
168
168
|
never completed. Use structured execution expectations instead, for example
|
|
169
169
|
`tool_completed` with `command: "npm test"` (command prefixes may include
|
|
170
170
|
additional arguments) or `tool_denied` when denial is the behavior under test.
|
|
171
|
+
When one completion must accept several explicit command shapes, use
|
|
172
|
+
`value: {command_pattern: "<regex>"}`; the regex is matched against the whole
|
|
173
|
+
command, so shell control operators must be excluded explicitly.
|
|
174
|
+
`tdd_quality_gate` takes the same `command_pattern` but does not trust the tool
|
|
175
|
+
call's completed status. For the last matching run the grader first runs the
|
|
176
|
+
workspace checks: the evidence file must be retained, valid JSON and re-evaluate
|
|
177
|
+
to `PASS` with this repository's `tdd/scripts/quality-gate.js`; a `--output`
|
|
178
|
+
report must be present and deep-equal the fresh re-evaluation over every
|
|
179
|
+
top-level key (key order ignored); and a full report printed to stdout must do
|
|
180
|
+
the same. Missing, invalid or disagreeing artifacts are definitive `fail`s, and
|
|
181
|
+
a missing required key or a differing key is named in the reason. Then it reads
|
|
182
|
+
the observed command outcome: the call fails when it reports an error, exits
|
|
183
|
+
non-zero, prints an npm error line (`npm ERR!`, `npm error`, or `Missing script:`)
|
|
184
|
+
or prints `TDD Quality Gate: FAIL`. It passes only when the output contains the
|
|
185
|
+
gate's own `TDD Quality Gate: PASS` line and none of those signals. When the
|
|
186
|
+
transcript shows neither output nor an exit code, the result is `inconclusive`
|
|
187
|
+
rather than a pass. The last matching attempt counts, in transcript order: a later failed or denied run fails the gate, and a later unresolved run is inconclusive, even when an earlier run passed.
|
|
171
188
|
Legacy one-object Claude JSON and incomplete streams are graded inconclusive
|
|
172
189
|
for execution evidence.
|
|
173
190
|
|
|
@@ -25,6 +25,7 @@ const EXPECTATION_TYPES = new Set([
|
|
|
25
25
|
'tool_completed',
|
|
26
26
|
'tool_denied',
|
|
27
27
|
'execution_evidence',
|
|
28
|
+
'tdd_quality_gate',
|
|
28
29
|
'final_output_json_only',
|
|
29
30
|
]);
|
|
30
31
|
|
|
@@ -119,19 +120,30 @@ function validateExpectation(expectation, index) {
|
|
|
119
120
|
errors.push(`expectations[${index}] ${type} requires a string value`);
|
|
120
121
|
}
|
|
121
122
|
if (type === 'command_exit_0' && (typeof expectation.command !== 'string' || !expectation.command.trim())) errors.push(`expectations[${index}] command_exit_0 requires a non-empty command`);
|
|
123
|
+
if (type === 'tdd_quality_gate' && !(isPlainObject(expectation.value) && expectation.value.command_pattern !== undefined)) {
|
|
124
|
+
errors.push(`expectations[${index}] tdd_quality_gate requires value.command_pattern`);
|
|
125
|
+
}
|
|
122
126
|
if (type === 'final_output_json_only' && expectation.required_keys !== undefined) {
|
|
123
127
|
const keys = expectation.required_keys;
|
|
124
128
|
if (!Array.isArray(keys) || keys.some(key => typeof key !== 'string' || !key.trim())) {
|
|
125
129
|
errors.push(`expectations[${index}] required_keys must be an array of non-empty strings`);
|
|
126
130
|
}
|
|
127
131
|
}
|
|
128
|
-
if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence'].includes(type)) {
|
|
132
|
+
if (['tool_attempted', 'tool_executed', 'tool_completed', 'tool_denied', 'execution_evidence', 'tdd_quality_gate'].includes(type)) {
|
|
129
133
|
if (expectation.value === undefined && expectation.tool === undefined && expectation.name === undefined && expectation.command === undefined) {
|
|
130
134
|
errors.push(`expectations[${index}] ${type} requires a tool/value target`);
|
|
131
135
|
}
|
|
132
136
|
if (expectation.value !== undefined && !['string', 'number'].includes(typeof expectation.value) && !isPlainObject(expectation.value)) {
|
|
133
137
|
errors.push(`expectations[${index}] execution target must be scalar or mapping`);
|
|
134
138
|
}
|
|
139
|
+
if (isPlainObject(expectation.value) && expectation.value.command_pattern !== undefined) {
|
|
140
|
+
if (typeof expectation.value.command_pattern !== 'string' || !expectation.value.command_pattern) {
|
|
141
|
+
errors.push(`expectations[${index}] command_pattern must be a non-empty string`);
|
|
142
|
+
} else {
|
|
143
|
+
try { new RegExp(expectation.value.command_pattern); }
|
|
144
|
+
catch { errors.push(`expectations[${index}] command_pattern must be a valid regular expression`); }
|
|
145
|
+
}
|
|
146
|
+
}
|
|
135
147
|
}
|
|
136
148
|
return errors;
|
|
137
149
|
}
|
|
@@ -210,6 +222,8 @@ function negativeControls() {
|
|
|
210
222
|
{ id: 'bad-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: '../escape.js', content: 'x' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
|
|
211
223
|
{ id: 'duplicate-path', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }, { path: 'x.js', content: '' }] }, expectations: [{ type: 'trace_contains', value: 'x' }] },
|
|
212
224
|
{ id: 'bad-execution', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tool_executed' }] },
|
|
225
|
+
{ id: 'quality-gate-no-pattern', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tdd_quality_gate', value: { name: 'Bash' } }] },
|
|
226
|
+
{ id: 'bad-command-pattern', prompt: 'x', max_turns: 1, fixture: { files: [{ path: 'x.js', content: '' }] }, expectations: [{ type: 'tool_executed', value: { command_pattern: '(' } }] },
|
|
213
227
|
];
|
|
214
228
|
return controls.map(control => ({ control, errors: validateCase(control) })).filter(result => result.errors.length === 0);
|
|
215
229
|
}
|
|
@@ -26,3 +26,7 @@ expectations:
|
|
|
26
26
|
path: "calculator.js"
|
|
27
27
|
value: "divide"
|
|
28
28
|
description: divide function exists
|
|
29
|
+
- type: tdd_quality_gate
|
|
30
|
+
value:
|
|
31
|
+
command_pattern: '^(?:npm (?:run|run-script) (?:(?:--silent|-s|--quiet) )*(?:tdd:quality|test:tdd-quality)|node (?:--no-warnings )?(?:[^\s;&|<>`$()]*/)?tdd/scripts/quality-gate\.js)(?: [^;&|<>`$()\n\r]*)?$'
|
|
32
|
+
description: TDD quality gate passed on retained evidence (independently re-evaluated; skill-core completion, not artifact pass)
|
package/behavioral-evals/run.js
CHANGED
|
@@ -150,6 +150,7 @@ function validate(cases) {
|
|
|
150
150
|
const EXPECT_TYPES = new Set([
|
|
151
151
|
'trace_contains', 'trace_not_contains', 'file_contains', 'file_not_exists', 'command_exit_0',
|
|
152
152
|
'tool_call', 'tool_attempted', 'tool_completed', 'tool_executed', 'tool_denied', 'execution_evidence',
|
|
153
|
+
'tdd_quality_gate',
|
|
153
154
|
'final_output_json_only',
|
|
154
155
|
]);
|
|
155
156
|
for (const c of cases) {
|
|
@@ -703,6 +704,173 @@ function finalAssistantText(events) {
|
|
|
703
704
|
return finalAssistantTextCandidates(events)[0];
|
|
704
705
|
}
|
|
705
706
|
|
|
707
|
+
// --- tdd_quality_gate -------------------------------------------------------
|
|
708
|
+
// A completed tool call only proves the shell returned; it does not prove the
|
|
709
|
+
// gate passed (exit 1 still "completes") or that auditable evidence exists.
|
|
710
|
+
// The grader therefore re-evaluates the evidence the agent pointed the gate at,
|
|
711
|
+
// independently and after the run, with this repository's own evaluator.
|
|
712
|
+
|
|
713
|
+
function splitCommandArgs(command) {
|
|
714
|
+
const args = [];
|
|
715
|
+
const re = /"([^"]*)"|'([^']*)'|(\S+)/g;
|
|
716
|
+
let match;
|
|
717
|
+
while ((match = re.exec(command))) args.push(match[1] ?? match[2] ?? match[3]);
|
|
718
|
+
return args;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
// Returns { evidence, output } paths named by a quality-gate command, or null
|
|
722
|
+
// when the command names no evidence file.
|
|
723
|
+
function qualityGateArgs(command) {
|
|
724
|
+
const args = splitCommandArgs(String(command || ''));
|
|
725
|
+
const scriptIndex = args.findIndex(arg => /(?:^|[/\\])quality-gate\.js$/.test(arg) || arg === 'tdd:quality' || arg === 'test:tdd-quality');
|
|
726
|
+
if (scriptIndex < 0) return null;
|
|
727
|
+
const rest = args.slice(scriptIndex + 1).filter(arg => arg !== '--');
|
|
728
|
+
let evidence = null;
|
|
729
|
+
let output = null;
|
|
730
|
+
for (let index = 0; index < rest.length; index++) {
|
|
731
|
+
if (rest[index] === '--output') { output = rest[index + 1] || null; index++; continue; }
|
|
732
|
+
if (rest[index].startsWith('-')) continue;
|
|
733
|
+
if (evidence === null) evidence = rest[index];
|
|
734
|
+
}
|
|
735
|
+
return { script: args[scriptIndex], evidence, output };
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
function resolveInWorkspace(ws, file) {
|
|
739
|
+
const root = path.resolve(ws);
|
|
740
|
+
const resolved = path.resolve(root, file);
|
|
741
|
+
return resolved === root || resolved.startsWith(root + path.sep) ? resolved : null;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// Judges what the transcript shows the gate command did. A completed status
|
|
745
|
+
// alone is not enough: OpenCode reports "completed" for a gate that exited 1.
|
|
746
|
+
// Returns { status: 'pass' | 'fail' | 'inconclusive', reason }.
|
|
747
|
+
// Line-anchored so prose such as "avoid npm error handling" is not an error.
|
|
748
|
+
const GATE_COMMAND_ERROR_RE = /^(?:npm (?:ERR!|error)(?=\s|$)|.*Missing script:)/m;
|
|
749
|
+
|
|
750
|
+
function isObject(value) {
|
|
751
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
function observedGateOutcome(call) {
|
|
755
|
+
const output = typeof call.output === 'string' ? call.output : '';
|
|
756
|
+
const exitCode = typeof call.exit_code === 'number' ? call.exit_code : null;
|
|
757
|
+
if (call.is_error === true) return { status: 'fail', reason: 'the tool call reported an error' };
|
|
758
|
+
if (exitCode !== null && exitCode !== 0) return { status: 'fail', reason: `the command exited with code ${exitCode}` };
|
|
759
|
+
if (GATE_COMMAND_ERROR_RE.test(output)) return { status: 'fail', reason: 'the command output reports an npm error' };
|
|
760
|
+
if (output.includes('TDD Quality Gate: FAIL')) return { status: 'fail', reason: 'the gate printed TDD Quality Gate: FAIL' };
|
|
761
|
+
if (output.includes('TDD Quality Gate: PASS')) return { status: 'pass', reason: 'the gate printed TDD Quality Gate: PASS' };
|
|
762
|
+
if (!output && exitCode === null) return { status: 'inconclusive', reason: 'quality gate command outcome is not observable in the transcript' };
|
|
763
|
+
return { status: 'inconclusive', reason: 'the gate success line TDD Quality Gate: PASS was not observed in the command output' };
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
// Key-sorted serialisation, so two reports compare equal regardless of key order.
|
|
767
|
+
function stableStringify(value) {
|
|
768
|
+
if (value === undefined) return 'undefined';
|
|
769
|
+
if (value === null || typeof value !== 'object') return JSON.stringify(value);
|
|
770
|
+
if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
|
|
771
|
+
return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(',')}}`;
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
// Key-sorted comparison of two reports. Returns the first top-level key that
|
|
775
|
+
// differs, or undefined when they are equal. Shared by the --output file and
|
|
776
|
+
// the report the gate prints to stdout.
|
|
777
|
+
function firstDifferingKey(actual, expected) {
|
|
778
|
+
return [...new Set([...Object.keys(actual), ...Object.keys(expected)])].sort()
|
|
779
|
+
.find(key => stableStringify(actual[key]) !== stableStringify(expected[key]));
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// The gate CLI prints a summary, then (without --output) the full report JSON.
|
|
783
|
+
// Returns that object when the output carries a parseable one, else null.
|
|
784
|
+
function embeddedReport(output) {
|
|
785
|
+
const lines = String(output || '').split(/\r?\n/);
|
|
786
|
+
const start = lines.findIndex(line => line.startsWith('{'));
|
|
787
|
+
if (start < 0) return null;
|
|
788
|
+
try {
|
|
789
|
+
const value = JSON.parse(lines.slice(start).join('\n'));
|
|
790
|
+
return isObject(value) ? value : null;
|
|
791
|
+
} catch {
|
|
792
|
+
return null;
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
// Workspace checks: the evidence the command names must be retained, valid and
|
|
797
|
+
// re-evaluate to PASS, and any report the gate wrote or printed must equal the
|
|
798
|
+
// fresh re-evaluation. Definitive failures here win over the observed outcome.
|
|
799
|
+
function checkRetainedGateArtifacts(call, args, ws) {
|
|
800
|
+
const fail = reason => ({ pass: false, status: 'fail', reason });
|
|
801
|
+
const evidencePath = resolveInWorkspace(ws, args.evidence);
|
|
802
|
+
if (!evidencePath) return fail(`evidence file ${args.evidence} is outside the workspace`);
|
|
803
|
+
if (!fs.existsSync(evidencePath)) return fail(`evidence file ${args.evidence} was not retained in the workspace`);
|
|
804
|
+
let evidence;
|
|
805
|
+
try { evidence = JSON.parse(fs.readFileSync(evidencePath, 'utf8')); }
|
|
806
|
+
catch (error) { return fail(`evidence file ${args.evidence} is not valid JSON (${error.message})`); }
|
|
807
|
+
const { evaluateEvidence } = require(path.join(ROOT, 'tdd', 'scripts', 'quality-gate.js'));
|
|
808
|
+
const report = evaluateEvidence(evidence);
|
|
809
|
+
if (report.result !== 'PASS') {
|
|
810
|
+
const failed = Object.entries(report.mandatoryGates || {}).filter(([, ok]) => !ok).map(([name]) => name);
|
|
811
|
+
return fail(`independent re-evaluation of ${args.evidence} is ${report.result} (gate would exit 1)${failed.length ? `; failed gates: ${failed.join(', ')}` : ''}`);
|
|
812
|
+
}
|
|
813
|
+
// Compare through JSON so the fresh report has the same shape as a retained one.
|
|
814
|
+
const expected = JSON.parse(JSON.stringify(report));
|
|
815
|
+
if (args.output) {
|
|
816
|
+
const outputPath = resolveInWorkspace(ws, args.output);
|
|
817
|
+
if (!outputPath || !fs.existsSync(outputPath)) return fail(`quality report ${args.output} was not retained in the workspace`);
|
|
818
|
+
let retained;
|
|
819
|
+
try { retained = JSON.parse(fs.readFileSync(outputPath, 'utf8')); }
|
|
820
|
+
catch (error) { return fail(`quality report ${args.output} is not valid JSON (${error.message})`); }
|
|
821
|
+
if (!isObject(retained)) return fail(`retained report ${args.output} is not a JSON object`);
|
|
822
|
+
const { required } = require(path.join(ROOT, 'tdd', 'schemas', 'quality-report.schema.json'));
|
|
823
|
+
const missing = required.filter(key => !Object.prototype.hasOwnProperty.call(retained, key));
|
|
824
|
+
if (missing.length) return fail(`retained report ${args.output} is missing required keys: ${missing.join(', ')}`);
|
|
825
|
+
// The retained file must be the full report, not a marker.
|
|
826
|
+
const differing = firstDifferingKey(retained, expected);
|
|
827
|
+
if (differing !== undefined) return fail(`retained report ${args.output} differs from re-evaluation at "${differing}"`);
|
|
828
|
+
}
|
|
829
|
+
const printed = embeddedReport(call.output);
|
|
830
|
+
if (printed) {
|
|
831
|
+
const differing = firstDifferingKey(printed, expected);
|
|
832
|
+
if (differing !== undefined) return fail(`gate output report differs from re-evaluation at "${differing}"`);
|
|
833
|
+
}
|
|
834
|
+
return { pass: true, status: 'pass', reason: `quality gate re-evaluated ${args.evidence}: PASS` };
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
function verifyQualityGateRun(call, ws) {
|
|
838
|
+
const fail = reason => ({ pass: false, status: 'fail', reason });
|
|
839
|
+
const command = call && call.input && typeof call.input === 'object' ? call.input.command : undefined;
|
|
840
|
+
const args = qualityGateArgs(command);
|
|
841
|
+
if (!args) return fail('matched command is not a TDD quality-gate invocation');
|
|
842
|
+
if (args.script === 'test:tdd-quality') return fail('test:tdd-quality runs the repository self-test, not the gate on task evidence');
|
|
843
|
+
if (!args.evidence) return fail('quality gate was run without an evidence file, so no auditable report exists');
|
|
844
|
+
const artifacts = checkRetainedGateArtifacts(call, args, ws);
|
|
845
|
+
if (!artifacts.pass) return artifacts;
|
|
846
|
+
const outcome = observedGateOutcome(call);
|
|
847
|
+
if (outcome.status === 'fail') return fail(`quality gate command failed: ${outcome.reason}`);
|
|
848
|
+
if (outcome.status === 'inconclusive') return { pass: false, status: 'inconclusive', reason: outcome.reason };
|
|
849
|
+
return artifacts;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// The last matching attempt is the one that counts: an agent may fix evidence and
|
|
853
|
+
// re-run. Attempts are graded in transcript order, including failed, denied and
|
|
854
|
+
// unresolved ones, so an earlier PASS cannot satisfy the gate once a later run
|
|
855
|
+
// did not complete. A completed last run is re-verified from retained evidence.
|
|
856
|
+
function gradeQualityGate(expectation, parsed, ws) {
|
|
857
|
+
const attempted = gradeExecutionEvidence({ ...expectation, type: 'tool_attempted' }, parsed);
|
|
858
|
+
if (attempted.status === 'inconclusive') return attempted;
|
|
859
|
+
if (!attempted.pass) return { pass: false, status: 'fail', reason: `quality gate not executed: ${attempted.reason}` };
|
|
860
|
+
const last = attempted.observed[attempted.observed.length - 1];
|
|
861
|
+
if (last.denied === true || last.status === 'denied') {
|
|
862
|
+
return { pass: false, status: 'fail', reason: `the last quality gate run (${last.id}) was denied; an earlier run cannot satisfy the gate` };
|
|
863
|
+
}
|
|
864
|
+
if (last.status === 'failed') {
|
|
865
|
+
return { pass: false, status: 'fail', reason: `the last quality gate run (${last.id}) failed; an earlier run cannot satisfy the gate` };
|
|
866
|
+
}
|
|
867
|
+
if (last.status === 'completed') {
|
|
868
|
+
const verdict = verifyQualityGateRun(last, ws);
|
|
869
|
+
return { pass: verdict.pass, status: verdict.status, reason: verdict.reason };
|
|
870
|
+
}
|
|
871
|
+
return { pass: false, status: 'inconclusive', reason: `the last quality gate run (${last.id}) has no observable result` };
|
|
872
|
+
}
|
|
873
|
+
|
|
706
874
|
function grade(c, ws, transcriptPath, engine = 'auto') {
|
|
707
875
|
const parsed = parseTranscriptFile(transcriptPath, engine);
|
|
708
876
|
const ordered = parseTranscriptEvents(transcriptPath);
|
|
@@ -723,6 +891,8 @@ function grade(c, ws, transcriptPath, engine = 'auto') {
|
|
|
723
891
|
status = 'fail';
|
|
724
892
|
reason = 'expected a completed edit before the matching tool execution';
|
|
725
893
|
}
|
|
894
|
+
} else if (e.type === 'tdd_quality_gate') {
|
|
895
|
+
({ pass, status, reason } = gradeQualityGate(e, parsed, ws));
|
|
726
896
|
} else if (e.type === 'trace_contains' || e.type === 'trace_not_contains') {
|
|
727
897
|
if (e.type === 'trace_contains' && isExecutionLikeTraceValue(e.value)) {
|
|
728
898
|
status = 'inconclusive';
|
|
@@ -1078,6 +1248,8 @@ module.exports = {
|
|
|
1078
1248
|
finalAssistantText,
|
|
1079
1249
|
finalAssistantTextCandidates,
|
|
1080
1250
|
grade,
|
|
1251
|
+
GATE_COMMAND_ERROR_RE,
|
|
1252
|
+
qualityGateArgs,
|
|
1081
1253
|
parseTranscriptEvents,
|
|
1082
1254
|
pairVerdict,
|
|
1083
1255
|
parseSimpleYaml,
|
|
@@ -28,6 +28,32 @@ function clone(value) {
|
|
|
28
28
|
return JSON.parse(JSON.stringify(value));
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
// Observable outcome of a tool result: the text the tool printed, whether the
|
|
32
|
+
// host flagged it as an error, and an exit code when the engine reports one.
|
|
33
|
+
// Claude tool_result blocks carry content and is_error; OpenCode carries
|
|
34
|
+
// state.output and state.metadata (merged into the result as `metadata`).
|
|
35
|
+
function normalizeOutput(content) {
|
|
36
|
+
if (content === null || content === undefined) return null;
|
|
37
|
+
if (typeof content === 'string') return content;
|
|
38
|
+
if (Array.isArray(content)) {
|
|
39
|
+
const texts = content
|
|
40
|
+
.map(block => {
|
|
41
|
+
if (typeof block === 'string') return block;
|
|
42
|
+
return isObject(block) && typeof block.text === 'string' ? block.text : null;
|
|
43
|
+
})
|
|
44
|
+
.filter(text => text !== null);
|
|
45
|
+
return texts.length ? texts.join('\n') : null;
|
|
46
|
+
}
|
|
47
|
+
if (isObject(content)) return JSON.stringify(content);
|
|
48
|
+
return String(content);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function exitCodeOf(result) {
|
|
52
|
+
const metadata = isObject(result.metadata) ? result.metadata : {};
|
|
53
|
+
const value = [result.exit_code, result.exitCode, metadata.exit].find(item => typeof item === 'number');
|
|
54
|
+
return value;
|
|
55
|
+
}
|
|
56
|
+
|
|
31
57
|
function makeMetadata() {
|
|
32
58
|
return {
|
|
33
59
|
cost: null,
|
|
@@ -144,6 +170,9 @@ function ensureCall(context, details = {}) {
|
|
|
144
170
|
denied: false,
|
|
145
171
|
status: 'attempted',
|
|
146
172
|
result_count: 0,
|
|
173
|
+
output: null,
|
|
174
|
+
is_error: null,
|
|
175
|
+
exit_code: null,
|
|
147
176
|
explicitId: Boolean(suppliedId),
|
|
148
177
|
anonymousKey: suppliedId ? null : anonymousKey,
|
|
149
178
|
};
|
|
@@ -194,6 +223,13 @@ function addToolResult(context, details = {}, result = {}, forcedStatus) {
|
|
|
194
223
|
call.result_count += 1;
|
|
195
224
|
}
|
|
196
225
|
applyStatus(call, status);
|
|
226
|
+
// Keep the latest observable outcome. A result that does not report a field
|
|
227
|
+
// leaves the earlier value alone rather than erasing it.
|
|
228
|
+
const output = normalizeOutput(content);
|
|
229
|
+
if (output !== null) call.output = output;
|
|
230
|
+
if (typeof result.is_error === 'boolean') call.is_error = result.is_error;
|
|
231
|
+
const exitCode = exitCodeOf(result);
|
|
232
|
+
if (exitCode !== undefined) call.exit_code = exitCode;
|
|
197
233
|
return call;
|
|
198
234
|
}
|
|
199
235
|
|
|
@@ -417,6 +453,9 @@ function publicCall(call) {
|
|
|
417
453
|
completed: call.completed,
|
|
418
454
|
denied: call.denied,
|
|
419
455
|
status: call.status,
|
|
456
|
+
output: call.output ?? null,
|
|
457
|
+
is_error: call.is_error ?? null,
|
|
458
|
+
exit_code: call.exit_code ?? null,
|
|
420
459
|
};
|
|
421
460
|
}
|
|
422
461
|
|
|
@@ -540,6 +579,10 @@ function targetMatches(call, target) {
|
|
|
540
579
|
if (target.id !== undefined && call.id !== target.id) return false;
|
|
541
580
|
if (target.name !== undefined && call.name !== target.name) return false;
|
|
542
581
|
if (target.command !== undefined && (!isObject(call.input) || !commandMatches(call.input.command, target.command))) return false;
|
|
582
|
+
// command_pattern lets one expectation accept several explicit command shapes
|
|
583
|
+
// (for example npm run tdd:quality or node tdd/scripts/quality-gate.js) with
|
|
584
|
+
// arbitrary evidence arguments; shell control operators are rejected.
|
|
585
|
+
if (target.command_pattern !== undefined && (!isObject(call.input) || typeof call.input.command !== 'string' || !new RegExp(target.command_pattern).test(call.input.command))) return false;
|
|
543
586
|
if (target.input_contains !== undefined && !stableJson(call.input).includes(String(target.input_contains))) return false;
|
|
544
587
|
return true;
|
|
545
588
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"id": "adr-evid",
|
|
4
|
+
"order": 1,
|
|
5
|
+
"summary": "Draft the row-17 EVID-001 verification-evidence ADR.",
|
|
6
|
+
"acceptance": "A real proposed O is independently reviewed before the draft D; D is a strict descendant of O and changes only the ADR.",
|
|
7
|
+
"suggestedSkills": [],
|
|
8
|
+
"stepType": "doc-change",
|
|
9
|
+
"status": "pending",
|
|
10
|
+
"evidence": null,
|
|
11
|
+
"reasonCode": null,
|
|
12
|
+
"updatedAt": null,
|
|
13
|
+
"requiredBindings": [],
|
|
14
|
+
"optionalBindings": []
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "verify-row17",
|
|
18
|
+
"order": 2,
|
|
19
|
+
"summary": "Generate the governed index and verify the final ADR through one combined registered check and independent review.",
|
|
20
|
+
"acceptance": "All four combined-check capabilities and the ADR-only final review are captured before normal Stop reconciliation.",
|
|
21
|
+
"suggestedSkills": [],
|
|
22
|
+
"stepType": "verification",
|
|
23
|
+
"status": "pending",
|
|
24
|
+
"evidence": null,
|
|
25
|
+
"reasonCode": null,
|
|
26
|
+
"updatedAt": null,
|
|
27
|
+
"requiredBindings": [
|
|
28
|
+
{
|
|
29
|
+
"id": "verification-loop",
|
|
30
|
+
"required": true,
|
|
31
|
+
"status": "pending",
|
|
32
|
+
"path": "verification-loop/SKILL.md",
|
|
33
|
+
"availability": "available",
|
|
34
|
+
"resolvedPath": "/private/tmp/row17/plugin-f/verification-loop/SKILL.md",
|
|
35
|
+
"evidence": null,
|
|
36
|
+
"reasonCode": null,
|
|
37
|
+
"updatedAt": null
|
|
38
|
+
}
|
|
39
|
+
],
|
|
40
|
+
"optionalBindings": []
|
|
41
|
+
}
|
|
42
|
+
]
|