harness-everything 0.25.1 â 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +23 -0
- package/ci/mechanism-35-verification-contract.test.js +416 -0
- package/ci/mutation-check.js +2 -0
- package/ci/mutation-loader.js +4 -0
- package/docs/audit.md +2 -0
- package/harness-everything/SKILL.md +2 -2
- package/harness-everything/scripts/verify-gate.js +223 -75
- package/hooks/scripts/lib/workflow-isolation.js +3 -4
- package/hooks/scripts/state-persist.js +1 -1
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/hooks/scripts/lib/workflow-isolation.js +3 -4
- package/plugins/harness-everything/hooks/scripts/state-persist.js +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/scripts/lib/verification-contract.js +357 -0
- package/plugins/harness-everything/skills/harness-everything/SKILL.md +2 -2
- package/plugins/harness-everything/skills/harness-everything/scripts/verify-gate.js +223 -75
- package/plugins/harness-everything/skills/verification-loop/SKILL.md +15 -23
- package/plugins/harness-everything/skills/verification-loop/references/project-verification-contract.md +112 -0
- package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md +4 -3
- package/plugins/harness-everything/skills/verification-loop/references/verify-contract.schema.json +32 -0
- package/plugins/harness-everything/skills/verification-loop/references/verify-result.schema.json +65 -0
- package/sbom.cdx.json +7 -7
- package/scripts/lib/verification-contract.js +357 -0
- package/verification-loop/SKILL.md +15 -23
- package/verification-loop/references/project-verification-contract.md +112 -0
- package/verification-loop/references/verification-phases.md +4 -3
- package/verification-loop/references/verify-contract.schema.json +32 -0
- package/verification-loop/references/verify-result.schema.json +65 -0
|
@@ -1,100 +1,248 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
2
4
|
/**
|
|
3
|
-
*
|
|
4
|
-
* Runs the target project's own mechanical checks (lint/test scripts from the
|
|
5
|
-
* nearest package.json) and refuses completion (Exit 1) when any of them fail.
|
|
5
|
+
* Harness project verification gate.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* broken project.
|
|
11
|
-
* - HARNESS_SKIP_PROJECT_CHECKS=1 skips the project-script phase (the tmp-file
|
|
12
|
-
* check above still applies). The gate sets it for its own children so a
|
|
13
|
-
* project whose test suite itself drives the Harness runtime (like this repo) cannot
|
|
14
|
-
* recurse back into the gate.
|
|
7
|
+
* Contract: .harness/verify.json
|
|
8
|
+
* Discovery: read-only candidates; legacy package lint/test remains auto-run.
|
|
9
|
+
* UNCHECKED_* is non-blocking control flow and is never passing-test evidence.
|
|
15
10
|
*/
|
|
16
|
-
const { execSync } = require('child_process');
|
|
17
11
|
const fs = require('fs');
|
|
18
12
|
const path = require('path');
|
|
13
|
+
const { spawnSync } = require('child_process');
|
|
19
14
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
15
|
+
function loadVerificationModel() {
|
|
16
|
+
for (const candidate of [
|
|
17
|
+
path.resolve(__dirname, '../../scripts/lib/verification-contract.js'),
|
|
18
|
+
path.resolve(__dirname, '../../../scripts/lib/verification-contract.js'),
|
|
19
|
+
]) {
|
|
20
|
+
if (fs.existsSync(candidate)) return require(candidate);
|
|
21
|
+
}
|
|
22
|
+
throw new Error('verification-contract runtime is missing from this Harness distribution');
|
|
27
23
|
}
|
|
28
24
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
25
|
+
const {
|
|
26
|
+
detectRoot,
|
|
27
|
+
enumerateSubmoduleRoots,
|
|
28
|
+
findGitRoot,
|
|
29
|
+
readContract,
|
|
30
|
+
} = loadVerificationModel();
|
|
31
|
+
|
|
32
|
+
const jsonMode = process.argv.includes('--json');
|
|
33
|
+
const cwd = path.resolve(process.cwd());
|
|
34
|
+
const repoRoot = findGitRoot(cwd);
|
|
35
|
+
|
|
36
|
+
function emptyReport(status, reason) {
|
|
37
|
+
return {
|
|
38
|
+
status,
|
|
39
|
+
reason,
|
|
40
|
+
repoRoot,
|
|
41
|
+
roots: [],
|
|
42
|
+
checks: [],
|
|
43
|
+
candidates: [],
|
|
44
|
+
};
|
|
33
45
|
}
|
|
34
46
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
47
|
+
function emit(report) {
|
|
48
|
+
if (jsonMode) {
|
|
49
|
+
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
console.log('=== đĄī¸ Harness Verification Gate ===');
|
|
54
|
+
console.log(`Status: ${report.status}`);
|
|
55
|
+
console.log(`Repository: ${report.repoRoot}`);
|
|
56
|
+
if (report.reason) console.log(`Reason: ${report.reason}`);
|
|
57
|
+
|
|
58
|
+
for (const root of report.roots || []) {
|
|
59
|
+
console.log(`\n[${root.status}] ${root.path}`);
|
|
60
|
+
if (root.reason) console.log(` ${root.reason}`);
|
|
61
|
+
for (const check of root.checks || []) {
|
|
62
|
+
const exit = check.exitCode === null ? 'timeout/error' : check.exitCode;
|
|
63
|
+
console.log(` check ${check.id}: ${check.run} (exit ${exit}, ${check.durationMs}ms)`);
|
|
64
|
+
if (check.stdout) process.stdout.write(check.stdout.endsWith('\n') ? check.stdout : check.stdout + '\n');
|
|
65
|
+
if (check.stderr) process.stderr.write(check.stderr.endsWith('\n') ? check.stderr : check.stderr + '\n');
|
|
66
|
+
}
|
|
67
|
+
for (const candidate of root.candidates || []) {
|
|
68
|
+
console.log(` candidate [${candidate.source}]: ${candidate.run}`);
|
|
69
|
+
}
|
|
45
70
|
}
|
|
71
|
+
|
|
72
|
+
if (report.status === 'UNCHECKED_NO_CHECKS' || report.status === 'UNCHECKED_DISCOVERED') {
|
|
73
|
+
console.log('\nExit code is 0 (non-blocking), but NO COMPLETE AUTOMATED EVIDENCE was produced.');
|
|
74
|
+
console.log('PROHIBITED: Do NOT cite this run as proof that tests pass. Perform the listed or project-declared verification first.');
|
|
75
|
+
} else if (report.status === 'PASSED') {
|
|
76
|
+
console.log('\nâ
Verification PASSED. Objective project checks completed successfully.');
|
|
77
|
+
} else if (report.status === 'FAILED') {
|
|
78
|
+
console.error('\nâ Verification FAILED. Fix the reported check/contract error and run this gate again.');
|
|
79
|
+
} else if (report.status === 'SKIPPED') {
|
|
80
|
+
console.log('âšī¸ Project checks skipped by HARNESS_SKIP_PROJECT_CHECKS=1.');
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function finish(report) {
|
|
85
|
+
emit(report);
|
|
86
|
+
process.exit(report.status === 'FAILED' ? 1 : 0);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function statusRank(status) {
|
|
90
|
+
return {
|
|
91
|
+
PASSED: 0,
|
|
92
|
+
UNCHECKED_NO_CHECKS: 1,
|
|
93
|
+
UNCHECKED_DISCOVERED: 2,
|
|
94
|
+
FAILED: 3,
|
|
95
|
+
}[status] ?? 0;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function aggregateStatus(roots) {
|
|
99
|
+
if (!roots.length) return 'UNCHECKED_NO_CHECKS';
|
|
100
|
+
return roots.reduce((worst, root) => statusRank(root.status) > statusRank(worst) ? root.status : worst, 'PASSED');
|
|
46
101
|
}
|
|
47
102
|
|
|
48
|
-
function
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
103
|
+
function executeCheck(check) {
|
|
104
|
+
const started = Date.now();
|
|
105
|
+
const result = spawnSync(check.run, {
|
|
106
|
+
cwd: check.cwd,
|
|
107
|
+
shell: true,
|
|
108
|
+
encoding: 'utf8',
|
|
109
|
+
windowsHide: true,
|
|
110
|
+
timeout: check.timeoutSec * 1000,
|
|
111
|
+
env: { ...process.env, HARNESS_SKIP_PROJECT_CHECKS: '1' },
|
|
112
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
113
|
+
});
|
|
114
|
+
const durationMs = Date.now() - started;
|
|
115
|
+
const timedOut = !!(result.error && result.error.code === 'ETIMEDOUT');
|
|
116
|
+
const exitCode = typeof result.status === 'number' ? result.status : null;
|
|
117
|
+
return {
|
|
118
|
+
id: check.id,
|
|
119
|
+
run: check.run,
|
|
120
|
+
cwd: check.cwd,
|
|
121
|
+
source: check.source,
|
|
122
|
+
exitCode,
|
|
123
|
+
durationMs,
|
|
124
|
+
timedOut,
|
|
125
|
+
signal: result.signal || null,
|
|
126
|
+
stdout: result.stdout || '',
|
|
127
|
+
stderr: result.stderr || '',
|
|
128
|
+
passed: !timedOut && exitCode === 0,
|
|
129
|
+
};
|
|
53
130
|
}
|
|
54
131
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
132
|
+
function publicCheck(check) {
|
|
133
|
+
return {
|
|
134
|
+
id: check.id,
|
|
135
|
+
run: check.run,
|
|
136
|
+
cwd: check.cwd,
|
|
137
|
+
source: check.source,
|
|
138
|
+
exitCode: check.exitCode,
|
|
139
|
+
durationMs: check.durationMs,
|
|
140
|
+
...(check.timedOut ? { timedOut: true } : {}),
|
|
141
|
+
...(check.signal ? { signal: check.signal } : {}),
|
|
142
|
+
...(!jsonMode && check.stdout ? { stdout: check.stdout } : {}),
|
|
143
|
+
...(!jsonMode && check.stderr ? { stderr: check.stderr } : {}),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function candidateView(candidate) {
|
|
148
|
+
return {
|
|
149
|
+
source: candidate.source,
|
|
150
|
+
run: candidate.run,
|
|
151
|
+
cwd: candidate.cwd,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function rootReason(status, checks, candidates) {
|
|
156
|
+
if (status === 'FAILED') {
|
|
157
|
+
const failed = checks.find(check => !check.passed);
|
|
158
|
+
if (!failed) return 'verification failed';
|
|
159
|
+
return failed.timedOut
|
|
160
|
+
? `check ${failed.id} timed out after ${failed.durationMs}ms`
|
|
161
|
+
: `check ${failed.id} exited ${failed.exitCode}`;
|
|
63
162
|
}
|
|
163
|
+
if (status === 'UNCHECKED_DISCOVERED') return `${candidates.length} verification candidate(s) detected but not executed`;
|
|
164
|
+
if (status === 'UNCHECKED_NO_CHECKS') return 'no project verification checks were declared or detected';
|
|
165
|
+
return `${checks.length} check(s) passed`;
|
|
64
166
|
}
|
|
65
167
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
168
|
+
function contractPlan(contract) {
|
|
169
|
+
const rootResults = contract.roots.map(root => ({
|
|
170
|
+
path: root.path,
|
|
171
|
+
status: 'UNCHECKED_NO_CHECKS',
|
|
172
|
+
reason: '',
|
|
173
|
+
checks: [],
|
|
174
|
+
candidates: [],
|
|
175
|
+
}));
|
|
176
|
+
const byPath = new Map(rootResults.map(root => [path.resolve(root.path), root]));
|
|
177
|
+
|
|
178
|
+
for (const check of contract.checks) {
|
|
179
|
+
const executed = executeCheck(check);
|
|
180
|
+
const root = byPath.get(path.resolve(check.root));
|
|
181
|
+
root.checks.push(executed);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
for (const root of rootResults) {
|
|
185
|
+
root.status = root.checks.length === 0
|
|
186
|
+
? 'UNCHECKED_NO_CHECKS'
|
|
187
|
+
: root.checks.some(check => !check.passed) ? 'FAILED' : 'PASSED';
|
|
188
|
+
root.reason = rootReason(root.status, root.checks, root.candidates);
|
|
189
|
+
}
|
|
190
|
+
return rootResults;
|
|
79
191
|
}
|
|
80
192
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
193
|
+
function discoveredPlan() {
|
|
194
|
+
const roots = [];
|
|
195
|
+
for (const rootPath of enumerateSubmoduleRoots(repoRoot)) {
|
|
196
|
+
const detected = detectRoot(rootPath);
|
|
197
|
+
const executed = detected.checks.map(executeCheck);
|
|
198
|
+
const candidates = detected.candidates;
|
|
199
|
+
const status = executed.some(check => !check.passed)
|
|
200
|
+
? 'FAILED'
|
|
201
|
+
: candidates.length > 0 ? 'UNCHECKED_DISCOVERED'
|
|
202
|
+
: executed.length > 0 ? 'PASSED' : 'UNCHECKED_NO_CHECKS';
|
|
203
|
+
roots.push({
|
|
204
|
+
path: rootPath,
|
|
205
|
+
status,
|
|
206
|
+
reason: rootReason(status, executed, candidates),
|
|
207
|
+
checks: executed,
|
|
208
|
+
candidates,
|
|
90
209
|
});
|
|
91
|
-
console.log(`â
${name} passed.`);
|
|
92
|
-
} catch (err) {
|
|
93
|
-
console.error(`\nâ Verification FAILED: "${runner} run ${name}" exited non-zero${err.signal ? ` (signal ${err.signal})` : ''}.`);
|
|
94
|
-
console.error("Fix the failures above and run this gate again.");
|
|
95
|
-
process.exit(1);
|
|
96
210
|
}
|
|
211
|
+
return roots;
|
|
97
212
|
}
|
|
98
213
|
|
|
99
|
-
|
|
100
|
-
|
|
214
|
+
function finalize(roots) {
|
|
215
|
+
const status = aggregateStatus(roots);
|
|
216
|
+
const failingRoot = roots.find(root => root.status === status);
|
|
217
|
+
const report = {
|
|
218
|
+
status,
|
|
219
|
+
reason: failingRoot ? failingRoot.reason : '',
|
|
220
|
+
repoRoot,
|
|
221
|
+
roots: roots.map(root => ({
|
|
222
|
+
path: root.path,
|
|
223
|
+
status: root.status,
|
|
224
|
+
reason: root.reason,
|
|
225
|
+
checks: root.checks.map(publicCheck),
|
|
226
|
+
candidates: root.candidates.map(candidateView),
|
|
227
|
+
})),
|
|
228
|
+
checks: roots.flatMap(root => root.checks.map(publicCheck)),
|
|
229
|
+
candidates: roots.flatMap(root => root.candidates.map(candidateView)),
|
|
230
|
+
};
|
|
231
|
+
finish(report);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
try {
|
|
235
|
+
// Injection remains first and cannot be bypassed by the recursion guard.
|
|
236
|
+
if (fs.existsSync(path.join(cwd, '.verify-fail.tmp'))) {
|
|
237
|
+
finish(emptyReport('FAILED', '.verify-fail.tmp present (injected failure)'));
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (process.env.HARNESS_SKIP_PROJECT_CHECKS === '1') {
|
|
241
|
+
finish(emptyReport('SKIPPED', 'HARNESS_SKIP_PROJECT_CHECKS=1 recursion guard'));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const contract = readContract(repoRoot);
|
|
245
|
+
finalize(contract ? contractPlan(contract) : discoveredPlan());
|
|
246
|
+
} catch (error) {
|
|
247
|
+
finish(emptyReport('FAILED', error && error.message ? error.message : String(error)));
|
|
248
|
+
}
|
|
@@ -4,44 +4,36 @@ description: "Run objective verification gates (build, types, lint, tests, secur
|
|
|
4
4
|
license: Apache-2.0
|
|
5
5
|
metadata:
|
|
6
6
|
author: Miya Daniel
|
|
7
|
-
version: 0.
|
|
7
|
+
version: 0.26.0
|
|
8
8
|
---
|
|
9
9
|
|
|
10
10
|
# Verification Loop
|
|
11
11
|
|
|
12
|
-
Run objective
|
|
12
|
+
Run objective gates before claiming delivery readiness.
|
|
13
13
|
|
|
14
14
|
## Skill Contract
|
|
15
15
|
|
|
16
16
|
| Component | Specification |
|
|
17
17
|
| :--- | :--- |
|
|
18
|
-
| **Trigger / Input** | Completed
|
|
19
|
-
| **Expected Output** | Evidence-backed verification report
|
|
20
|
-
| **State Mutations** | Fills
|
|
21
|
-
| **Enforcement Gate** |
|
|
18
|
+
| **Trigger / Input** | Completed change awaiting delivery. |
|
|
19
|
+
| **Expected Output** | Evidence-backed verification report. |
|
|
20
|
+
| **State Mutations** | Fills `<this-skill-dir>/templates/verification-report.template.md`. |
|
|
21
|
+
| **Enforcement Gate** | Every applicable gate passes. |
|
|
22
22
|
|
|
23
23
|
## USE FOR:
|
|
24
|
-
|
|
25
|
-
-
|
|
26
|
-
- Run quality gates before a PR
|
|
24
|
+
- Verify a completed change
|
|
25
|
+
- Run quality gates before delivery/PR
|
|
27
26
|
|
|
28
27
|
## DO NOT USE FOR:
|
|
29
|
-
|
|
30
|
-
-
|
|
31
|
-
- Writing new features or tests (use `tdd`)
|
|
28
|
+
- Fixing failures without re-running verification
|
|
29
|
+
- New feature/test implementation (use `tdd`)
|
|
32
30
|
|
|
33
31
|
## Workflow
|
|
34
32
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
3. Lint: `npm run lint` or `ruff check .`.
|
|
41
|
-
4. Tests and coverage: `npm run test -- --coverage`.
|
|
42
|
-
5. Security scan: secrets and stray logs.
|
|
43
|
-
6. Diff review: `git diff --stat` and unintended-change checks.
|
|
44
|
-
7. Contract reconciliation when the work cites `REQ-*` / ADR/spec/ticket lineage or a contract-integrity trace exists: run the strict `<this-skill-dir>/scripts/contract-integrity-audit.js` gate with current #58 evidence and `--workspace <workspace>` so traced source/test/implementation artifacts are fingerprinted. `AUDIT`, `NOT_EVALUATED`, stale source binding, or drift is **not ready** for delivery.
|
|
45
|
-
8. Fill `<this-skill-dir>/templates/verification-report.template.md` with evidence.
|
|
33
|
+
1. Run applicable build, type, lint, test/coverage, security, and diff gates.
|
|
34
|
+
2. Reconcile cited REQ/ADR/spec lineage with `<this-skill-dir>/scripts/contract-integrity-audit.js`; stale/failed/audit-only evidence is not ready.
|
|
35
|
+
3. For project-declared/discovered mechanical gates, follow `<this-skill-dir>/references/project-verification-contract.md`.
|
|
36
|
+
4. On failure, fix and rerun from the first affected gate.
|
|
37
|
+
5. Fill `<this-skill-dir>/templates/verification-report.template.md` with actual evidence.
|
|
46
38
|
|
|
47
39
|
Deep dive: `<this-skill-dir>/references/verification-phases.md`
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# Project Verification Contract
|
|
2
|
+
|
|
3
|
+
Harness project verification has two layers:
|
|
4
|
+
|
|
5
|
+
1. **Declared contract** â `.harness/verify.json` is authoritative and executable.
|
|
6
|
+
2. **Read-only discovery** â ecosystem signals are reported as candidates when no contract exists. Detected candidates are never auto-run except the legacy `package.json` `lint` / `test` scripts.
|
|
7
|
+
|
|
8
|
+
This keeps verification explicit without turning discovery into an accidental command runner.
|
|
9
|
+
|
|
10
|
+
## Contract
|
|
11
|
+
|
|
12
|
+
Create `.harness/verify.json` at the enclosing Git repository root. Its documented JSON Schema is `<this-skill-dir>/references/verify-contract.schema.json`:
|
|
13
|
+
|
|
14
|
+
```json
|
|
15
|
+
{
|
|
16
|
+
"version": 1,
|
|
17
|
+
"roots": [".", "sub/a", "sub/b"],
|
|
18
|
+
"checks": [
|
|
19
|
+
{
|
|
20
|
+
"id": "precommit",
|
|
21
|
+
"run": "pre-commit run --all-files",
|
|
22
|
+
"cwd": ".",
|
|
23
|
+
"timeoutSec": 600
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"id": "sub-a-e2e",
|
|
27
|
+
"run": "bash scripts/e2e.sh",
|
|
28
|
+
"cwd": "sub/a",
|
|
29
|
+
"timeoutSec": 600
|
|
30
|
+
}
|
|
31
|
+
]
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Rules:
|
|
36
|
+
|
|
37
|
+
- `version` MUST be `1`.
|
|
38
|
+
- `roots` is optional; default is `["."]`. Every root MUST stay inside the enclosing Git repository and exist.
|
|
39
|
+
- `checks` MUST be a non-empty array.
|
|
40
|
+
- Each check MUST have a unique non-empty `id`, non-empty `run`, repository-contained `cwd` (default `.`), and optional integer `timeoutSec` from 1 through 3600.
|
|
41
|
+
- Unknown fields, invalid JSON, duplicate IDs, escaping paths, or invalid values make verification `FAILED`.
|
|
42
|
+
- Contract checks run with `HARNESS_SKIP_PROJECT_CHECKS=1` so a project's checks cannot recursively invoke the same project gate.
|
|
43
|
+
- When a contract exists, ecosystem detection and legacy npm auto-run are not used.
|
|
44
|
+
|
|
45
|
+
A `git commit` is accepted as successful verification evidence only when:
|
|
46
|
+
- the declared contract contains a `pre-commit ...` check,
|
|
47
|
+
- a runnable pre-commit hook is configured for that repository (on POSIX, the hook file must have an executable bit), and
|
|
48
|
+
- the commit command does not use `--no-verify` or `-n`.
|
|
49
|
+
|
|
50
|
+
This prevents a generic commit from being mistaken for proof that the declared verification contract ran.
|
|
51
|
+
|
|
52
|
+
## Status taxonomy
|
|
53
|
+
|
|
54
|
+
| Status | Meaning | Exit |
|
|
55
|
+
|---|---|---:|
|
|
56
|
+
| `PASSED` | Every executed authoritative check exited 0. | 0 |
|
|
57
|
+
| `FAILED` | A check failed/timed out, the injected failure marker exists, or declared/project metadata is invalid. | 1 |
|
|
58
|
+
| `UNCHECKED_NO_CHECKS` | No contract, no runnable legacy npm checks, and no verification candidates were detected. | 0 |
|
|
59
|
+
| `UNCHECKED_DISCOVERED` | Verification candidates exist but at least one was not executed. | 0 |
|
|
60
|
+
| `SKIPPED` | `HARNESS_SKIP_PROJECT_CHECKS=1` recursion guard. | 0 |
|
|
61
|
+
|
|
62
|
+
Every `UNCHECKED_*` human report MUST include:
|
|
63
|
+
|
|
64
|
+
> PROHIBITED: Do NOT cite this run as proof that tests pass.
|
|
65
|
+
|
|
66
|
+
An exit code of 0 from `UNCHECKED_*` is non-blocking control flow, not passing-test evidence.
|
|
67
|
+
|
|
68
|
+
## Discovery when no contract exists
|
|
69
|
+
|
|
70
|
+
Discovery is read-only. Harness may report:
|
|
71
|
+
|
|
72
|
+
- `package.json`: `lint` / `test` remain legacy auto-run; `check`, `typecheck`, `verify` are candidates.
|
|
73
|
+
- `.pre-commit-config.yaml`: `pre-commit run --all-files`.
|
|
74
|
+
- `Makefile`: `test`, `check`, `lint`, `ci` targets.
|
|
75
|
+
- `justfile`: `test`, `check`, `ci` recipes.
|
|
76
|
+
- Python: pytest/ruff configuration in `pyproject.toml`, `tox.ini`, `noxfile.py`.
|
|
77
|
+
- `Cargo.toml`: `cargo test`.
|
|
78
|
+
- `go.mod`: `go test ./...`.
|
|
79
|
+
- `core.hooksPath`, `.husky/`, or `.git/hooks/pre-commit`: commit-time gate presence.
|
|
80
|
+
|
|
81
|
+
Detected commands are suggestions only. Their existence does not become pass evidence until they are explicitly run or declared in the contract.
|
|
82
|
+
|
|
83
|
+
## Multi-root behavior
|
|
84
|
+
|
|
85
|
+
With a contract, `roots` defines the verification scope. Each check is attributed to the most-specific declared root containing its `cwd`.
|
|
86
|
+
|
|
87
|
+
Without a contract, Harness reports the main Git root plus initialized submodule roots from `git submodule status --recursive`.
|
|
88
|
+
|
|
89
|
+
Overall status uses the most conservative root result:
|
|
90
|
+
|
|
91
|
+
```text
|
|
92
|
+
FAILED > UNCHECKED_DISCOVERED > UNCHECKED_NO_CHECKS > PASSED
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
A submodule with no checks remains visible as its own `UNCHECKED_NO_CHECKS` root; it is never silently merged into the parent result.
|
|
96
|
+
|
|
97
|
+
## Structured output
|
|
98
|
+
|
|
99
|
+
Run:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
node harness-everything/scripts/verify-gate.js --json
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The output contains:
|
|
106
|
+
|
|
107
|
+
- `status`, `reason`, and `repoRoot`
|
|
108
|
+
- `roots[]` with per-root status/reason
|
|
109
|
+
- `checks[]` with `id`, `run`, `cwd`, `source`, `exitCode`, and `durationMs`
|
|
110
|
+
- `candidates[]` with `source`, `run`, and `cwd`
|
|
111
|
+
|
|
112
|
+
The machine-readable result conforms to `<this-skill-dir>/references/verify-result.schema.json`. It is evidence metadata; only `PASSED` means the gate produced passing verification evidence.
|
package/plugins/harness-everything/skills/verification-loop/references/verification-phases.md
CHANGED
|
@@ -25,11 +25,12 @@ flowchart TD
|
|
|
25
25
|
RunScript --> ReportResult{All Verification Gates Passed?}
|
|
26
26
|
DirectReport --> ReportResult
|
|
27
27
|
|
|
28
|
-
ReportResult --
|
|
29
|
-
ReportResult --
|
|
28
|
+
ReportResult -- PASSED --> Deliver[Task Ready for Delivery / PR]
|
|
29
|
+
ReportResult -- FAILED --> Fix[Fix Issues in Code & Re-verify] --> Build
|
|
30
|
+
ReportResult -- UNCHECKED_* --> Manual[Run declared/listed verification; exit 0 alone is not evidence] --> Build
|
|
30
31
|
```
|
|
31
32
|
|
|
32
|
-
Adapt commands according to the project's ecosystem and active environment (`environment-detection`). Run commands directly without non-portable POSIX pipe assumptions (avoid raw `head`, `tail`, `grep`, `2>/dev/null` piping on Windows).
|
|
33
|
+
Adapt commands according to the project's ecosystem and active environment (`environment-detection`). Run commands directly without non-portable POSIX pipe assumptions (avoid raw `head`, `tail`, `grep`, `2>/dev/null` piping on Windows). For repository-specific mechanical gates and structured `PASSED` / `FAILED` / `UNCHECKED_*` semantics, follow `<this-skill-dir>/references/project-verification-contract.md`.
|
|
33
34
|
|
|
34
35
|
## Phase 1: Build Verification
|
|
35
36
|
|
package/plugins/harness-everything/skills/verification-loop/references/verify-contract.schema.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/dyphn1/Harness-everything/verification-loop/references/verify-contract.schema.json",
|
|
4
|
+
"title": "Harness project verification contract",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["version", "checks"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"version": { "const": 1 },
|
|
10
|
+
"roots": {
|
|
11
|
+
"type": "array",
|
|
12
|
+
"minItems": 1,
|
|
13
|
+
"uniqueItems": true,
|
|
14
|
+
"items": { "type": "string", "minLength": 1 }
|
|
15
|
+
},
|
|
16
|
+
"checks": {
|
|
17
|
+
"type": "array",
|
|
18
|
+
"minItems": 1,
|
|
19
|
+
"items": {
|
|
20
|
+
"type": "object",
|
|
21
|
+
"additionalProperties": false,
|
|
22
|
+
"required": ["id", "run"],
|
|
23
|
+
"properties": {
|
|
24
|
+
"id": { "type": "string", "minLength": 1 },
|
|
25
|
+
"run": { "type": "string", "minLength": 1 },
|
|
26
|
+
"cwd": { "type": "string", "minLength": 1, "default": "." },
|
|
27
|
+
"timeoutSec": { "type": "integer", "minimum": 1, "maximum": 3600, "default": 600 }
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
package/plugins/harness-everything/skills/verification-loop/references/verify-result.schema.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/dyphn1/Harness-everything/verification-loop/references/verify-result.schema.json",
|
|
4
|
+
"title": "Harness verification result",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["status", "reason", "repoRoot", "roots", "checks", "candidates"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"status": {
|
|
10
|
+
"enum": ["PASSED", "FAILED", "UNCHECKED_NO_CHECKS", "UNCHECKED_DISCOVERED", "SKIPPED"]
|
|
11
|
+
},
|
|
12
|
+
"reason": { "type": "string" },
|
|
13
|
+
"repoRoot": { "type": "string" },
|
|
14
|
+
"roots": {
|
|
15
|
+
"type": "array",
|
|
16
|
+
"items": {
|
|
17
|
+
"type": "object",
|
|
18
|
+
"additionalProperties": false,
|
|
19
|
+
"required": ["path", "status", "reason", "checks", "candidates"],
|
|
20
|
+
"properties": {
|
|
21
|
+
"path": { "type": "string" },
|
|
22
|
+
"status": {
|
|
23
|
+
"enum": ["PASSED", "FAILED", "UNCHECKED_NO_CHECKS", "UNCHECKED_DISCOVERED"]
|
|
24
|
+
},
|
|
25
|
+
"reason": { "type": "string" },
|
|
26
|
+
"checks": { "$ref": "#/$defs/checkArray" },
|
|
27
|
+
"candidates": { "$ref": "#/$defs/candidateArray" }
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"checks": { "$ref": "#/$defs/checkArray" },
|
|
32
|
+
"candidates": { "$ref": "#/$defs/candidateArray" }
|
|
33
|
+
},
|
|
34
|
+
"$defs": {
|
|
35
|
+
"checkArray": {
|
|
36
|
+
"type": "array",
|
|
37
|
+
"items": {
|
|
38
|
+
"type": "object",
|
|
39
|
+
"additionalProperties": true,
|
|
40
|
+
"required": ["id", "run", "cwd", "source", "exitCode", "durationMs"],
|
|
41
|
+
"properties": {
|
|
42
|
+
"id": { "type": "string" },
|
|
43
|
+
"run": { "type": "string" },
|
|
44
|
+
"cwd": { "type": "string" },
|
|
45
|
+
"source": { "type": "string" },
|
|
46
|
+
"exitCode": { "type": ["integer", "null"] },
|
|
47
|
+
"durationMs": { "type": "integer", "minimum": 0 }
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
"candidateArray": {
|
|
52
|
+
"type": "array",
|
|
53
|
+
"items": {
|
|
54
|
+
"type": "object",
|
|
55
|
+
"additionalProperties": false,
|
|
56
|
+
"required": ["source", "run", "cwd"],
|
|
57
|
+
"properties": {
|
|
58
|
+
"source": { "type": "string" },
|
|
59
|
+
"run": { "type": "string" },
|
|
60
|
+
"cwd": { "type": "string" }
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
package/sbom.cdx.json
CHANGED
|
@@ -2,23 +2,23 @@
|
|
|
2
2
|
"$schema": "https://cyclonedx.org/schema/bom-1.7.schema.json",
|
|
3
3
|
"bomFormat": "CycloneDX",
|
|
4
4
|
"specVersion": "1.7",
|
|
5
|
-
"serialNumber": "urn:uuid:
|
|
5
|
+
"serialNumber": "urn:uuid:d5232bfe-e479-5e87-a012-d5dafb7acdc2",
|
|
6
6
|
"version": 1,
|
|
7
7
|
"metadata": {
|
|
8
|
-
"timestamp": "2026-09-
|
|
8
|
+
"timestamp": "2026-09-24T14:53:53.083Z",
|
|
9
9
|
"tools": [
|
|
10
10
|
{
|
|
11
11
|
"vendor": "Harness Everything",
|
|
12
12
|
"name": "generate-sbom.js",
|
|
13
|
-
"version": "0.
|
|
13
|
+
"version": "0.26.0"
|
|
14
14
|
}
|
|
15
15
|
],
|
|
16
16
|
"component": {
|
|
17
17
|
"type": "application",
|
|
18
|
-
"bom-ref": "pkg:npm/harness-everything@0.
|
|
18
|
+
"bom-ref": "pkg:npm/harness-everything@0.26.0",
|
|
19
19
|
"name": "harness-everything",
|
|
20
|
-
"version": "0.
|
|
21
|
-
"purl": "pkg:npm/harness-everything@0.
|
|
20
|
+
"version": "0.26.0",
|
|
21
|
+
"purl": "pkg:npm/harness-everything@0.26.0"
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"components": [
|
|
@@ -1999,7 +1999,7 @@
|
|
|
1999
1999
|
]
|
|
2000
2000
|
},
|
|
2001
2001
|
{
|
|
2002
|
-
"ref": "pkg:npm/harness-everything@0.
|
|
2002
|
+
"ref": "pkg:npm/harness-everything@0.26.0",
|
|
2003
2003
|
"dependsOn": [
|
|
2004
2004
|
"pkg:npm/%40commitlint/cli@19.8.1",
|
|
2005
2005
|
"pkg:npm/%40commitlint/config-conventional@19.8.1",
|