@dogfood-lab/verify 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -9
- package/index.js +251 -176
- package/package.json +2 -4
- package/validators/provenance.js +121 -121
- package/validators/schema.js +58 -59
- package/validators/steps.js +102 -102
- package/validators/verdict.js +93 -93
package/README.md
CHANGED
|
@@ -30,9 +30,11 @@ const result = verify(submission, {
|
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
if (!result.ok) {
|
|
33
|
+
// rejection_reasons is an array of STRINGS with stable prefixes (see
|
|
34
|
+
// "Error shape" below). Operators discriminate failure class via the
|
|
35
|
+
// prefix; the rest of the string carries the human-readable detail.
|
|
33
36
|
for (const reason of result.rejection_reasons) {
|
|
34
|
-
console.error(
|
|
35
|
-
if (reason.hint) console.error(` hint: ${reason.hint}`);
|
|
37
|
+
console.error(reason);
|
|
36
38
|
}
|
|
37
39
|
process.exit(1);
|
|
38
40
|
}
|
|
@@ -76,17 +78,60 @@ Provenance fields (`github_run_id`, `github_workflow_ref`) are required when `pr
|
|
|
76
78
|
|
|
77
79
|
## Error shape
|
|
78
80
|
|
|
79
|
-
|
|
81
|
+
`rejection_reasons[]` is an array of **strings** — the persisted-record schema (`dogfood-record.schema.json` → `verification.rejection_reasons`) enforces `items: { type: 'string' }`. Machine-readable discrimination happens via **stable string prefixes**.
|
|
80
82
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
83
|
+
### Prefix taxonomy
|
|
84
|
+
|
|
85
|
+
The verifier emits two prefix classes:
|
|
86
|
+
|
|
87
|
+
**Submission-bad** (the submitter's payload failed a validator gate — the operator should fix the submission and resubmit):
|
|
88
|
+
|
|
89
|
+
| Prefix | Source | Meaning |
|
|
90
|
+
|---|---|---|
|
|
91
|
+
| `schema:` | `validators/schema.js` | JSON Schema check on the submission/record envelope failed. The rest of the string carries the AJV path + message. |
|
|
92
|
+
| `policy:` | `validators/policy.js` | Per-repo policy gate failed (forbidden tags, missing required fields, version-floor violation, etc.). |
|
|
93
|
+
| `steps[<id>]:` | `validators/steps.js` | Step-level contract check failed on a specific step id (gate accumulation, ordering, evidence shape). |
|
|
94
|
+
| `provenance:` | `validators/provenance.js` | The GitHub run-id confirmation could not match the submitted commit/repo at the GitHub API. |
|
|
95
|
+
| `scenario-load:` | `packages/ingest/load-context.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id`). |
|
|
96
|
+
|
|
97
|
+
**Validator-crashed** (the validator itself threw an internal error — this is an operational fault, NOT submission-bad; the operator should investigate the verifier itself):
|
|
98
|
+
|
|
99
|
+
| Prefix | Source | Meaning |
|
|
100
|
+
|---|---|---|
|
|
101
|
+
| `VALIDATOR_FAULT_SCHEMA:` | `runValidator('schema', …)` catch | Internal exception inside the schema validator. The rest of the string carries the thrown `.message`. |
|
|
102
|
+
| `VALIDATOR_FAULT_POLICY:` | `runValidator('policy', …)` catch | Internal exception inside the policy validator. |
|
|
103
|
+
| `VALIDATOR_FAULT_STEPS:` | `runValidator('steps', …)` catch | Internal exception inside the steps validator. |
|
|
104
|
+
|
|
105
|
+
### Operator hygiene
|
|
106
|
+
|
|
107
|
+
```js
|
|
108
|
+
// Discriminate by prefix
|
|
109
|
+
for (const r of result.rejection_reasons) {
|
|
110
|
+
if (r.startsWith('VALIDATOR_FAULT_')) {
|
|
111
|
+
// Operational incident — verifier-side. Page someone; do NOT route
|
|
112
|
+
// back to the submitter as a "fix your payload" message.
|
|
113
|
+
notifyOps(r);
|
|
114
|
+
} else if (r.startsWith('schema:') || r.startsWith('policy:') || r.startsWith('steps[')) {
|
|
115
|
+
// Submission-bad — surface to the submitter.
|
|
116
|
+
surfaceToSubmitter(r);
|
|
117
|
+
} else if (r.startsWith('provenance:')) {
|
|
118
|
+
// May be either class — GitHub API timeouts are operational; a real
|
|
119
|
+
// commit/repo mismatch is submission-bad. The string detail carries
|
|
120
|
+
// the discriminator.
|
|
121
|
+
triageProvenance(r);
|
|
122
|
+
} else if (r.startsWith('scenario-load:')) {
|
|
123
|
+
// Ingest-side: scenario fetch reason determines class. timeout is
|
|
124
|
+
// operational; not_found / parse_error / invalid_id are submission-bad.
|
|
125
|
+
triageScenarioLoad(r);
|
|
126
|
+
} else {
|
|
127
|
+
// Unknown prefix — log and surface as raw text.
|
|
128
|
+
log.warn('unknown rejection_reason prefix', r);
|
|
129
|
+
}
|
|
87
130
|
}
|
|
88
131
|
```
|
|
89
132
|
|
|
133
|
+
Persistence note: every entry above is round-tripped verbatim through `verification.rejection_reasons` in the persisted-record JSON; the schema enforces `array of string` so any consumer of the audit-DB ground truth sees the same prefix vocabulary.
|
|
134
|
+
|
|
90
135
|
## Docs
|
|
91
136
|
|
|
92
137
|
📖 Full handbook: **<https://dogfood-lab.github.io/testing-os/handbook/>**
|
package/index.js
CHANGED
|
@@ -1,176 +1,251 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* dogfood-labs verifier
|
|
3
|
-
*
|
|
4
|
-
* Central law engine. Takes a submission payload and produces a persisted record.
|
|
5
|
-
* Validates schema, policy, provenance. Sets verifier-owned fields.
|
|
6
|
-
* Never upgrades a proposed verdict.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import { validateSubmissionSchema } from './validators/schema.js';
|
|
10
|
-
import { validatePolicy } from './validators/policy.js';
|
|
11
|
-
import { validateStepResults } from './validators/steps.js';
|
|
12
|
-
import { computeVerdict } from './validators/verdict.js';
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
//
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
//
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
1
|
+
/**
|
|
2
|
+
* dogfood-labs verifier
|
|
3
|
+
*
|
|
4
|
+
* Central law engine. Takes a submission payload and produces a persisted record.
|
|
5
|
+
* Validates schema, policy, provenance. Sets verifier-owned fields.
|
|
6
|
+
* Never upgrades a proposed verdict.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { validateSubmissionSchema as _defaultValidateSubmissionSchema } from './validators/schema.js';
|
|
10
|
+
import { validatePolicy as _defaultValidatePolicy } from './validators/policy.js';
|
|
11
|
+
import { validateStepResults as _defaultValidateStepResults } from './validators/steps.js';
|
|
12
|
+
import { computeVerdict } from './validators/verdict.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* D1B-003 (Stage C humanization): the SOLE catch wrapper for synchronous
|
|
16
|
+
* validator calls. Distinguishes operator-actionable signals:
|
|
17
|
+
*
|
|
18
|
+
* - Validator returned a structured result → caller pushes the existing
|
|
19
|
+
* `'<class>: <details>'` prefix into `rejection_reasons` (unchanged
|
|
20
|
+
* submission-bad path; back-compat is preserved).
|
|
21
|
+
* - Validator THREW (operational incident — ajv compile fault, policy
|
|
22
|
+
* merge cycle, out-of-memory) → wrapper synthesizes a STABLE coded
|
|
23
|
+
* prefix `'VALIDATOR_FAULT_<NAME>: <details>'` for the caller to push.
|
|
24
|
+
* The prefix is greppable across runner logs:
|
|
25
|
+
* `grep -E '"VALIDATOR_FAULT_' …` → every operational incident
|
|
26
|
+
* `grep -E '^(schema|policy|steps): ' …` → every submission-bad signal
|
|
27
|
+
*
|
|
28
|
+
* `name` is upper-cased to match the prefix vocabulary documented in
|
|
29
|
+
* verify/README.md (SCHEMA / POLICY / STEPS). Returns:
|
|
30
|
+
* { ok: true, result } when fn() returned cleanly
|
|
31
|
+
* { ok: false, faultReason } when fn() threw
|
|
32
|
+
*
|
|
33
|
+
* The helper is the canonical seam for any new synchronous validator
|
|
34
|
+
* — adding a 4th validator class becomes a one-call site, not a
|
|
35
|
+
* five-line try/catch boilerplate.
|
|
36
|
+
*
|
|
37
|
+
* @param {string} name - Validator class name (will be upper-cased).
|
|
38
|
+
* @param {() => T} fn - The validator call.
|
|
39
|
+
* @returns {{ ok: true, result: T } | { ok: false, faultReason: string }}
|
|
40
|
+
* @template T
|
|
41
|
+
*/
|
|
42
|
+
function runValidator(name, fn) {
|
|
43
|
+
try {
|
|
44
|
+
return { ok: true, result: fn() };
|
|
45
|
+
} catch (e) {
|
|
46
|
+
const cls = name.toUpperCase();
|
|
47
|
+
const detail = e && e.message ? e.message : String(e);
|
|
48
|
+
return { ok: false, faultReason: `VALIDATOR_FAULT_${cls}: ${detail}` };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Verify a dogfood submission and produce a persisted record.
|
|
54
|
+
*
|
|
55
|
+
* @param {object} submission - Source-authored submission payload
|
|
56
|
+
* @param {object} options
|
|
57
|
+
* @param {object} options.globalPolicy - Parsed global policy
|
|
58
|
+
* @param {object|null} options.repoPolicy - Parsed repo policy (null if none)
|
|
59
|
+
* @param {object} options.provenance - Provenance adapter { confirm(source) => Promise<boolean> }
|
|
60
|
+
* @param {string} options.policyVersion - Semver of the policy set being applied
|
|
61
|
+
* @param {object} [options.validators] - Test-only override hook (D1B-003).
|
|
62
|
+
* Pass `{ validateSubmissionSchema, validateStepResults, validatePolicy }`
|
|
63
|
+
* to swap in fault-injecting stubs. Production callers leave this unset
|
|
64
|
+
* and the helper falls back to the module-level imports. The override
|
|
65
|
+
* exists so the wrapper's catch behaviour can be tested deterministically
|
|
66
|
+
* without monkey-patching ESM modules (which is a no-op for live bindings).
|
|
67
|
+
* Documented but not part of the public stability contract — the parameter
|
|
68
|
+
* may shift shape in future minors.
|
|
69
|
+
* @returns {Promise<object>} Persisted record (accepted or rejected)
|
|
70
|
+
*/
|
|
71
|
+
export async function verify(submission, options) {
|
|
72
|
+
if (!submission || typeof submission !== 'object' || Array.isArray(submission)) {
|
|
73
|
+
const now = new Date().toISOString();
|
|
74
|
+
// Null/non-object input cannot drive computeRecordPath() (needs repo + run_id +
|
|
75
|
+
// timing.finished_at). Mark _skipPersist so the ingest layer surfaces the
|
|
76
|
+
// rejection without crashing the persist layer with `invalid repo format: undefined`.
|
|
77
|
+
return {
|
|
78
|
+
schema_version: '1.0.0',
|
|
79
|
+
_skipPersist: true,
|
|
80
|
+
verification: {
|
|
81
|
+
status: 'rejected',
|
|
82
|
+
verified_at: now,
|
|
83
|
+
provenance_confirmed: false,
|
|
84
|
+
schema_valid: false,
|
|
85
|
+
policy_valid: false,
|
|
86
|
+
rejection_reasons: ['submission is null or not an object']
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const { globalPolicy, repoPolicy, provenance, policyVersion, validators: validatorOverrides = {} } = options;
|
|
92
|
+
// Resolve the three validators per-call so tests can inject fault-
|
|
93
|
+
// injecting stubs. Production callers leave `validators` unset and the
|
|
94
|
+
// module-level imports flow through unchanged.
|
|
95
|
+
const validateSubmissionSchema = validatorOverrides.validateSubmissionSchema || _defaultValidateSubmissionSchema;
|
|
96
|
+
const validateStepResults = validatorOverrides.validateStepResults || _defaultValidateStepResults;
|
|
97
|
+
const validatePolicy = validatorOverrides.validatePolicy || _defaultValidatePolicy;
|
|
98
|
+
const now = new Date().toISOString();
|
|
99
|
+
const reasons = [];
|
|
100
|
+
|
|
101
|
+
// 0. Cross-field guard: submission.repo MUST match the owner/repo encoded in
|
|
102
|
+
// source.run_url. Without this, a submitter can claim
|
|
103
|
+
// submission.repo='victim-org/victim-repo' while supplying source.run_url for a
|
|
104
|
+
// real, legitimate run from their own repo. Provenance would confirm (the run
|
|
105
|
+
// exists), and the persist layer would file the record under victim-org's path
|
|
106
|
+
// — a forged "pass" verdict for a repo the submitter does not control.
|
|
107
|
+
// Format: https://github.com/{owner}/{repo}/actions/runs/{id}
|
|
108
|
+
if (submission.repo && submission.source?.run_url) {
|
|
109
|
+
const m = submission.source.run_url.match(
|
|
110
|
+
/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/actions\/runs\/\d+$/
|
|
111
|
+
);
|
|
112
|
+
if (m) {
|
|
113
|
+
const sourceRepo = `${m[1]}/${m[2]}`;
|
|
114
|
+
if (sourceRepo !== submission.repo) {
|
|
115
|
+
reasons.push(
|
|
116
|
+
`repo:mismatch: submission.repo (${submission.repo}) does not match source.run_url repo (${sourceRepo})`
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// 1. Schema validation
|
|
123
|
+
// D1B-003: route both happy + fault paths through `runValidator` so
|
|
124
|
+
// submission-bad (`'schema: …'`) and operational incidents
|
|
125
|
+
// (`'VALIDATOR_FAULT_SCHEMA: …'`) emit DISTINCT, greppable prefixes.
|
|
126
|
+
let schemaResult = { valid: false, errors: [] };
|
|
127
|
+
const schemaRun = runValidator('schema', () => validateSubmissionSchema(submission));
|
|
128
|
+
if (schemaRun.ok) {
|
|
129
|
+
schemaResult = schemaRun.result;
|
|
130
|
+
if (!schemaResult.valid) {
|
|
131
|
+
reasons.push(...schemaResult.errors.map(e => `schema: ${e}`));
|
|
132
|
+
}
|
|
133
|
+
} else {
|
|
134
|
+
reasons.push(schemaRun.faultReason);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// 2. Reject if submission includes verifier-owned fields
|
|
138
|
+
const verifierFields = ['policy_version', 'verification'];
|
|
139
|
+
for (const field of verifierFields) {
|
|
140
|
+
if (field in submission) {
|
|
141
|
+
reasons.push(`submission-contains-verifier-field: ${field}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (typeof submission.overall_verdict === 'object') {
|
|
145
|
+
reasons.push('submission-contains-verifier-field: overall_verdict must be a string in submissions');
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// 3. Provenance check
|
|
149
|
+
let provenanceConfirmed = false;
|
|
150
|
+
if (schemaResult.valid && submission.source) {
|
|
151
|
+
try {
|
|
152
|
+
provenanceConfirmed = await provenance.confirm(submission.source);
|
|
153
|
+
} catch (err) {
|
|
154
|
+
reasons.push(`provenance: verification failed: ${err.message}`);
|
|
155
|
+
}
|
|
156
|
+
if (!provenanceConfirmed && !reasons.some(r => r.startsWith('provenance:'))) {
|
|
157
|
+
reasons.push('provenance: source run could not be confirmed');
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// 4. Step results validation (only if schema passed)
|
|
162
|
+
// D1B-003: same submission-bad-vs-operational-fault split as schema.
|
|
163
|
+
if (schemaResult.valid && submission.scenario_results) {
|
|
164
|
+
for (const scenario of submission.scenario_results) {
|
|
165
|
+
const stepsRun = runValidator('steps', () => validateStepResults(scenario));
|
|
166
|
+
if (stepsRun.ok) {
|
|
167
|
+
reasons.push(...stepsRun.result.map(e => `steps[${scenario.scenario_id}]: ${e}`));
|
|
168
|
+
} else {
|
|
169
|
+
reasons.push(stepsRun.faultReason);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// 5. Policy evaluation (only if schema passed)
|
|
175
|
+
// D1B-003: see runValidator JSDoc for the split. Policy-fault on
|
|
176
|
+
// `globalPolicy` corruption surfaces as `VALIDATOR_FAULT_POLICY:` and
|
|
177
|
+
// is the operator's signal that the POLICY FILE (not the submission)
|
|
178
|
+
// is broken.
|
|
179
|
+
//
|
|
180
|
+
// D1B-006: a torn repo-policy file (sentinel `{ __torn: true, reason }`
|
|
181
|
+
// returned by `loadRepoPolicy`) MUST reject the submission with
|
|
182
|
+
// `policy_valid=false` + a `policy: repo policy unreadable` rejection
|
|
183
|
+
// reason. Pre-fix the sentinel was a silent `null` — the verifier
|
|
184
|
+
// happily ran defaults against a corrupt policy. Catch the sentinel
|
|
185
|
+
// BEFORE handing the (broken) policy to `validatePolicy`, so the
|
|
186
|
+
// operator sees a real "policy:" rejection in `rejection_reasons`.
|
|
187
|
+
let policyValid = false;
|
|
188
|
+
if (schemaResult.valid) {
|
|
189
|
+
if (repoPolicy && repoPolicy.__torn === true) {
|
|
190
|
+
const detail = repoPolicy.reason || 'repo policy YAML failed to parse';
|
|
191
|
+
reasons.push(`policy: repo policy unreadable — ${detail}`);
|
|
192
|
+
// policyValid stays false.
|
|
193
|
+
} else {
|
|
194
|
+
const policyRun = runValidator('policy', () => validatePolicy(submission, { globalPolicy, repoPolicy }));
|
|
195
|
+
if (policyRun.ok) {
|
|
196
|
+
policyValid = policyRun.result.valid;
|
|
197
|
+
reasons.push(...policyRun.result.errors.map(e => `policy: ${e}`));
|
|
198
|
+
} else {
|
|
199
|
+
reasons.push(policyRun.faultReason);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// 6. Compute verdict
|
|
205
|
+
const proposedVerdict = typeof submission.overall_verdict === 'string'
|
|
206
|
+
? submission.overall_verdict
|
|
207
|
+
: null;
|
|
208
|
+
|
|
209
|
+
const hasErrors = reasons.length > 0;
|
|
210
|
+
const status = hasErrors ? 'rejected' : 'accepted';
|
|
211
|
+
|
|
212
|
+
const verdictResult = computeVerdict(proposedVerdict, {
|
|
213
|
+
schemaValid: schemaResult.valid,
|
|
214
|
+
policyValid,
|
|
215
|
+
provenanceConfirmed,
|
|
216
|
+
scenarioResults: schemaResult.valid ? submission.scenario_results : [],
|
|
217
|
+
reasons
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
// 7. Assemble persisted record
|
|
221
|
+
const persisted = {
|
|
222
|
+
schema_version: '1.0.0',
|
|
223
|
+
policy_version: policyVersion,
|
|
224
|
+
run_id: submission.run_id,
|
|
225
|
+
repo: submission.repo,
|
|
226
|
+
ref: submission.ref,
|
|
227
|
+
source: submission.source,
|
|
228
|
+
timing: submission.timing,
|
|
229
|
+
...(submission.ci_checks ? { ci_checks: submission.ci_checks } : {}),
|
|
230
|
+
scenario_results: submission.scenario_results || [],
|
|
231
|
+
overall_verdict: {
|
|
232
|
+
proposed: proposedVerdict,
|
|
233
|
+
verified: verdictResult.verified,
|
|
234
|
+
downgraded: verdictResult.downgraded,
|
|
235
|
+
...(verdictResult.downgrade_reasons.length > 0
|
|
236
|
+
? { downgrade_reasons: verdictResult.downgrade_reasons }
|
|
237
|
+
: {})
|
|
238
|
+
},
|
|
239
|
+
verification: {
|
|
240
|
+
status,
|
|
241
|
+
verified_at: now,
|
|
242
|
+
provenance_confirmed: provenanceConfirmed,
|
|
243
|
+
schema_valid: schemaResult.valid,
|
|
244
|
+
policy_valid: policyValid,
|
|
245
|
+
rejection_reasons: reasons
|
|
246
|
+
},
|
|
247
|
+
...(submission.notes ? { notes: submission.notes } : {})
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
return persisted;
|
|
251
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dogfood-lab/verify",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Central verifier for testing-os. Validates submissions against schema and policy, produces persisted records.",
|
|
6
6
|
"main": "index.js",
|
|
@@ -24,12 +24,10 @@
|
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"@dogfood-lab/schemas": "^1.2.0",
|
|
27
|
-
"ajv": "^8.18.0",
|
|
28
|
-
"ajv-formats": "^3.0.1",
|
|
29
27
|
"js-yaml": "^4.1.0"
|
|
30
28
|
},
|
|
31
29
|
"engines": {
|
|
32
|
-
"node": ">=
|
|
30
|
+
"node": ">=22"
|
|
33
31
|
},
|
|
34
32
|
"author": "mcp-tool-shop",
|
|
35
33
|
"license": "MIT",
|
package/validators/provenance.js
CHANGED
|
@@ -1,121 +1,121 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Provenance adapters
|
|
3
|
-
*
|
|
4
|
-
* The verifier checks that a source run actually exists and matches claims.
|
|
5
|
-
* Two adapters:
|
|
6
|
-
* - stub: always confirms (for tests and local development)
|
|
7
|
-
* - github: confirms via GitHub Actions API (for production)
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
/**
|
|
11
|
-
* Default per-request timeout for the GitHub provenance fetch.
|
|
12
|
-
* A hung GitHub API call would otherwise stall every consumer's ingest until
|
|
13
|
-
* the surrounding GitHub Actions runner timeout fires (default 6h). Fail fast
|
|
14
|
-
* with a clear AbortError so the verifier records 'provenance: timeout' in
|
|
15
|
-
* rejection_reasons.
|
|
16
|
-
*/
|
|
17
|
-
export const GITHUB_PROVENANCE_TIMEOUT_MS = 30000;
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Stub provenance adapter. Always confirms.
|
|
21
|
-
* Use in tests and local development.
|
|
22
|
-
*/
|
|
23
|
-
export const stubProvenance = {
|
|
24
|
-
async confirm(_source) {
|
|
25
|
-
return true;
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Stub provenance adapter that always rejects.
|
|
31
|
-
* Use in tests to verify rejection paths.
|
|
32
|
-
*/
|
|
33
|
-
export const rejectingProvenance = {
|
|
34
|
-
async confirm(_source) {
|
|
35
|
-
return false;
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* GitHub provenance adapter.
|
|
41
|
-
* Confirms a workflow run exists and matches the claimed repo, SHA, and workflow.
|
|
42
|
-
*
|
|
43
|
-
* @param {string} token - GitHub PAT with actions:read scope
|
|
44
|
-
* @param {{ timeoutMs?: number, fetchImpl?: typeof fetch }} [opts]
|
|
45
|
-
* @returns {object} Provenance adapter
|
|
46
|
-
*/
|
|
47
|
-
export function githubProvenance(token, opts = {}) {
|
|
48
|
-
const timeoutMs = opts.timeoutMs ?? GITHUB_PROVENANCE_TIMEOUT_MS;
|
|
49
|
-
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
50
|
-
return {
|
|
51
|
-
async confirm(source) {
|
|
52
|
-
if (source.provider !== 'github') {
|
|
53
|
-
throw new Error(`unsupported provider: ${source.provider}`);
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
const { provider_run_id, run_url } = source;
|
|
57
|
-
if (!provider_run_id || !run_url) {
|
|
58
|
-
return false;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
// Extract owner/repo from run_url
|
|
62
|
-
// Format: https://github.com/{owner}/{repo}/actions/runs/{id}
|
|
63
|
-
const match = run_url.match(
|
|
64
|
-
/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/actions\/runs\/(\d+)$/
|
|
65
|
-
);
|
|
66
|
-
if (!match) return false;
|
|
67
|
-
|
|
68
|
-
const [, owner, repo, urlRunId] = match;
|
|
69
|
-
|
|
70
|
-
// run_id in URL must match claimed provider_run_id
|
|
71
|
-
if (urlRunId !== String(provider_run_id)) return false;
|
|
72
|
-
|
|
73
|
-
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/actions/runs/${provider_run_id}`;
|
|
74
|
-
|
|
75
|
-
// Per-request timeout. Without this, a hung GitHub API call (rate-limit
|
|
76
|
-
// throttle, regional outage, slow connection) blocks ingest indefinitely.
|
|
77
|
-
// AbortController fires AbortError on timeout — we re-throw with a clear
|
|
78
|
-
// message so the verifier records it in rejection_reasons instead of
|
|
79
|
-
// silently treating it as 'provenance returned false.'
|
|
80
|
-
const controller = new AbortController();
|
|
81
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
82
|
-
|
|
83
|
-
let run;
|
|
84
|
-
try {
|
|
85
|
-
const resp = await fetchImpl(apiUrl, {
|
|
86
|
-
headers: {
|
|
87
|
-
Authorization: `Bearer ${token}`,
|
|
88
|
-
Accept: 'application/vnd.github+json',
|
|
89
|
-
'X-GitHub-Api-Version': '2022-11-28'
|
|
90
|
-
},
|
|
91
|
-
signal: controller.signal
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
if (!resp.ok) return false;
|
|
95
|
-
|
|
96
|
-
run = await resp.json();
|
|
97
|
-
} catch (err) {
|
|
98
|
-
if (err && (err.name === 'AbortError' || err.code === 'ABORT_ERR')) {
|
|
99
|
-
throw new Error(`provenance: GitHub API timeout after ${timeoutMs}ms`);
|
|
100
|
-
}
|
|
101
|
-
return false;
|
|
102
|
-
} finally {
|
|
103
|
-
clearTimeout(timer);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
if (run.id !== Number(provider_run_id)) return false;
|
|
107
|
-
|
|
108
|
-
// Contract: provenance confirms the workflow run actually EXECUTED
|
|
109
|
-
// (status === 'completed'). Pass/fail is a separate signal carried
|
|
110
|
-
// by submission.ci_checks and scenario verdicts — the verifier still
|
|
111
|
-
// persists failed runs, it just refuses to accept a record before the
|
|
112
|
-
// underlying CI evidence exists. Rejects 'queued' / 'in_progress' / 'waiting'.
|
|
113
|
-
if (run.status !== 'completed') return false;
|
|
114
|
-
|
|
115
|
-
if (source.commit_sha && run.head_sha !== source.commit_sha) return false;
|
|
116
|
-
if (source.repo && run.repository?.full_name !== source.repo) return false;
|
|
117
|
-
|
|
118
|
-
return true;
|
|
119
|
-
}
|
|
120
|
-
};
|
|
121
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Provenance adapters
|
|
3
|
+
*
|
|
4
|
+
* The verifier checks that a source run actually exists and matches claims.
|
|
5
|
+
* Two adapters:
|
|
6
|
+
* - stub: always confirms (for tests and local development)
|
|
7
|
+
* - github: confirms via GitHub Actions API (for production)
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Default per-request timeout for the GitHub provenance fetch.
|
|
12
|
+
* A hung GitHub API call would otherwise stall every consumer's ingest until
|
|
13
|
+
* the surrounding GitHub Actions runner timeout fires (default 6h). Fail fast
|
|
14
|
+
* with a clear AbortError so the verifier records 'provenance: timeout' in
|
|
15
|
+
* rejection_reasons.
|
|
16
|
+
*/
|
|
17
|
+
export const GITHUB_PROVENANCE_TIMEOUT_MS = 30000;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Stub provenance adapter. Always confirms.
|
|
21
|
+
* Use in tests and local development.
|
|
22
|
+
*/
|
|
23
|
+
export const stubProvenance = {
|
|
24
|
+
async confirm(_source) {
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Stub provenance adapter that always rejects.
|
|
31
|
+
* Use in tests to verify rejection paths.
|
|
32
|
+
*/
|
|
33
|
+
export const rejectingProvenance = {
|
|
34
|
+
async confirm(_source) {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* GitHub provenance adapter.
|
|
41
|
+
* Confirms a workflow run exists and matches the claimed repo, SHA, and workflow.
|
|
42
|
+
*
|
|
43
|
+
* @param {string} token - GitHub PAT with actions:read scope
|
|
44
|
+
* @param {{ timeoutMs?: number, fetchImpl?: typeof fetch }} [opts]
|
|
45
|
+
* @returns {object} Provenance adapter
|
|
46
|
+
*/
|
|
47
|
+
export function githubProvenance(token, opts = {}) {
|
|
48
|
+
const timeoutMs = opts.timeoutMs ?? GITHUB_PROVENANCE_TIMEOUT_MS;
|
|
49
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
50
|
+
return {
|
|
51
|
+
async confirm(source) {
|
|
52
|
+
if (source.provider !== 'github') {
|
|
53
|
+
throw new Error(`unsupported provider: ${source.provider}`);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const { provider_run_id, run_url } = source;
|
|
57
|
+
if (!provider_run_id || !run_url) {
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Extract owner/repo from run_url
|
|
62
|
+
// Format: https://github.com/{owner}/{repo}/actions/runs/{id}
|
|
63
|
+
const match = run_url.match(
|
|
64
|
+
/^https:\/\/github\.com\/([^/]+)\/([^/]+)\/actions\/runs\/(\d+)$/
|
|
65
|
+
);
|
|
66
|
+
if (!match) return false;
|
|
67
|
+
|
|
68
|
+
const [, owner, repo, urlRunId] = match;
|
|
69
|
+
|
|
70
|
+
// run_id in URL must match claimed provider_run_id
|
|
71
|
+
if (urlRunId !== String(provider_run_id)) return false;
|
|
72
|
+
|
|
73
|
+
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/actions/runs/${provider_run_id}`;
|
|
74
|
+
|
|
75
|
+
// Per-request timeout. Without this, a hung GitHub API call (rate-limit
|
|
76
|
+
// throttle, regional outage, slow connection) blocks ingest indefinitely.
|
|
77
|
+
// AbortController fires AbortError on timeout — we re-throw with a clear
|
|
78
|
+
// message so the verifier records it in rejection_reasons instead of
|
|
79
|
+
// silently treating it as 'provenance returned false.'
|
|
80
|
+
const controller = new AbortController();
|
|
81
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
82
|
+
|
|
83
|
+
let run;
|
|
84
|
+
try {
|
|
85
|
+
const resp = await fetchImpl(apiUrl, {
|
|
86
|
+
headers: {
|
|
87
|
+
Authorization: `Bearer ${token}`,
|
|
88
|
+
Accept: 'application/vnd.github+json',
|
|
89
|
+
'X-GitHub-Api-Version': '2022-11-28'
|
|
90
|
+
},
|
|
91
|
+
signal: controller.signal
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
if (!resp.ok) return false;
|
|
95
|
+
|
|
96
|
+
run = await resp.json();
|
|
97
|
+
} catch (err) {
|
|
98
|
+
if (err && (err.name === 'AbortError' || err.code === 'ABORT_ERR')) {
|
|
99
|
+
throw new Error(`provenance: GitHub API timeout after ${timeoutMs}ms`);
|
|
100
|
+
}
|
|
101
|
+
return false;
|
|
102
|
+
} finally {
|
|
103
|
+
clearTimeout(timer);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (run.id !== Number(provider_run_id)) return false;
|
|
107
|
+
|
|
108
|
+
// Contract: provenance confirms the workflow run actually EXECUTED
|
|
109
|
+
// (status === 'completed'). Pass/fail is a separate signal carried
|
|
110
|
+
// by submission.ci_checks and scenario verdicts — the verifier still
|
|
111
|
+
// persists failed runs, it just refuses to accept a record before the
|
|
112
|
+
// underlying CI evidence exists. Rejects 'queued' / 'in_progress' / 'waiting'.
|
|
113
|
+
if (run.status !== 'completed') return false;
|
|
114
|
+
|
|
115
|
+
if (source.commit_sha && run.head_sha !== source.commit_sha) return false;
|
|
116
|
+
if (source.repo && run.repository?.full_name !== source.repo) return false;
|
|
117
|
+
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
}
|
package/validators/schema.js
CHANGED
|
@@ -1,59 +1,58 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Schema validator — validates submissions against dogfood-record-submission.schema.json
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Schema validator — validates submissions against dogfood-record-submission.schema.json.
|
|
3
|
+
*
|
|
4
|
+
* H3 hop 3: delegates to the canonical {@link validatePayload} from
|
|
5
|
+
* `@dogfood-lab/schemas`. Pre-H3 this module compiled its own
|
|
6
|
+
* Ajv2020 + ajv-formats instance for the submission schema; that
|
|
7
|
+
* duplicated the verifier's compile path against the inbound payload
|
|
8
|
+
* AND was the third of four sites contributing to the C1 two-Ajv
|
|
9
|
+
* structural gap. The migration collapses submission-validation to
|
|
10
|
+
* the single cached validator the canonical seam shares with the
|
|
11
|
+
* rest of the workspace.
|
|
12
|
+
*
|
|
13
|
+
* Return contract preserved: `{ valid, errors: string[] }`. The
|
|
14
|
+
* string-prefix shape is consumed by verify/index.js, which prepends
|
|
15
|
+
* `schema: ` / `policy: ` / `VALIDATOR_FAULT_SCHEMA: ` per the
|
|
16
|
+
* Stage C D1B-003 operator-legibility cluster. The migration keeps
|
|
17
|
+
* the `${path} ${message}` projection so downstream prefixes stay
|
|
18
|
+
* exact.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { validatePayload } from '@dogfood-lab/schemas';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Validate a submission payload against the submission JSON Schema.
|
|
25
|
+
*
|
|
26
|
+
* @param {object} submission
|
|
27
|
+
* @returns {{ valid: boolean, errors: string[] }}
|
|
28
|
+
*/
|
|
29
|
+
export function validateSubmissionSchema(submission) {
|
|
30
|
+
let result;
|
|
31
|
+
try {
|
|
32
|
+
result = validatePayload('recordSubmission', submission);
|
|
33
|
+
} catch (e) {
|
|
34
|
+
// The canonical compile path can throw at compileSchema time if the
|
|
35
|
+
// schema file is unreadable or malformed (Ajv compile fault). Pre-H3
|
|
36
|
+
// the corresponding fault was surfaced via a `{ __loadError }` sentinel
|
|
37
|
+
// that the runValidator helper in verify/index.js never actually
|
|
38
|
+
// routed through `VALIDATOR_FAULT_*` — `getValidator` returned a plain
|
|
39
|
+
// object and the call site special-cased its `__loadError` key. The
|
|
40
|
+
// post-H3 path is cleaner: a thrown error here propagates up to
|
|
41
|
+
// runValidator which already wraps thrown validators in
|
|
42
|
+
// `VALIDATOR_FAULT_SCHEMA` (D1B-003 humanization). Same operator-
|
|
43
|
+
// facing prefix, just routed through the lawful seam instead of a
|
|
44
|
+
// sentinel. No catch — propagate.
|
|
45
|
+
throw e;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (result.valid) {
|
|
49
|
+
return { valid: true, errors: [] };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// String-prefix projection: pre-H3 wrote `${path} ${message}` directly
|
|
53
|
+
// from Ajv.errors. The canonical ValidationError has the same
|
|
54
|
+
// `{ path, message }` shape, so the formatting is verbatim.
|
|
55
|
+
const errors = result.errors.map(err => `${err.path} ${err.message}`);
|
|
56
|
+
|
|
57
|
+
return { valid: false, errors };
|
|
58
|
+
}
|
package/validators/steps.js
CHANGED
|
@@ -1,102 +1,102 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Step results validator
|
|
3
|
-
*
|
|
4
|
-
* Enforces the bridge between scenario definitions and record evidence:
|
|
5
|
-
* - Every required step must have a matching step_result
|
|
6
|
-
* - A scenario cannot be "pass" if any required step is "fail" or "blocked"
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Validate step results for a single scenario result.
|
|
11
|
-
*
|
|
12
|
-
* Note: Without access to the source repo's scenario definition, we validate
|
|
13
|
-
* structural integrity. The full required_steps check is done by policy
|
|
14
|
-
* evaluation when scenario definitions are available.
|
|
15
|
-
*
|
|
16
|
-
* @param {object} scenarioResult - A single scenario_results[] item
|
|
17
|
-
* @returns {string[]} Array of error messages (empty if valid)
|
|
18
|
-
*/
|
|
19
|
-
export function validateStepResults(scenarioResult) {
|
|
20
|
-
const errors = [];
|
|
21
|
-
const { step_results, verdict, scenario_id } = scenarioResult;
|
|
22
|
-
|
|
23
|
-
if (!step_results || step_results.length === 0) {
|
|
24
|
-
errors.push('step_results is required and must have at least one entry');
|
|
25
|
-
return errors;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
const VALID_STATUSES = new Set(['pass', 'fail', 'blocked', 'skip']);
|
|
29
|
-
|
|
30
|
-
for (let i = 0; i < step_results.length; i++) {
|
|
31
|
-
const step = step_results[i];
|
|
32
|
-
if (step == null || typeof step !== 'object' || typeof step.step_id !== 'string') {
|
|
33
|
-
errors.push(`step_results[${i}] is malformed: must be a non-null object with a string step_id`);
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
const seenIds = new Set();
|
|
38
|
-
for (const step of step_results) {
|
|
39
|
-
if (step == null || typeof step !== 'object') continue;
|
|
40
|
-
if (seenIds.has(step.step_id)) {
|
|
41
|
-
errors.push(`duplicate step_id: ${step.step_id}`);
|
|
42
|
-
}
|
|
43
|
-
seenIds.add(step.step_id);
|
|
44
|
-
if (step.status != null && !VALID_STATUSES.has(step.status)) {
|
|
45
|
-
errors.push(`step "${step.step_id}" has unknown status: "${step.status}"`);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// A scenario cannot be "pass" if any step is "fail" or "blocked"
|
|
50
|
-
if (verdict === 'pass') {
|
|
51
|
-
const failingSteps = step_results.filter(
|
|
52
|
-
s => s.status === 'fail' || s.status === 'blocked'
|
|
53
|
-
);
|
|
54
|
-
if (failingSteps.length > 0) {
|
|
55
|
-
const ids = failingSteps.map(s => s.step_id).join(', ');
|
|
56
|
-
errors.push(
|
|
57
|
-
`scenario verdict is "pass" but steps [${ids}] have status fail/blocked`
|
|
58
|
-
);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
return errors;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Validate step results against a scenario definition's required_steps.
|
|
67
|
-
* Used when the scenario definition is available (policy evaluation phase).
|
|
68
|
-
*
|
|
69
|
-
* @param {object} scenarioResult - A single scenario_results[] item
|
|
70
|
-
* @param {string[]} requiredSteps - Step IDs from scenario definition's success_criteria.required_steps
|
|
71
|
-
* @returns {string[]} Array of error messages (empty if valid)
|
|
72
|
-
*/
|
|
73
|
-
export function validateRequiredSteps(scenarioResult, requiredSteps) {
|
|
74
|
-
const errors = [];
|
|
75
|
-
const { step_results, verdict } = scenarioResult;
|
|
76
|
-
|
|
77
|
-
if (!step_results) return ['step_results missing'];
|
|
78
|
-
|
|
79
|
-
const resultMap = new Map(step_results.map(s => [s.step_id, s]));
|
|
80
|
-
|
|
81
|
-
// Every required step must have a matching step_result
|
|
82
|
-
for (const stepId of requiredSteps) {
|
|
83
|
-
const result = resultMap.get(stepId);
|
|
84
|
-
if (!result) {
|
|
85
|
-
errors.push(`required step "${stepId}" has no matching step_result`);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
// A scenario cannot be "pass" if any required step is fail/blocked
|
|
90
|
-
if (verdict === 'pass') {
|
|
91
|
-
for (const stepId of requiredSteps) {
|
|
92
|
-
const result = resultMap.get(stepId);
|
|
93
|
-
if (result && (result.status === 'fail' || result.status === 'blocked')) {
|
|
94
|
-
errors.push(
|
|
95
|
-
`scenario verdict is "pass" but required step "${stepId}" has status "${result.status}"`
|
|
96
|
-
);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
return errors;
|
|
102
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Step results validator
|
|
3
|
+
*
|
|
4
|
+
* Enforces the bridge between scenario definitions and record evidence:
|
|
5
|
+
* - Every required step must have a matching step_result
|
|
6
|
+
* - A scenario cannot be "pass" if any required step is "fail" or "blocked"
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Validate step results for a single scenario result.
|
|
11
|
+
*
|
|
12
|
+
* Note: Without access to the source repo's scenario definition, we validate
|
|
13
|
+
* structural integrity. The full required_steps check is done by policy
|
|
14
|
+
* evaluation when scenario definitions are available.
|
|
15
|
+
*
|
|
16
|
+
* @param {object} scenarioResult - A single scenario_results[] item
|
|
17
|
+
* @returns {string[]} Array of error messages (empty if valid)
|
|
18
|
+
*/
|
|
19
|
+
export function validateStepResults(scenarioResult) {
|
|
20
|
+
const errors = [];
|
|
21
|
+
const { step_results, verdict, scenario_id } = scenarioResult;
|
|
22
|
+
|
|
23
|
+
if (!step_results || step_results.length === 0) {
|
|
24
|
+
errors.push('step_results is required and must have at least one entry');
|
|
25
|
+
return errors;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const VALID_STATUSES = new Set(['pass', 'fail', 'blocked', 'skip']);
|
|
29
|
+
|
|
30
|
+
for (let i = 0; i < step_results.length; i++) {
|
|
31
|
+
const step = step_results[i];
|
|
32
|
+
if (step == null || typeof step !== 'object' || typeof step.step_id !== 'string') {
|
|
33
|
+
errors.push(`step_results[${i}] is malformed: must be a non-null object with a string step_id`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const seenIds = new Set();
|
|
38
|
+
for (const step of step_results) {
|
|
39
|
+
if (step == null || typeof step !== 'object') continue;
|
|
40
|
+
if (seenIds.has(step.step_id)) {
|
|
41
|
+
errors.push(`duplicate step_id: ${step.step_id}`);
|
|
42
|
+
}
|
|
43
|
+
seenIds.add(step.step_id);
|
|
44
|
+
if (step.status != null && !VALID_STATUSES.has(step.status)) {
|
|
45
|
+
errors.push(`step "${step.step_id}" has unknown status: "${step.status}"`);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// A scenario cannot be "pass" if any step is "fail" or "blocked"
|
|
50
|
+
if (verdict === 'pass') {
|
|
51
|
+
const failingSteps = step_results.filter(
|
|
52
|
+
s => s.status === 'fail' || s.status === 'blocked'
|
|
53
|
+
);
|
|
54
|
+
if (failingSteps.length > 0) {
|
|
55
|
+
const ids = failingSteps.map(s => s.step_id).join(', ');
|
|
56
|
+
errors.push(
|
|
57
|
+
`scenario verdict is "pass" but steps [${ids}] have status fail/blocked`
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return errors;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Validate step results against a scenario definition's required_steps.
|
|
67
|
+
* Used when the scenario definition is available (policy evaluation phase).
|
|
68
|
+
*
|
|
69
|
+
* @param {object} scenarioResult - A single scenario_results[] item
|
|
70
|
+
* @param {string[]} requiredSteps - Step IDs from scenario definition's success_criteria.required_steps
|
|
71
|
+
* @returns {string[]} Array of error messages (empty if valid)
|
|
72
|
+
*/
|
|
73
|
+
export function validateRequiredSteps(scenarioResult, requiredSteps) {
|
|
74
|
+
const errors = [];
|
|
75
|
+
const { step_results, verdict } = scenarioResult;
|
|
76
|
+
|
|
77
|
+
if (!step_results) return ['step_results missing'];
|
|
78
|
+
|
|
79
|
+
const resultMap = new Map(step_results.map(s => [s.step_id, s]));
|
|
80
|
+
|
|
81
|
+
// Every required step must have a matching step_result
|
|
82
|
+
for (const stepId of requiredSteps) {
|
|
83
|
+
const result = resultMap.get(stepId);
|
|
84
|
+
if (!result) {
|
|
85
|
+
errors.push(`required step "${stepId}" has no matching step_result`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// A scenario cannot be "pass" if any required step is fail/blocked
|
|
90
|
+
if (verdict === 'pass') {
|
|
91
|
+
for (const stepId of requiredSteps) {
|
|
92
|
+
const result = resultMap.get(stepId);
|
|
93
|
+
if (result && (result.status === 'fail' || result.status === 'blocked')) {
|
|
94
|
+
errors.push(
|
|
95
|
+
`scenario verdict is "pass" but required step "${stepId}" has status "${result.status}"`
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return errors;
|
|
102
|
+
}
|
package/validators/verdict.js
CHANGED
|
@@ -1,93 +1,93 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Verdict computation
|
|
3
|
-
*
|
|
4
|
-
* Core rule: verifier may confirm or downgrade the proposed verdict, never upgrade.
|
|
5
|
-
*
|
|
6
|
-
* Verdict severity (highest to lowest): fail > blocked > partial > pass
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
const VERDICT_RANK = { fail: 0, blocked: 1, partial: 2, pass: 3 };
|
|
10
|
-
|
|
11
|
-
/**
|
|
12
|
-
* Compute the verified verdict.
|
|
13
|
-
*
|
|
14
|
-
* @param {string|null} proposed - Source-proposed verdict
|
|
15
|
-
* @param {object} context
|
|
16
|
-
* @param {boolean} context.schemaValid
|
|
17
|
-
* @param {boolean} context.policyValid
|
|
18
|
-
* @param {boolean} context.provenanceConfirmed
|
|
19
|
-
* @param {object[]} context.scenarioResults - scenario_results from submission
|
|
20
|
-
* @param {string[]} context.reasons - accumulated rejection reasons
|
|
21
|
-
* @returns {{ verified: string, downgraded: boolean, downgrade_reasons: string[] }}
|
|
22
|
-
*/
|
|
23
|
-
export function computeVerdict(proposed, context) {
|
|
24
|
-
const { schemaValid, policyValid, provenanceConfirmed, scenarioResults, reasons } = context;
|
|
25
|
-
const downgrade_reasons = [];
|
|
26
|
-
|
|
27
|
-
// If fundamentals fail, verdict is "fail" regardless
|
|
28
|
-
if (!schemaValid || !provenanceConfirmed) {
|
|
29
|
-
const verified = 'fail';
|
|
30
|
-
if (proposed && proposed !== 'fail') {
|
|
31
|
-
downgrade_reasons.push('schema or provenance validation failed');
|
|
32
|
-
}
|
|
33
|
-
return {
|
|
34
|
-
verified,
|
|
35
|
-
downgraded: proposed != null && VERDICT_RANK[verified] < VERDICT_RANK[proposed],
|
|
36
|
-
downgrade_reasons
|
|
37
|
-
};
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// Compute the worst scenario verdict
|
|
41
|
-
let worstScenarioRank = VERDICT_RANK.pass;
|
|
42
|
-
for (const sr of scenarioResults || []) {
|
|
43
|
-
let rank = VERDICT_RANK[sr.verdict];
|
|
44
|
-
if (rank == null) {
|
|
45
|
-
rank = VERDICT_RANK.fail;
|
|
46
|
-
downgrade_reasons.push('verdict: unrecognized scenario verdict "' + sr.verdict + '", treating as fail');
|
|
47
|
-
}
|
|
48
|
-
if (rank < worstScenarioRank) {
|
|
49
|
-
worstScenarioRank = rank;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// Determine the floor verdict from evidence
|
|
54
|
-
let floorVerdict = Object.entries(VERDICT_RANK)
|
|
55
|
-
.find(([, rank]) => rank === worstScenarioRank)?.[0] || 'pass';
|
|
56
|
-
|
|
57
|
-
// Policy failure forces at least "fail"
|
|
58
|
-
if (!policyValid) {
|
|
59
|
-
floorVerdict = 'fail';
|
|
60
|
-
downgrade_reasons.push('policy validation failed');
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// The verified verdict is the worse of proposed and floor
|
|
64
|
-
// (we never upgrade, so if proposed is worse than floor, keep proposed)
|
|
65
|
-
if (proposed && VERDICT_RANK[proposed] == null) {
|
|
66
|
-
downgrade_reasons.push('verdict: unrecognized proposed verdict "' + proposed + '", treating as fail');
|
|
67
|
-
}
|
|
68
|
-
if (!proposed) {
|
|
69
|
-
downgrade_reasons.push('verdict: no proposed verdict provided, defaulting to fail');
|
|
70
|
-
}
|
|
71
|
-
const proposedRank = proposed ? (VERDICT_RANK[proposed] ?? VERDICT_RANK.fail) : VERDICT_RANK.fail;
|
|
72
|
-
const floorRank = VERDICT_RANK[floorVerdict];
|
|
73
|
-
|
|
74
|
-
let verified;
|
|
75
|
-
if (floorRank < proposedRank) {
|
|
76
|
-
// Floor is worse (lower rank = more severe) — downgrade
|
|
77
|
-
verified = floorVerdict;
|
|
78
|
-
if (proposed && proposed !== floorVerdict) {
|
|
79
|
-
downgrade_reasons.push(
|
|
80
|
-
`scenario/policy evidence requires "${floorVerdict}" but source proposed "${proposed}"`
|
|
81
|
-
);
|
|
82
|
-
}
|
|
83
|
-
} else {
|
|
84
|
-
// Proposed is same or worse — keep proposed (never upgrade)
|
|
85
|
-
verified = proposed || 'fail';
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
return {
|
|
89
|
-
verified,
|
|
90
|
-
downgraded: proposed != null && VERDICT_RANK[verified] < VERDICT_RANK[proposed],
|
|
91
|
-
downgrade_reasons
|
|
92
|
-
};
|
|
93
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Verdict computation
|
|
3
|
+
*
|
|
4
|
+
* Core rule: verifier may confirm or downgrade the proposed verdict, never upgrade.
|
|
5
|
+
*
|
|
6
|
+
* Verdict severity (highest to lowest): fail > blocked > partial > pass
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const VERDICT_RANK = { fail: 0, blocked: 1, partial: 2, pass: 3 };
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Compute the verified verdict.
|
|
13
|
+
*
|
|
14
|
+
* @param {string|null} proposed - Source-proposed verdict
|
|
15
|
+
* @param {object} context
|
|
16
|
+
* @param {boolean} context.schemaValid
|
|
17
|
+
* @param {boolean} context.policyValid
|
|
18
|
+
* @param {boolean} context.provenanceConfirmed
|
|
19
|
+
* @param {object[]} context.scenarioResults - scenario_results from submission
|
|
20
|
+
* @param {string[]} context.reasons - accumulated rejection reasons
|
|
21
|
+
* @returns {{ verified: string, downgraded: boolean, downgrade_reasons: string[] }}
|
|
22
|
+
*/
|
|
23
|
+
export function computeVerdict(proposed, context) {
|
|
24
|
+
const { schemaValid, policyValid, provenanceConfirmed, scenarioResults, reasons } = context;
|
|
25
|
+
const downgrade_reasons = [];
|
|
26
|
+
|
|
27
|
+
// If fundamentals fail, verdict is "fail" regardless
|
|
28
|
+
if (!schemaValid || !provenanceConfirmed) {
|
|
29
|
+
const verified = 'fail';
|
|
30
|
+
if (proposed && proposed !== 'fail') {
|
|
31
|
+
downgrade_reasons.push('schema or provenance validation failed');
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
verified,
|
|
35
|
+
downgraded: proposed != null && VERDICT_RANK[verified] < VERDICT_RANK[proposed],
|
|
36
|
+
downgrade_reasons
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Compute the worst scenario verdict
|
|
41
|
+
let worstScenarioRank = VERDICT_RANK.pass;
|
|
42
|
+
for (const sr of scenarioResults || []) {
|
|
43
|
+
let rank = VERDICT_RANK[sr.verdict];
|
|
44
|
+
if (rank == null) {
|
|
45
|
+
rank = VERDICT_RANK.fail;
|
|
46
|
+
downgrade_reasons.push('verdict: unrecognized scenario verdict "' + sr.verdict + '", treating as fail');
|
|
47
|
+
}
|
|
48
|
+
if (rank < worstScenarioRank) {
|
|
49
|
+
worstScenarioRank = rank;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Determine the floor verdict from evidence
|
|
54
|
+
let floorVerdict = Object.entries(VERDICT_RANK)
|
|
55
|
+
.find(([, rank]) => rank === worstScenarioRank)?.[0] || 'pass';
|
|
56
|
+
|
|
57
|
+
// Policy failure forces at least "fail"
|
|
58
|
+
if (!policyValid) {
|
|
59
|
+
floorVerdict = 'fail';
|
|
60
|
+
downgrade_reasons.push('policy validation failed');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// The verified verdict is the worse of proposed and floor
|
|
64
|
+
// (we never upgrade, so if proposed is worse than floor, keep proposed)
|
|
65
|
+
if (proposed && VERDICT_RANK[proposed] == null) {
|
|
66
|
+
downgrade_reasons.push('verdict: unrecognized proposed verdict "' + proposed + '", treating as fail');
|
|
67
|
+
}
|
|
68
|
+
if (!proposed) {
|
|
69
|
+
downgrade_reasons.push('verdict: no proposed verdict provided, defaulting to fail');
|
|
70
|
+
}
|
|
71
|
+
const proposedRank = proposed ? (VERDICT_RANK[proposed] ?? VERDICT_RANK.fail) : VERDICT_RANK.fail;
|
|
72
|
+
const floorRank = VERDICT_RANK[floorVerdict];
|
|
73
|
+
|
|
74
|
+
let verified;
|
|
75
|
+
if (floorRank < proposedRank) {
|
|
76
|
+
// Floor is worse (lower rank = more severe) — downgrade
|
|
77
|
+
verified = floorVerdict;
|
|
78
|
+
if (proposed && proposed !== floorVerdict) {
|
|
79
|
+
downgrade_reasons.push(
|
|
80
|
+
`scenario/policy evidence requires "${floorVerdict}" but source proposed "${proposed}"`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
} else {
|
|
84
|
+
// Proposed is same or worse — keep proposed (never upgrade)
|
|
85
|
+
verified = proposed || 'fail';
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return {
|
|
89
|
+
verified,
|
|
90
|
+
downgraded: proposed != null && VERDICT_RANK[verified] < VERDICT_RANK[proposed],
|
|
91
|
+
downgrade_reasons
|
|
92
|
+
};
|
|
93
|
+
}
|