@dogfood-lab/verify 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/cli.js +4 -1
- package/index.js +22 -3
- package/package.json +1 -1
- package/parse-rejection.js +21 -2
- package/validators/policy.js +164 -12
- package/validators/predicate.js +289 -0
- package/validators/provenance-gitlab.test.js +75 -0
- package/validators/provenance.js +151 -78
package/README.md
CHANGED
|
@@ -83,7 +83,7 @@ Provenance fields (`github_run_id`, `github_workflow_ref`) are required when `pr
|
|
|
83
83
|
|
|
84
84
|
### Prefix taxonomy
|
|
85
85
|
|
|
86
|
-
The verifier emits
|
|
86
|
+
The verifier emits rejection-reason strings under stable prefixes, each mapping to one of four routing classes:
|
|
87
87
|
|
|
88
88
|
Discrimination happens by **class**, surfaced by `parseRejectionReason` (below). Every prefix maps to one of four classes: **submission-bad** (the submitter fixes the payload), **operational** (the verifier/tooling faulted), **ingest** (an ingest-side load fault), or **unknown** (unrecognized prefix).
|
|
89
89
|
|
|
@@ -92,9 +92,10 @@ Discrimination happens by **class**, surfaced by `parseRejectionReason` (below).
|
|
|
92
92
|
| Prefix | Source | Meaning |
|
|
93
93
|
|---|---|---|
|
|
94
94
|
| `schema:` | `validators/schema.js` | JSON Schema check on the submission/record envelope failed. The rest of the string carries the AJV path + message. |
|
|
95
|
-
| `policy:` | `validators/policy.js` | Per-repo policy gate failed (forbidden tags, missing required fields, surface evidence/CI requirements,
|
|
95
|
+
| `policy:` | `validators/policy.js` | Per-repo policy gate failed (forbidden tags, missing required fields, surface evidence/CI requirements, or a declarative `when`/`custom_rules` predicate matched — see the [policy DSL](https://dogfood-lab.github.io/testing-os/handbook/policy-dsl/)). |
|
|
96
|
+
| `policy-config:` | `validators/policy.js` | **VERIFY-F1.** A REPO custom-rule predicate hit an eval-time semantic fault the schema could not catch — an unknown leading field, a numeric operator against a non-number, or a depth/width/fan-out budget. The repo authored the bad rule, so the fix is the submitter's. (A malformed GLOBAL predicate is `VALIDATOR_FAULT_POLICY:` operational instead — see below.) |
|
|
96
97
|
| `steps[<id>]:` | `validators/steps.js` | Step-level contract check failed on a specific step id (gate accumulation, ordering, evidence shape). |
|
|
97
|
-
| `provenance:` | `validators/provenance.js` | The
|
|
98
|
+
| `provenance:` | `validators/provenance.js` | The run was genuinely **absent / not confirmable** — a 404 from the provider API, or the run head did not match the submitted commit/repo. The submitter's payload points at a run that does not exist or does not bind. (Operational provider faults — 429/5xx/401/403 — are NOT this class; see `provenance-fault:` below.) |
|
|
98
99
|
| `repo:` | `index.js` cross-field guard | `submission.repo` does not match the owner/repo encoded in `source.run_url` (anti-forgery guard). Emitted as `repo:mismatch: …`. |
|
|
99
100
|
| `submission-contains-verifier-field:` | `index.js` | The submission carried a verifier-owned field (`policy_version`, `verification`, or an object `overall_verdict`) it must not author. |
|
|
100
101
|
| `CONTRACT_SCHEMA_TOO_NEW:` | `validators/schema-version.js` | The submission's `schema_version` declares a MAJOR **above** what this build supports (see `SUPPORTED_SCHEMA_VERSIONS` in `@dogfood-lab/schemas`). This build cannot understand a future contract — **the operator must upgrade testing-os**, but the routing class stays submission-bad (the payload as-shipped cannot be accepted by THIS build). |
|
|
@@ -105,10 +106,11 @@ Discrimination happens by **class**, surfaced by `parseRejectionReason` (below).
|
|
|
105
106
|
| Prefix | Source | Meaning |
|
|
106
107
|
|---|---|---|
|
|
107
108
|
| `VALIDATOR_FAULT_SCHEMA:` | `runValidator('schema', …)` catch | Internal exception inside the schema validator. The rest of the string carries the thrown `.message`. |
|
|
108
|
-
| `VALIDATOR_FAULT_POLICY:` | `runValidator('policy', …)` catch | Internal exception inside the policy validator. |
|
|
109
|
+
| `VALIDATOR_FAULT_POLICY:` | `runValidator('policy', …)` catch | Internal exception inside the policy validator — including a **GLOBAL** declarative-rule predicate fault (VERIFY-F1): a broken `policies/global-policy.yaml` is an ops incident (the studio's own config), so its predicate fault throws here rather than bouncing to the submitter. The repo-authored counterpart is `policy-config:` submission-bad. |
|
|
109
110
|
| `VALIDATOR_FAULT_STEPS:` | `runValidator('steps', …)` catch | Internal exception inside the steps validator. |
|
|
110
111
|
| `VALIDATOR_FAULT_CONTRACT_SCHEMA_VERSION:` | `runValidator('contract_schema_version', …)` catch | The version gate was called with an unknown contract key (a programmer error at the call site, not a submission fault). |
|
|
111
112
|
| `submission-malformed:` | `index.js` null/non-object early-return | The submission itself was `null` or not an object — a malfunctioning **dispatcher** sent garbage, not a submitter who authored a bad-but-shaped payload. Page ops / inspect the dispatch pipeline; do NOT bounce it to a submitter. |
|
|
113
|
+
| `provenance-fault:` | `index.js` provenance catch | The provenance adapter THREW an operational error confirming the run — a provider **429 rate-limit, 5xx outage, or 401/403 token** fault (`validators/provenance.js` throws these on purpose for non-404 responses). The submitter's payload is fine; the verifier could not reach a verdict. Page ops / retry; do NOT bounce it to a submitter. Distinct from the submission-bad `provenance:` (genuine absence/404). |
|
|
112
114
|
|
|
113
115
|
Any future `VALIDATOR_FAULT_<NEW>:` prefix is classified `operational` by family — `parseRejectionReason` matches the `VALIDATOR_FAULT_` head, so a new validator class needs no parser edit. The `submission-malformed:` prefix is matched literally (it is not part of the `VALIDATOR_FAULT_` family).
|
|
114
116
|
|
|
@@ -153,6 +155,10 @@ for (const r of result.rejection_reasons) {
|
|
|
153
155
|
|
|
154
156
|
Persistence note: every entry above is round-tripped verbatim through `verification.rejection_reasons` in the persisted-record JSON; the schema enforces `array of string` so any consumer of the audit-DB ground truth sees the same prefix vocabulary.
|
|
155
157
|
|
|
158
|
+
### Warnings channel (accepted-with-warning)
|
|
159
|
+
|
|
160
|
+
Not every policy signal is a rejection. A policy rule declared `severity: warn` produces a `policy: <id>: <message>` entry on `verification.warnings` (an optional `array of string` on the persisted record) **without** flipping the verdict to `rejected` — the submission is accepted and recorded, the warning rides alongside it. (`severity: info` rules are logged only and never persisted; `severity: reject` rules go to `rejection_reasons` as above.) Consumers that want advisory signals read `verification.warnings`; the routing decision (`parseRejectionReason`) only concerns `rejection_reasons`. A clean accepted submission carries no `warnings` key at all.
|
|
161
|
+
|
|
156
162
|
## Docs
|
|
157
163
|
|
|
158
164
|
📖 Full handbook: **<https://dogfood-lab.github.io/testing-os/handbook/>**
|
package/cli.js
CHANGED
|
@@ -316,7 +316,10 @@ export function renderExplain(record) {
|
|
|
316
316
|
const accepted = v.status === 'accepted';
|
|
317
317
|
const lines = [];
|
|
318
318
|
|
|
319
|
-
|
|
319
|
+
// Uppercase the rendered state word to match the sibling verdict banners
|
|
320
|
+
// (findings-render.js). This is display-only; the underlying enum
|
|
321
|
+
// (v.status, --json output) stays lowercase.
|
|
322
|
+
lines.push(`VERDICT: ${(v.status || (accepted ? 'accepted' : 'rejected')).toUpperCase()}`);
|
|
320
323
|
lines.push('');
|
|
321
324
|
lines.push(` schema_valid: ${v.schema_valid}`);
|
|
322
325
|
lines.push(` policy_valid: ${v.policy_valid}`);
|
package/index.js
CHANGED
|
@@ -209,9 +209,15 @@ export async function verify(submission, options) {
|
|
|
209
209
|
refCommitSha: submission.ref?.commit_sha
|
|
210
210
|
});
|
|
211
211
|
} catch (err) {
|
|
212
|
-
|
|
212
|
+
// verify-A-002: the adapter THROWS on operational provider faults
|
|
213
|
+
// (429 rate-limit, 5xx outage, 401/403 token) and returns false only for
|
|
214
|
+
// a genuinely-absent run (404/transport). Emit a DISTINCT `provenance-fault:`
|
|
215
|
+
// prefix here so parseRejectionReason routes the incident to ops instead of
|
|
216
|
+
// bouncing an outage back to the submitter as submission-bad. The not-confirmed
|
|
217
|
+
// case below keeps the bare `provenance:` prefix (still submission-bad).
|
|
218
|
+
reasons.push(`provenance-fault: verification failed: ${err.message}`);
|
|
213
219
|
}
|
|
214
|
-
if (!provenanceConfirmed && !reasons.some(r => r.startsWith('provenance
|
|
220
|
+
if (!provenanceConfirmed && !reasons.some(r => r.startsWith('provenance'))) {
|
|
215
221
|
reasons.push('provenance: source run could not be confirmed');
|
|
216
222
|
}
|
|
217
223
|
}
|
|
@@ -243,6 +249,12 @@ export async function verify(submission, options) {
|
|
|
243
249
|
// BEFORE handing the (broken) policy to `validatePolicy`, so the
|
|
244
250
|
// operator sees a real "policy:" rejection in `rejection_reasons`.
|
|
245
251
|
let policyValid = false;
|
|
252
|
+
// VERIFY-F4: severity:warn / severity:info policy rules surface here as an
|
|
253
|
+
// accepted-with-warning channel. Warnings NEVER enter `reasons` (which become
|
|
254
|
+
// rejection_reasons and flip status to 'rejected') — they land on
|
|
255
|
+
// verification.warnings at assembly so an operator sees the advisory without
|
|
256
|
+
// the submission being bounced.
|
|
257
|
+
const warnings = [];
|
|
246
258
|
if (schemaResult.valid) {
|
|
247
259
|
if (repoPolicy && repoPolicy.__torn === true) {
|
|
248
260
|
const detail = repoPolicy.reason || 'repo policy YAML failed to parse';
|
|
@@ -253,6 +265,12 @@ export async function verify(submission, options) {
|
|
|
253
265
|
if (policyRun.ok) {
|
|
254
266
|
policyValid = policyRun.result.valid;
|
|
255
267
|
reasons.push(...policyRun.result.errors.map(e => `policy: ${e}`));
|
|
268
|
+
// VERIFY-F1: a malformed REPO custom-rule predicate is a distinct
|
|
269
|
+
// `policy-config:` rejection (submission-bad — the repo authored the bad
|
|
270
|
+
// rule). A malformed GLOBAL predicate never reaches here; it throws and
|
|
271
|
+
// surfaces as VALIDATOR_FAULT_POLICY (operational) via the else branch.
|
|
272
|
+
reasons.push(...(policyRun.result.configErrors || []).map(e => `policy-config: ${e}`));
|
|
273
|
+
warnings.push(...(policyRun.result.warnings || []).map(w => `policy: ${w}`));
|
|
256
274
|
} else {
|
|
257
275
|
reasons.push(policyRun.faultReason);
|
|
258
276
|
}
|
|
@@ -300,7 +318,8 @@ export async function verify(submission, options) {
|
|
|
300
318
|
provenance_confirmed: provenanceConfirmed,
|
|
301
319
|
schema_valid: schemaResult.valid,
|
|
302
320
|
policy_valid: policyValid,
|
|
303
|
-
rejection_reasons: reasons
|
|
321
|
+
rejection_reasons: reasons,
|
|
322
|
+
...(warnings.length ? { warnings } : {})
|
|
304
323
|
},
|
|
305
324
|
...(submission.notes ? { notes: submission.notes } : {})
|
|
306
325
|
};
|
package/package.json
CHANGED
package/parse-rejection.js
CHANGED
|
@@ -23,8 +23,12 @@
|
|
|
23
23
|
*
|
|
24
24
|
* The prefix vocabulary below is enumerated from the ACTUAL emitters — it is
|
|
25
25
|
* NOT invented:
|
|
26
|
-
* - verify/index.js: schema:, policy:,
|
|
27
|
-
*
|
|
26
|
+
* - verify/index.js: schema:, policy:, policy-config: (VERIFY-F1,
|
|
27
|
+
* repo custom-rule predicate fault → submission-bad),
|
|
28
|
+
* steps[<id>]:,
|
|
29
|
+
* provenance: (run absent → submission-bad),
|
|
30
|
+
* provenance-fault: (provider 429/5xx/401/403
|
|
31
|
+
* → operational), repo:,
|
|
28
32
|
* submission-contains-verifier-field:,
|
|
29
33
|
* submission-malformed:,
|
|
30
34
|
* VALIDATOR_FAULT_<NAME>:
|
|
@@ -60,7 +64,22 @@
|
|
|
60
64
|
const LITERAL_PREFIXES = [
|
|
61
65
|
// submission-bad
|
|
62
66
|
{ match: 'schema:', prefix: 'schema:', class: 'submission-bad' },
|
|
67
|
+
// VERIFY-F1: a malformed REPO custom-rule predicate (eval-time semantic fault the
|
|
68
|
+
// schema could not catch). submission-bad — the repo authored the bad policy YAML,
|
|
69
|
+
// so the fix belongs to the submitter, not studio ops. Ordered BEFORE `policy:`
|
|
70
|
+
// to keep the most-specific-first invariant (the tokens don't overlap — `policy:`
|
|
71
|
+
// ends at the colon, `policy-config:` continues with `-config:` — but the order is
|
|
72
|
+
// explicit). A malformed GLOBAL predicate is operational (VALIDATOR_FAULT_POLICY).
|
|
73
|
+
{ match: 'policy-config:', prefix: 'policy-config:', class: 'submission-bad' },
|
|
63
74
|
{ match: 'policy:', prefix: 'policy:', class: 'submission-bad' },
|
|
75
|
+
// operational — a provider-side provenance FAULT (429/5xx/401/403). The
|
|
76
|
+
// adapter THROWS these (verify-A-002); index.js catches the throw and emits
|
|
77
|
+
// this distinct `provenance-fault:` prefix so the incident pages ops. Ordered
|
|
78
|
+
// before the bare `provenance:` literal — it is a distinct token, but keeping
|
|
79
|
+
// the longer, more-specific match first preserves the most-specific-first
|
|
80
|
+
// invariant the array is sorted by. The genuine not-confirmed case
|
|
81
|
+
// (`provenance: source run could not be confirmed`) stays submission-bad below.
|
|
82
|
+
{ match: 'provenance-fault:', prefix: 'provenance-fault:', class: 'operational' },
|
|
64
83
|
{ match: 'provenance:', prefix: 'provenance:', class: 'submission-bad' },
|
|
65
84
|
{ match: 'repo:', prefix: 'repo:', class: 'submission-bad' },
|
|
66
85
|
{
|
package/validators/policy.js
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* Global rules are non-overridable. Repo policies add surface-specific requirements.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
import { evaluatePredicate, buildReason, PredicateError } from './predicate.js';
|
|
9
|
+
|
|
8
10
|
function deepMerge(target, source) {
|
|
9
11
|
const result = { ...target };
|
|
10
12
|
for (const key of Object.keys(source)) {
|
|
@@ -29,6 +31,58 @@ function deepMerge(target, source) {
|
|
|
29
31
|
return result;
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Global reject-rule ids the build can account for. Two groups:
|
|
36
|
+
* - HANDLED HERE — enforced by the switch below.
|
|
37
|
+
* - ENFORCED ELSEWHERE — owned by another validator or by verify() itself
|
|
38
|
+
* (the switch's `default` arm intentionally no-ops them).
|
|
39
|
+
*
|
|
40
|
+
* PROACT-VERIFY-002: any `severity: reject` rule whose id is OUTSIDE this set is
|
|
41
|
+
* an operator-added gate the build does not enforce. Silently no-op'ing it (the
|
|
42
|
+
* old `default: break`) meant the rule looked active in global-policy.yaml but
|
|
43
|
+
* never ran. We now surface an actionable diagnostic instead. Keep this set in
|
|
44
|
+
* sync when a new reject rule gains real enforcement.
|
|
45
|
+
*/
|
|
46
|
+
const KNOWN_REJECT_RULE_IDS = new Set([
|
|
47
|
+
// handled by the switch in validatePolicy
|
|
48
|
+
'scenario-minimum',
|
|
49
|
+
'blocked-needs-reason',
|
|
50
|
+
// enforced by other validators or by verify() itself (default-arm no-op is correct)
|
|
51
|
+
'schema-valid',
|
|
52
|
+
'provenance-confirmed',
|
|
53
|
+
'step-results-present',
|
|
54
|
+
'step-verdict-consistent',
|
|
55
|
+
'no-verdict-upgrade',
|
|
56
|
+
]);
|
|
57
|
+
|
|
58
|
+
// VERIFY-F1: `attested-if-human` is intentionally NOT in the set above. It used to
|
|
59
|
+
// be a switch arm; it is now enforced DECLARATIVELY (it carries a `when` predicate
|
|
60
|
+
// in policies/global-policy.yaml and flows through the engine). If it ever appears
|
|
61
|
+
// WITHOUT a `when` — a misconfigured migration — it falls to the default arm and is
|
|
62
|
+
// NOT in KNOWN_REJECT_RULE_IDS, so it surfaces the "no enforcement" diagnostic
|
|
63
|
+
// (fail-closed) instead of silently doing nothing.
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* VERIFY-F1: evaluate a declarative rule's `when` predicate against each root and
|
|
67
|
+
* return one reason string (`[id] body`) per match. Throws {@link PredicateError}
|
|
68
|
+
* on a semantic fault; the caller decides whether that is operational (global rule)
|
|
69
|
+
* or submission-bad (repo custom rule).
|
|
70
|
+
*/
|
|
71
|
+
function collectMatches(rule, roots, scope) {
|
|
72
|
+
const reasons = [];
|
|
73
|
+
for (const root of roots) {
|
|
74
|
+
if (evaluatePredicate(rule.when, root, scope)) reasons.push(buildReason(rule, root));
|
|
75
|
+
}
|
|
76
|
+
return reasons;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Route matched reasons by severity. reject -> errors, warn -> warnings, info -> logged-only. */
|
|
80
|
+
function routeReasons(rule, reasons, errors, warnings) {
|
|
81
|
+
if (rule.severity === 'reject') errors.push(...reasons);
|
|
82
|
+
else if (rule.severity === 'warn') warnings.push(...reasons);
|
|
83
|
+
// info: intentionally non-surfacing
|
|
84
|
+
}
|
|
85
|
+
|
|
32
86
|
/**
|
|
33
87
|
* Resolve the effective surface policy for a given product surface.
|
|
34
88
|
* Repo policy overrides global defaults per surface.
|
|
@@ -55,16 +109,59 @@ function resolveSurfacePolicy(surface, globalPolicy, repoPolicy) {
|
|
|
55
109
|
* @param {object} options
|
|
56
110
|
* @param {object} options.globalPolicy
|
|
57
111
|
* @param {object|null} options.repoPolicy
|
|
58
|
-
* @returns {{ valid: boolean, errors: string[] }}
|
|
112
|
+
* @returns {{ valid: boolean, errors: string[], warnings: string[], configErrors: string[] }}
|
|
113
|
+
* `configErrors` (VERIFY-F1) are eval-time repo custom-rule predicate faults; the
|
|
114
|
+
* caller emits them with a `policy-config:` prefix (submission-bad). A non-empty
|
|
115
|
+
* `configErrors` makes `valid` false.
|
|
59
116
|
*/
|
|
60
117
|
export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
61
118
|
const errors = [];
|
|
119
|
+
// VERIFY-F4: warn-severity rules record an accepted-with-warning note here
|
|
120
|
+
// instead of being silently dropped. A populated warnings[] never affects
|
|
121
|
+
// `valid` — the caller (index.js) routes it to verification.warnings.
|
|
122
|
+
const warnings = [];
|
|
123
|
+
// VERIFY-F1: a malformed REPO custom-rule predicate (an eval-time semantic fault
|
|
124
|
+
// the schema could not catch — unknown leading field, numeric op vs non-number,
|
|
125
|
+
// over-depth) is a `policy-config:` submission-bad reason recorded here (the repo
|
|
126
|
+
// authored the bad rule; do NOT page studio ops). A non-empty configErrors flips
|
|
127
|
+
// `valid` false (fail-closed). The caller prefixes these `policy-config: `, not
|
|
128
|
+
// `policy: `. Global-rule predicate faults are NOT collected here — they throw
|
|
129
|
+
// (-> VALIDATOR_FAULT_POLICY, operational).
|
|
130
|
+
const configErrors = [];
|
|
62
131
|
|
|
63
132
|
// --- Global rules (non-overridable) ---
|
|
64
133
|
|
|
65
134
|
const globalRules = globalPolicy.global_rules || [];
|
|
66
135
|
|
|
67
136
|
for (const rule of globalRules) {
|
|
137
|
+
// VERIFY-F1: a rule carrying a `when` predicate is enforced DECLARATIVELY by
|
|
138
|
+
// the engine; a rule WITHOUT `when` falls to the legacy severity handling +
|
|
139
|
+
// id switch below (the seven built-ins that stay code-enforced). A GLOBAL-rule
|
|
140
|
+
// predicate fault is allowed to THROW (-> VALIDATOR_FAULT_POLICY, operational):
|
|
141
|
+
// a broken global policy is an operator/ops incident, mirroring loadGlobalPolicy's
|
|
142
|
+
// fail-loud schema gate. scope 'scenario_result' evaluates per element in array
|
|
143
|
+
// order (one reason per offending element); 'submission' (default) evaluates once.
|
|
144
|
+
if (rule.when) {
|
|
145
|
+
const scope = rule.scope || 'submission';
|
|
146
|
+
const roots = scope === 'scenario_result'
|
|
147
|
+
? (submission.scenario_results || [])
|
|
148
|
+
: [submission];
|
|
149
|
+
routeReasons(rule, collectMatches(rule, roots, scope), errors, warnings);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// VERIFY-F4: a global warn-rule is accepted-with-warning; an info-rule logs
|
|
154
|
+
// only. Neither may reject. Mirrors the PROACT-VERIFY-002 discipline of not
|
|
155
|
+
// dropping operator-authored rules on the floor: a declared rule that the
|
|
156
|
+
// build does nothing with is invisible to the operator who wrote it.
|
|
157
|
+
if (rule.severity === 'warn') {
|
|
158
|
+
warnings.push(`${rule.id}: ${rule.description || 'policy warning'}`);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
if (rule.severity === 'info') {
|
|
162
|
+
// Logged only — info rules are intentionally non-surfacing in the record.
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
68
165
|
if (rule.severity !== 'reject') continue;
|
|
69
166
|
|
|
70
167
|
switch (rule.id) {
|
|
@@ -74,15 +171,10 @@ export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
|
74
171
|
}
|
|
75
172
|
break;
|
|
76
173
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
`[${rule.id}] scenario "${sr.scenario_id}": execution_mode is "${sr.execution_mode}" but attested_by is missing`
|
|
82
|
-
);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
break;
|
|
174
|
+
// VERIFY-F1: the `attested-if-human` switch arm was removed in v1.7.0. The rule
|
|
175
|
+
// is now enforced declaratively via a `when` predicate (see the `if (rule.when)`
|
|
176
|
+
// branch above and policies/global-policy.yaml). The differential-equivalence
|
|
177
|
+
// gate proves the declarative form is byte-identical to this removed arm.
|
|
86
178
|
|
|
87
179
|
case 'blocked-needs-reason':
|
|
88
180
|
for (const sr of submission.scenario_results || []) {
|
|
@@ -95,8 +187,20 @@ export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
|
95
187
|
break;
|
|
96
188
|
|
|
97
189
|
// schema-valid, provenance-confirmed, step-results-present, step-verdict-consistent,
|
|
98
|
-
// no-verdict-upgrade are enforced by other validators or the main verify() function
|
|
190
|
+
// no-verdict-upgrade are enforced by other validators or the main verify() function.
|
|
191
|
+
// PROACT-VERIFY-002: a reject rule the build neither handles here NOR enforces
|
|
192
|
+
// elsewhere would otherwise pass SILENTLY — the operator's new gate never runs.
|
|
193
|
+
// Reject the submission with a diagnostic naming the unenforced rule so the gap
|
|
194
|
+
// is visible instead of failing open.
|
|
99
195
|
default:
|
|
196
|
+
if (!KNOWN_REJECT_RULE_IDS.has(rule.id)) {
|
|
197
|
+
// No `policy:` prefix here — index.js prepends it to every policy
|
|
198
|
+
// error (mirrors the `[rule.id]` / `surface[...]` messages above).
|
|
199
|
+
errors.push(
|
|
200
|
+
`rule "${rule.id}" is declared severity:reject but has no enforcement in this build — ` +
|
|
201
|
+
`add an enforcement arm in validators/policy.js or register it in KNOWN_REJECT_RULE_IDS`
|
|
202
|
+
);
|
|
203
|
+
}
|
|
100
204
|
break;
|
|
101
205
|
}
|
|
102
206
|
}
|
|
@@ -134,6 +238,49 @@ export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
|
134
238
|
}
|
|
135
239
|
}
|
|
136
240
|
}
|
|
241
|
+
|
|
242
|
+
// VERIFY-F2: tag gating. forbidden_tags rejects a scenario_result carrying
|
|
243
|
+
// any listed tag; required_tags rejects one missing any listed tag (a
|
|
244
|
+
// tagless scenario fails every required_tags rule). Tags are optional on
|
|
245
|
+
// the scenario_result, so an absent `tags` array trips required_tags but
|
|
246
|
+
// never forbidden_tags.
|
|
247
|
+
const tags = new Set(sr.tags || []);
|
|
248
|
+
|
|
249
|
+
if (evidenceReqs.forbidden_tags) {
|
|
250
|
+
for (const tag of evidenceReqs.forbidden_tags) {
|
|
251
|
+
if (tags.has(tag)) {
|
|
252
|
+
errors.push(
|
|
253
|
+
`surface[${surface}]: scenario "${sr.scenario_id}" carries forbidden tag "${tag}"`
|
|
254
|
+
);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (evidenceReqs.required_tags) {
|
|
260
|
+
for (const tag of evidenceReqs.required_tags) {
|
|
261
|
+
if (!tags.has(tag)) {
|
|
262
|
+
errors.push(
|
|
263
|
+
`surface[${surface}]: scenario "${sr.scenario_id}" is missing required tag "${tag}"`
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// VERIFY-F1: declarative custom rules for this surface, evaluated per
|
|
271
|
+
// scenario_result. Additive-only (reject/warn/info, no accept verb) so a repo
|
|
272
|
+
// can never weaken a global gate. A predicate fault here is a `policy-config:`
|
|
273
|
+
// submission-bad reason (the repo authored the bad rule), NOT an ops page.
|
|
274
|
+
for (const customRule of surfacePolicy.custom_rules || []) {
|
|
275
|
+
try {
|
|
276
|
+
routeReasons(customRule, collectMatches(customRule, [sr], 'scenario_result'), errors, warnings);
|
|
277
|
+
} catch (err) {
|
|
278
|
+
if (err instanceof PredicateError) {
|
|
279
|
+
configErrors.push(`rule "${customRule.id}": ${err.message}`);
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
throw err;
|
|
283
|
+
}
|
|
137
284
|
}
|
|
138
285
|
}
|
|
139
286
|
|
|
@@ -176,5 +323,10 @@ export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
|
176
323
|
}
|
|
177
324
|
}
|
|
178
325
|
|
|
179
|
-
return {
|
|
326
|
+
return {
|
|
327
|
+
valid: errors.length === 0 && configErrors.length === 0,
|
|
328
|
+
errors,
|
|
329
|
+
warnings,
|
|
330
|
+
configErrors,
|
|
331
|
+
};
|
|
180
332
|
}
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Predicate engine (VERIFY-F1)
|
|
3
|
+
*
|
|
4
|
+
* A bounded, NO-EVAL interpreter for the declarative policy DSL. Operators author
|
|
5
|
+
* rules as structured predicates in their YAML policy file (field-selector +
|
|
6
|
+
* operator + value, composed with all/any/not/implies); this module evaluates
|
|
7
|
+
* them. There is no `eval`, `Function`, `vm`, dynamic import, or template-string
|
|
8
|
+
* execution anywhere in the path — the evaluator is a pure tree-walk. That is the
|
|
9
|
+
* whole safety thesis (see docs/policy-dsl.md).
|
|
10
|
+
*
|
|
11
|
+
* The schema (`policy.schema.json`, recursive `predicate` $def) is the FIRST gate:
|
|
12
|
+
* unknown operators, malformed nodes, banned path segments, and arity errors are
|
|
13
|
+
* rejected at policy-LOAD time by `validatePayload('policy', …)` and inherit the
|
|
14
|
+
* existing origin-based classification (global policy → throws/operational; repo
|
|
15
|
+
* policy → `__torn`/submission-bad). This module is the SECOND gate: the residual
|
|
16
|
+
* SEMANTIC faults the schema cannot express — an unknown leading field, a numeric
|
|
17
|
+
* operator against a non-number, or a predicate nested past the depth cap — surface
|
|
18
|
+
* as a {@link PredicateError} that the caller (policy.js) turns into a
|
|
19
|
+
* `policy-config:` reason, classified by predicate origin. A predicate is never
|
|
20
|
+
* silently skipped and never throws uncaught (fail-closed; Saltzer & Schroeder 1975).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { createRequire } from 'node:module';
|
|
24
|
+
import { readFileSync } from 'node:fs';
|
|
25
|
+
|
|
26
|
+
/** Combinator nesting cap. A predicate nested deeper is a `max_depth` fault. */
|
|
27
|
+
export const PREDICATE_MAX_DEPTH = 5;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Total-work budgets (VERIFY-F1 Phase 3 hardening — the depth cap bounds NESTING
|
|
31
|
+
* but not WIDTH or array fan-out). The engine runs synchronously inside the ingest
|
|
32
|
+
* pipeline, so an unbounded tree-walk would block the event loop. Both budgets are
|
|
33
|
+
* generous relative to any real policy (a hand-authored rule is a handful of nodes
|
|
34
|
+
* over a handful of elements) and exist only to refuse a pathological / hostile
|
|
35
|
+
* predicate as a classified `policy-config:` fault rather than letting it spin.
|
|
36
|
+
*/
|
|
37
|
+
export const PREDICATE_MAX_NODES = 10_000; // visited predicate nodes per evaluation
|
|
38
|
+
export const PREDICATE_MAX_FRONTIER = 500_000; // selected values from a `[]` fan-out
|
|
39
|
+
|
|
40
|
+
/** Field-path segments that must never be traversed (prototype-pollution guard). */
|
|
41
|
+
const POISON_SEGMENTS = new Set(['__proto__', 'constructor', 'prototype']);
|
|
42
|
+
|
|
43
|
+
/** The closed operator set. Mirrors the `op` enum in policy.schema.json. */
|
|
44
|
+
const OPERATORS = new Set([
|
|
45
|
+
'equals', 'not_equals', 'in', 'not_in', 'contains', 'not_contains',
|
|
46
|
+
'exists', 'not_exists', 'gt', 'gte', 'lt', 'lte',
|
|
47
|
+
]);
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* A structured predicate fault (compile- or eval-time). The `code` is a stable
|
|
51
|
+
* machine token; the message is operator-actionable (names the rule + the exact
|
|
52
|
+
* problem). The caller maps this to a `policy-config:` rejection reason, classified
|
|
53
|
+
* by the predicate's origin (global → operational, repo → submission-bad).
|
|
54
|
+
*/
|
|
55
|
+
export class PredicateError extends Error {
|
|
56
|
+
constructor(code, message) {
|
|
57
|
+
super(message);
|
|
58
|
+
this.name = 'PredicateError';
|
|
59
|
+
this.code = code;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Known leading-field sets per scope, derived from the submission schema so the
|
|
65
|
+
* allowlist never drifts from the contract (CLAUDE.md rule #5: schemas via
|
|
66
|
+
* createRequire). Only the LEADING path segment is validated — a deeper typo just
|
|
67
|
+
* resolves to `undefined` (no match), but a wrong leading field is an operator
|
|
68
|
+
* error worth a diagnostic.
|
|
69
|
+
*/
|
|
70
|
+
const KNOWN_FIELDS = (() => {
|
|
71
|
+
const require = createRequire(import.meta.url);
|
|
72
|
+
const schemaPath = require.resolve('@dogfood-lab/schemas/json/dogfood-record-submission.schema.json');
|
|
73
|
+
const schema = JSON.parse(readFileSync(schemaPath, 'utf-8'));
|
|
74
|
+
const submission = new Set(Object.keys(schema.properties || {}));
|
|
75
|
+
const srProps = schema.properties?.scenario_results?.items?.properties || {};
|
|
76
|
+
const scenario_result = new Set(Object.keys(srProps));
|
|
77
|
+
return { submission, scenario_result };
|
|
78
|
+
})();
|
|
79
|
+
|
|
80
|
+
/** Split a dotted field path into segments, normalizing the `[]` array marker. */
|
|
81
|
+
function splitPath(field) {
|
|
82
|
+
// 'scenario_results[].tags' -> ['scenario_results', '[]', 'tags']
|
|
83
|
+
const segments = [];
|
|
84
|
+
for (const raw of String(field).split('.')) {
|
|
85
|
+
const m = raw.match(/^([A-Za-z_][A-Za-z0-9_]*)(\[\])?$/);
|
|
86
|
+
if (!m) throw new PredicateError('bad_field', `field path segment "${raw}" is malformed`);
|
|
87
|
+
segments.push(m[1]);
|
|
88
|
+
if (m[2]) segments.push('[]');
|
|
89
|
+
}
|
|
90
|
+
return segments;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Read one own-property segment without ever touching the prototype chain. */
|
|
94
|
+
function safeGet(value, segment) {
|
|
95
|
+
if (POISON_SEGMENTS.has(segment)) {
|
|
96
|
+
// Defense in depth — the schema `pattern` already bans these, but the
|
|
97
|
+
// evaluator never trusts the schema alone for a security-critical read.
|
|
98
|
+
throw new PredicateError('banned_segment', `field path segment "${segment}" is forbidden`);
|
|
99
|
+
}
|
|
100
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) return undefined;
|
|
101
|
+
return Object.hasOwn(value, segment) ? value[segment] : undefined;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Resolve a field path to the list of values it selects. A path with no `[]`
|
|
106
|
+
* yields exactly one value (possibly `undefined`, preserving "field absent" so
|
|
107
|
+
* `not_exists` works). Each `[]` segment expands the current frontier across the
|
|
108
|
+
* elements of the arrays at that position. The only iteration in the whole DSL.
|
|
109
|
+
*/
|
|
110
|
+
function resolvePath(root, segments) {
|
|
111
|
+
let frontier = [root];
|
|
112
|
+
for (const seg of segments) {
|
|
113
|
+
if (seg === '[]') {
|
|
114
|
+
const next = [];
|
|
115
|
+
for (const v of frontier) {
|
|
116
|
+
if (!Array.isArray(v)) continue;
|
|
117
|
+
// Explicit element loop, NOT `next.push(...v)` — the spread form blows the
|
|
118
|
+
// call stack on a large array (RangeError) before any budget check fires.
|
|
119
|
+
for (const el of v) {
|
|
120
|
+
if (next.length >= PREDICATE_MAX_FRONTIER) {
|
|
121
|
+
throw new PredicateError(
|
|
122
|
+
'fanout_budget',
|
|
123
|
+
`field path selection exceeds the limit of ${PREDICATE_MAX_FRONTIER} elements`
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
next.push(el);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
frontier = next;
|
|
130
|
+
} else {
|
|
131
|
+
frontier = frontier.map(v => safeGet(v, seg));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return frontier;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Validate the leading field segment against the scope's known-field set. */
|
|
138
|
+
function checkLeadingField(field, scope) {
|
|
139
|
+
const leading = String(field).split('.')[0].replace('[]', '');
|
|
140
|
+
const known = KNOWN_FIELDS[scope];
|
|
141
|
+
if (known && !known.has(leading)) {
|
|
142
|
+
throw new PredicateError(
|
|
143
|
+
'unknown_field',
|
|
144
|
+
`field "${field}" references unknown leading field "${leading}" (known ${scope} fields: ${[...known].join(', ')})`
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Apply one operator to a single resolved value + comparand. */
|
|
150
|
+
function applyOp(op, value, comparand) {
|
|
151
|
+
switch (op) {
|
|
152
|
+
case 'equals': return value === comparand;
|
|
153
|
+
case 'not_equals': return value !== comparand;
|
|
154
|
+
case 'in': return Array.isArray(comparand) && comparand.includes(value);
|
|
155
|
+
case 'not_in': return Array.isArray(comparand) && !comparand.includes(value);
|
|
156
|
+
case 'contains':
|
|
157
|
+
if (Array.isArray(value)) return value.includes(comparand);
|
|
158
|
+
if (typeof value === 'string') return value.includes(comparand);
|
|
159
|
+
return false;
|
|
160
|
+
case 'not_contains':
|
|
161
|
+
// not_contains is the negation of contains, so a missing/non-collection
|
|
162
|
+
// field genuinely "does not contain" the value (matches the v1.6.0
|
|
163
|
+
// required_tags semantics — a tagless scenario fails required_tags).
|
|
164
|
+
return !applyOp('contains', value, comparand);
|
|
165
|
+
case 'exists': return Boolean(value);
|
|
166
|
+
case 'not_exists': return !value;
|
|
167
|
+
case 'gt': case 'gte': case 'lt': case 'lte': {
|
|
168
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
169
|
+
throw new PredicateError(
|
|
170
|
+
'type_mismatch',
|
|
171
|
+
`operator "${op}" requires a numeric field value but got ${describe(value)}`
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
if (op === 'gt') return value > comparand;
|
|
175
|
+
if (op === 'gte') return value >= comparand;
|
|
176
|
+
if (op === 'lt') return value < comparand;
|
|
177
|
+
return value <= comparand;
|
|
178
|
+
}
|
|
179
|
+
default:
|
|
180
|
+
// Unreachable when the schema gate ran (op is enum-constrained), but the
|
|
181
|
+
// evaluator fails closed rather than trusting the schema.
|
|
182
|
+
throw new PredicateError('unknown_op', `unknown operator "${op}"`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function describe(value) {
|
|
187
|
+
if (value === undefined) return 'undefined (field absent)';
|
|
188
|
+
if (value === null) return 'null';
|
|
189
|
+
if (Array.isArray(value)) return 'an array';
|
|
190
|
+
return `${typeof value} (${JSON.stringify(value)})`;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function isCombinator(node) {
|
|
194
|
+
return node != null && typeof node === 'object' &&
|
|
195
|
+
(Array.isArray(node.all) || Array.isArray(node.any) || node.not != null || Array.isArray(node.implies));
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Evaluate a leaf `{field, op, value?}` against a root, with "any element" semantics. */
|
|
199
|
+
function evalLeaf(node, root, scope) {
|
|
200
|
+
if (!OPERATORS.has(node.op)) {
|
|
201
|
+
throw new PredicateError('unknown_op', `unknown operator "${node.op}"`);
|
|
202
|
+
}
|
|
203
|
+
checkLeadingField(node.field, scope);
|
|
204
|
+
const values = resolvePath(root, splitPath(node.field));
|
|
205
|
+
// The predicate is true when ANY selected value satisfies the operator. For a
|
|
206
|
+
// scalar (no `[]`) path this is just that single value; for an empty `[]`
|
|
207
|
+
// selection there is nothing to match, so the leaf is false.
|
|
208
|
+
for (const v of values) {
|
|
209
|
+
if (applyOp(node.op, v, node.value)) return true;
|
|
210
|
+
}
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Evaluate a predicate node against a root object (the whole submission for
|
|
216
|
+
* `submission` scope, or a single scenario_result for `scenario_result` scope).
|
|
217
|
+
*
|
|
218
|
+
* @param {object} node - The predicate tree.
|
|
219
|
+
* @param {object} root - The data to evaluate against.
|
|
220
|
+
* @param {'submission'|'scenario_result'} scope - Field-resolution scope.
|
|
221
|
+
* @returns {boolean} True when the (violation) predicate matches.
|
|
222
|
+
* @throws {PredicateError} On a semantic fault (unknown field, type mismatch, depth, budget).
|
|
223
|
+
*/
|
|
224
|
+
export function evaluatePredicate(node, root, scope) {
|
|
225
|
+
// The budget is per-evaluation (one call over one root). It bounds the WIDTH a
|
|
226
|
+
// single predicate can consume, complementing the per-node depth cap.
|
|
227
|
+
return evalNode(node, root, scope, 1, { nodes: 0 });
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function evalNode(node, root, scope, depth, budget) {
|
|
231
|
+
if (++budget.nodes > PREDICATE_MAX_NODES) {
|
|
232
|
+
throw new PredicateError(
|
|
233
|
+
'node_budget',
|
|
234
|
+
`predicate exceeds the evaluation budget of ${PREDICATE_MAX_NODES} nodes`
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
if (isCombinator(node)) {
|
|
238
|
+
if (depth > PREDICATE_MAX_DEPTH) {
|
|
239
|
+
throw new PredicateError(
|
|
240
|
+
'max_depth',
|
|
241
|
+
`predicate nests deeper than the limit of ${PREDICATE_MAX_DEPTH} combinator levels`
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
if (Array.isArray(node.all)) return node.all.every(c => evalNode(c, root, scope, depth + 1, budget));
|
|
245
|
+
if (Array.isArray(node.any)) return node.any.some(c => evalNode(c, root, scope, depth + 1, budget));
|
|
246
|
+
if (node.not != null) return !evalNode(node.not, root, scope, depth + 1, budget);
|
|
247
|
+
// implies: [antecedent, consequent] is the VIOLATION all(antecedent, not(consequent))
|
|
248
|
+
// — it matches the inputs that BREAK the implication antecedent => consequent.
|
|
249
|
+
const [antecedent, consequent] = node.implies;
|
|
250
|
+
return evalNode(antecedent, root, scope, depth + 1, budget)
|
|
251
|
+
&& !evalNode(consequent, root, scope, depth + 1, budget);
|
|
252
|
+
}
|
|
253
|
+
return evalLeaf(node, root, scope);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Interpolate a `reason_template`'s `{slot}` placeholders from a root object.
|
|
258
|
+
* Slots are read via the same prototype-safe accessor as the field selector and
|
|
259
|
+
* interpolated RAW (no escaping) — the differential-equivalence oracle wraps
|
|
260
|
+
* values in literal quotes, and any escaping would diverge byte-for-byte. A slot
|
|
261
|
+
* that resolves to nothing renders empty.
|
|
262
|
+
*/
|
|
263
|
+
function renderTemplate(template, root) {
|
|
264
|
+
return String(template).replace(/\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, slot) => {
|
|
265
|
+
// A poison slot is rendered empty — never traversed, never thrown. The own-
|
|
266
|
+
// property read avoids the prototype chain entirely (no inherited values leak).
|
|
267
|
+
if (POISON_SEGMENTS.has(slot)) return '';
|
|
268
|
+
const present = root !== null && typeof root === 'object' && Object.hasOwn(root, slot);
|
|
269
|
+
const v = present ? root[slot] : undefined;
|
|
270
|
+
return v === undefined || v === null ? '' : String(v);
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Build the full reason string for a matched rule: `[<id>] <body>`, where the
|
|
276
|
+
* body is the rendered `reason_template` or, absent that, the rule's
|
|
277
|
+
* `description`. The `policy:` prefix is NOT added here — verify/index.js prepends
|
|
278
|
+
* it to every policy error.
|
|
279
|
+
*
|
|
280
|
+
* @param {object} rule - The policy rule ({ id, description?, reason_template? }).
|
|
281
|
+
* @param {object} root - The matched element (scenario_result) or submission.
|
|
282
|
+
* @returns {string}
|
|
283
|
+
*/
|
|
284
|
+
export function buildReason(rule, root) {
|
|
285
|
+
const body = rule.reason_template != null
|
|
286
|
+
? renderTemplate(rule.reason_template, root)
|
|
287
|
+
: (rule.description ?? '');
|
|
288
|
+
return `[${rule.id}] ${body}`;
|
|
289
|
+
}
|
|
@@ -212,6 +212,7 @@ describe('gitlabProvenance distinguishes ops failures from missing runs', () =>
|
|
|
212
212
|
it(`throws an operational error on HTTP ${status} (not a submission-bad false)`, async () => {
|
|
213
213
|
const adapter = gitlabProvenance('token', {
|
|
214
214
|
timeoutMs: 1000,
|
|
215
|
+
retries: 0,
|
|
215
216
|
fetchImpl: fetchWithStatus(status)
|
|
216
217
|
});
|
|
217
218
|
await assert.rejects(
|
|
@@ -298,3 +299,77 @@ describe('gitlabProvenance input guards', () => {
|
|
|
298
299
|
assert.equal(ok, false);
|
|
299
300
|
});
|
|
300
301
|
});
|
|
302
|
+
|
|
303
|
+
// ── Bounded retry over transient faults (PROACT-VERIFY-001, mirrors GitHub) ──
|
|
304
|
+
//
|
|
305
|
+
// MEDIUM/resilience: a single 429/5xx blip used to fail the submission as an
|
|
306
|
+
// operational incident. The adapter now retries 429/5xx within a bounded budget
|
|
307
|
+
// (opts.retries, default 2) with exponential backoff (honoring Retry-After), and
|
|
308
|
+
// still THROWS on exhaustion so a genuinely-down provider surfaces as
|
|
309
|
+
// operational — never a false 'confirmed'. 404 is NOT retried.
|
|
310
|
+
|
|
311
|
+
describe('gitlabProvenance bounded retry (PROACT-VERIFY-001)', () => {
|
|
312
|
+
it('retries a 429 then confirms on the following 200 (retry worked)', async () => {
|
|
313
|
+
let calls = 0;
|
|
314
|
+
const fetchImpl = async () => {
|
|
315
|
+
calls++;
|
|
316
|
+
if (calls === 1) {
|
|
317
|
+
return { ok: false, status: 429, headers: { get: () => null }, json: async () => ({}) };
|
|
318
|
+
}
|
|
319
|
+
return { ok: true, status: 200, json: async () => mockPipeline() };
|
|
320
|
+
};
|
|
321
|
+
const adapter = gitlabProvenance('token', { timeoutMs: 1000, backoffMs: 1, fetchImpl });
|
|
322
|
+
const ok = await adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD });
|
|
323
|
+
assert.equal(ok, true, '429-then-200 must confirm — the retry succeeded');
|
|
324
|
+
assert.equal(calls, 2, 'expected exactly one retry (2 total requests)');
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
it('honors Retry-After (numeric seconds) before the retry', async () => {
|
|
328
|
+
let calls = 0;
|
|
329
|
+
const waited = [];
|
|
330
|
+
const fetchImpl = async () => {
|
|
331
|
+
calls++;
|
|
332
|
+
if (calls === 1) {
|
|
333
|
+
return { ok: false, status: 503, headers: { get: h => (h === 'retry-after' ? '2' : null) }, json: async () => ({}) };
|
|
334
|
+
}
|
|
335
|
+
return { ok: true, status: 200, json: async () => mockPipeline() };
|
|
336
|
+
};
|
|
337
|
+
const adapter = gitlabProvenance('token', {
|
|
338
|
+
timeoutMs: 1000,
|
|
339
|
+
sleepImpl: ms => { waited.push(ms); return Promise.resolve(); },
|
|
340
|
+
fetchImpl
|
|
341
|
+
});
|
|
342
|
+
const ok = await adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD });
|
|
343
|
+
assert.equal(ok, true);
|
|
344
|
+
assert.deepEqual(waited, [2000], 'Retry-After: 2 must drive a 2000ms wait');
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
it('THROWS on exhausted 5xx retries (a genuinely-down provider still surfaces)', async () => {
|
|
348
|
+
let calls = 0;
|
|
349
|
+
const fetchImpl = async () => {
|
|
350
|
+
calls++;
|
|
351
|
+
return { ok: false, status: 500, headers: { get: () => null }, json: async () => ({}) };
|
|
352
|
+
};
|
|
353
|
+
const adapter = gitlabProvenance('token', { timeoutMs: 1000, retries: 2, backoffMs: 1, fetchImpl });
|
|
354
|
+
await assert.rejects(
|
|
355
|
+
adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD }),
|
|
356
|
+
err => {
|
|
357
|
+
assert.match(err.message, /provenance: GitLab API returned 500/);
|
|
358
|
+
return true;
|
|
359
|
+
}
|
|
360
|
+
);
|
|
361
|
+
assert.equal(calls, 3, 'expected 1 initial + 2 retries = 3 requests before throwing');
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
it('does NOT retry a 404 (run genuinely absent → single immediate false)', async () => {
|
|
365
|
+
let calls = 0;
|
|
366
|
+
const fetchImpl = async () => {
|
|
367
|
+
calls++;
|
|
368
|
+
return { ok: false, status: 404, headers: { get: () => null }, json: async () => ({}) };
|
|
369
|
+
};
|
|
370
|
+
const adapter = gitlabProvenance('token', { timeoutMs: 1000, retries: 2, backoffMs: 1, fetchImpl });
|
|
371
|
+
const ok = await adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD });
|
|
372
|
+
assert.equal(ok, false);
|
|
373
|
+
assert.equal(calls, 1, '404 must not be retried');
|
|
374
|
+
});
|
|
375
|
+
});
|
package/validators/provenance.js
CHANGED
|
@@ -31,6 +31,66 @@ export const GITHUB_PROVENANCE_TIMEOUT_MS = 30000;
|
|
|
31
31
|
*/
|
|
32
32
|
export const GITLAB_PROVENANCE_TIMEOUT_MS = 30000;
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Default RETRY budget for a transient provider fault (HTTP 429 rate-limit or a
|
|
36
|
+
* 5xx provider outage). PROACT-VERIFY-001: a single 429/5xx during a momentary
|
|
37
|
+
* blip used to fail the whole submission as an operational incident even though
|
|
38
|
+
* one retry would have confirmed the run. We retry a BOUNDED number of times
|
|
39
|
+
* with exponential backoff (honoring `Retry-After` when the provider sends it),
|
|
40
|
+
* then THROW on exhaustion so a genuinely-down provider still surfaces as an
|
|
41
|
+
* operational signal — never a false 'confirmed'. 404 is NOT retried (the run is
|
|
42
|
+
* genuinely absent, a single immediate rejection).
|
|
43
|
+
*
|
|
44
|
+
* `retries` is the number of ADDITIONAL attempts after the first, so the default
|
|
45
|
+
* 2 means up to 3 total requests. It is an opts field (like `timeoutMs`) so it
|
|
46
|
+
* stays test-injectable and the offline stub path is unaffected.
|
|
47
|
+
*/
|
|
48
|
+
export const PROVENANCE_RETRIES = 2;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Base backoff between retry attempts, in ms. Attempt N waits
|
|
52
|
+
* `PROVENANCE_BACKOFF_MS * 2^(N-1)` (250 → 500 → …), unless the provider sent a
|
|
53
|
+
* `Retry-After` header, which takes precedence. Injectable via opts so tests run
|
|
54
|
+
* without real delay.
|
|
55
|
+
*/
|
|
56
|
+
export const PROVENANCE_BACKOFF_MS = 250;
|
|
57
|
+
|
|
58
|
+
/** HTTP statuses worth retrying: 429 rate-limit + the 5xx provider-outage band. */
|
|
59
|
+
function isRetryableStatus(status) {
|
|
60
|
+
return status === 429 || (status >= 500 && status <= 599);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Resolve the wait before the next attempt. A `Retry-After` header (delay in
|
|
65
|
+
* seconds, or an HTTP-date) is honored when present and parseable; otherwise
|
|
66
|
+
* fall back to exponential backoff. `attempt` is 1-based (1 = wait before the
|
|
67
|
+
* 2nd request).
|
|
68
|
+
*
|
|
69
|
+
* @param {Response} resp - The non-ok response carrying a possible Retry-After.
|
|
70
|
+
* @param {number} attempt - 1-based retry index.
|
|
71
|
+
* @param {number} backoffMs - Base backoff.
|
|
72
|
+
* @returns {number} Milliseconds to wait (never negative).
|
|
73
|
+
*/
|
|
74
|
+
function nextBackoffMs(resp, attempt, backoffMs) {
|
|
75
|
+
const header = resp?.headers?.get?.('retry-after');
|
|
76
|
+
if (header != null && header !== '') {
|
|
77
|
+
const asSeconds = Number(header);
|
|
78
|
+
if (Number.isFinite(asSeconds) && asSeconds >= 0) {
|
|
79
|
+
return Math.round(asSeconds * 1000);
|
|
80
|
+
}
|
|
81
|
+
const asDate = Date.parse(header);
|
|
82
|
+
if (!Number.isNaN(asDate)) {
|
|
83
|
+
return Math.max(0, asDate - Date.now());
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return backoffMs * 2 ** (attempt - 1);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Default sleep: a real timer. Injectable via `opts.sleepImpl` for tests. */
|
|
90
|
+
function defaultSleep(ms) {
|
|
91
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
92
|
+
}
|
|
93
|
+
|
|
34
94
|
/**
|
|
35
95
|
* Stub provenance adapter. Always confirms.
|
|
36
96
|
* Use in tests and local development.
|
|
@@ -62,6 +122,9 @@ export const rejectingProvenance = {
|
|
|
62
122
|
export function githubProvenance(token, opts = {}) {
|
|
63
123
|
const timeoutMs = opts.timeoutMs ?? GITHUB_PROVENANCE_TIMEOUT_MS;
|
|
64
124
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
125
|
+
const retries = opts.retries ?? PROVENANCE_RETRIES;
|
|
126
|
+
const backoffMs = opts.backoffMs ?? PROVENANCE_BACKOFF_MS;
|
|
127
|
+
const sleep = opts.sleepImpl ?? defaultSleep;
|
|
65
128
|
return {
|
|
66
129
|
/**
|
|
67
130
|
* @param {object} source - submission.source (provider-scoped run claim)
|
|
@@ -96,54 +159,57 @@ export function githubProvenance(token, opts = {}) {
|
|
|
96
159
|
|
|
97
160
|
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/actions/runs/${provider_run_id}`;
|
|
98
161
|
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
// AbortController
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
162
|
+
// PROACT-VERIFY-001: bounded retry over transient provider faults (429
|
|
163
|
+
// rate-limit, 5xx outage). Each attempt carries its own per-request
|
|
164
|
+
// AbortController timeout — without it a hung GitHub API call blocks ingest
|
|
165
|
+
// indefinitely. We retry up to `retries` extra times with exponential
|
|
166
|
+
// backoff (honoring Retry-After), then THROW on exhaustion so a
|
|
167
|
+
// genuinely-down provider surfaces as an operational signal — never a
|
|
168
|
+
// false 'confirmed'. 404 is NOT retried (run genuinely absent). The
|
|
169
|
+
// earlier non-retry behavior (single 429/5xx → immediate throw) failed a
|
|
170
|
+
// submission on a momentary blip that one retry would have confirmed.
|
|
107
171
|
let run;
|
|
108
|
-
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
// (
|
|
127
|
-
|
|
128
|
-
|
|
172
|
+
for (let attempt = 0; ; attempt++) {
|
|
173
|
+
const controller = new AbortController();
|
|
174
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
175
|
+
|
|
176
|
+
let resp;
|
|
177
|
+
try {
|
|
178
|
+
resp = await fetchImpl(apiUrl, {
|
|
179
|
+
headers: {
|
|
180
|
+
Authorization: `Bearer ${token}`,
|
|
181
|
+
Accept: 'application/vnd.github+json',
|
|
182
|
+
'X-GitHub-Api-Version': '2022-11-28'
|
|
183
|
+
},
|
|
184
|
+
signal: controller.signal
|
|
185
|
+
});
|
|
186
|
+
} catch (err) {
|
|
187
|
+
if (err && (err.name === 'AbortError' || err.code === 'ABORT_ERR')) {
|
|
188
|
+
throw new Error(`provenance: GitHub API timeout after ${timeoutMs}ms`);
|
|
189
|
+
}
|
|
190
|
+
// Genuine transport error (DNS, connection refused) — no run to
|
|
191
|
+
// confirm. Reserve `return false` for this case (mirrors GitLab).
|
|
192
|
+
return false;
|
|
193
|
+
} finally {
|
|
194
|
+
clearTimeout(timer);
|
|
129
195
|
}
|
|
130
196
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
throw new Error(`provenance: GitHub API timeout after ${timeoutMs}ms`);
|
|
197
|
+
if (resp.ok) {
|
|
198
|
+
run = await resp.json();
|
|
199
|
+
break;
|
|
135
200
|
}
|
|
136
|
-
|
|
137
|
-
//
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
|
|
142
|
-
|
|
201
|
+
|
|
202
|
+
// verify-A-002: only 404 means the run is genuinely absent — a real
|
|
203
|
+
// submission-bad rejection, never retried. 429/5xx are transient
|
|
204
|
+
// OPERATIONAL signals: retry within budget. 401/403 (expired/insufficient
|
|
205
|
+
// token) are NOT transient — throw immediately. On exhausted retries the
|
|
206
|
+
// last 429/5xx throws so a real outage still surfaces (operational).
|
|
207
|
+
if (resp.status === 404) return false;
|
|
208
|
+
if (isRetryableStatus(resp.status) && attempt < retries) {
|
|
209
|
+
await sleep(nextBackoffMs(resp, attempt + 1, backoffMs));
|
|
210
|
+
continue;
|
|
143
211
|
}
|
|
144
|
-
|
|
145
|
-
} finally {
|
|
146
|
-
clearTimeout(timer);
|
|
212
|
+
throw new Error(`provenance: GitHub API returned ${resp.status}`);
|
|
147
213
|
}
|
|
148
214
|
|
|
149
215
|
if (run.id !== Number(provider_run_id)) return false;
|
|
@@ -211,6 +277,9 @@ export function gitlabProvenance(token, opts = {}) {
|
|
|
211
277
|
const timeoutMs = opts.timeoutMs ?? GITLAB_PROVENANCE_TIMEOUT_MS;
|
|
212
278
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
213
279
|
const apiBase = (opts.apiBase ?? 'https://gitlab.com').replace(/\/+$/, '');
|
|
280
|
+
const retries = opts.retries ?? PROVENANCE_RETRIES;
|
|
281
|
+
const backoffMs = opts.backoffMs ?? PROVENANCE_BACKOFF_MS;
|
|
282
|
+
const sleep = opts.sleepImpl ?? defaultSleep;
|
|
214
283
|
return {
|
|
215
284
|
/**
|
|
216
285
|
* @param {object} source - submission.source (provider-scoped run claim)
|
|
@@ -252,46 +321,50 @@ export function gitlabProvenance(token, opts = {}) {
|
|
|
252
321
|
? `${apiBase}/api/v4/projects/${projectId}/jobs/${provider_run_id}`
|
|
253
322
|
: `${apiBase}/api/v4/projects/${projectId}/pipelines/${provider_run_id}`;
|
|
254
323
|
|
|
255
|
-
//
|
|
256
|
-
//
|
|
257
|
-
//
|
|
258
|
-
//
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
324
|
+
// PROACT-VERIFY-001: bounded retry over transient provider faults — same
|
|
325
|
+
// discipline as githubProvenance. Each attempt carries its own per-request
|
|
326
|
+
// AbortController timeout. 429/5xx retry within budget (exponential backoff,
|
|
327
|
+
// honoring Retry-After), then THROW on exhaustion so a real outage still
|
|
328
|
+
// surfaces as operational — never a false 'confirmed'. 404 is a single
|
|
329
|
+
// immediate rejection; 401/403 throw immediately (not transient).
|
|
262
330
|
let run;
|
|
263
|
-
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
if (
|
|
278
|
-
|
|
331
|
+
for (let attempt = 0; ; attempt++) {
|
|
332
|
+
const controller = new AbortController();
|
|
333
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
334
|
+
|
|
335
|
+
let resp;
|
|
336
|
+
try {
|
|
337
|
+
resp = await fetchImpl(endpoint, {
|
|
338
|
+
headers: {
|
|
339
|
+
'PRIVATE-TOKEN': token,
|
|
340
|
+
Accept: 'application/json'
|
|
341
|
+
},
|
|
342
|
+
signal: controller.signal
|
|
343
|
+
});
|
|
344
|
+
} catch (err) {
|
|
345
|
+
if (err && (err.name === 'AbortError' || err.code === 'ABORT_ERR')) {
|
|
346
|
+
throw new Error(`provenance: GitLab API timeout after ${timeoutMs}ms`);
|
|
347
|
+
}
|
|
348
|
+
// Genuine transport error (DNS, connection refused) — no run to confirm.
|
|
349
|
+
return false;
|
|
350
|
+
} finally {
|
|
351
|
+
clearTimeout(timer);
|
|
279
352
|
}
|
|
280
353
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
throw new Error(`provenance: GitLab API timeout after ${timeoutMs}ms`);
|
|
354
|
+
if (resp.ok) {
|
|
355
|
+
run = await resp.json();
|
|
356
|
+
break;
|
|
285
357
|
}
|
|
286
|
-
|
|
287
|
-
//
|
|
288
|
-
//
|
|
289
|
-
|
|
290
|
-
|
|
358
|
+
|
|
359
|
+
// Mirror verify-A-002: 404 = pipeline/job genuinely absent (submission-bad,
|
|
360
|
+
// never retried). 429/5xx = transient operational fault, retry within
|
|
361
|
+
// budget. 401/403 = non-transient operational, throw immediately.
|
|
362
|
+
if (resp.status === 404) return false;
|
|
363
|
+
if (isRetryableStatus(resp.status) && attempt < retries) {
|
|
364
|
+
await sleep(nextBackoffMs(resp, attempt + 1, backoffMs));
|
|
365
|
+
continue;
|
|
291
366
|
}
|
|
292
|
-
|
|
293
|
-
} finally {
|
|
294
|
-
clearTimeout(timer);
|
|
367
|
+
throw new Error(`provenance: GitLab API returned ${resp.status}`);
|
|
295
368
|
}
|
|
296
369
|
|
|
297
370
|
// id in the response must match the claimed run id.
|