@dogfood-lab/verify 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/cli.js +4 -1
- package/index.js +17 -3
- package/package.json +1 -1
- package/parse-rejection.js +11 -1
- package/validators/policy.js +83 -3
- package/validators/provenance-gitlab.test.js +75 -0
- package/validators/provenance.js +151 -78
package/README.md
CHANGED
|
@@ -83,7 +83,7 @@ Provenance fields (`github_run_id`, `github_workflow_ref`) are required when `pr
|
|
|
83
83
|
|
|
84
84
|
### Prefix taxonomy
|
|
85
85
|
|
|
86
|
-
The verifier emits
|
|
86
|
+
The verifier emits rejection-reason strings under stable prefixes, each mapping to one of four routing classes:
|
|
87
87
|
|
|
88
88
|
Discrimination happens by **class**, surfaced by `parseRejectionReason` (below). Every prefix maps to one of four classes: **submission-bad** (the submitter fixes the payload), **operational** (the verifier/tooling faulted), **ingest** (an ingest-side load fault), or **unknown** (unrecognized prefix).
|
|
89
89
|
|
|
@@ -94,7 +94,7 @@ Discrimination happens by **class**, surfaced by `parseRejectionReason` (below).
|
|
|
94
94
|
| `schema:` | `validators/schema.js` | JSON Schema check on the submission/record envelope failed. The rest of the string carries the AJV path + message. |
|
|
95
95
|
| `policy:` | `validators/policy.js` | Per-repo policy gate failed (forbidden tags, missing required fields, surface evidence/CI requirements, etc.). |
|
|
96
96
|
| `steps[<id>]:` | `validators/steps.js` | Step-level contract check failed on a specific step id (gate accumulation, ordering, evidence shape). |
|
|
97
|
-
| `provenance:` | `validators/provenance.js` | The
|
|
97
|
+
| `provenance:` | `validators/provenance.js` | The run was genuinely **absent / not confirmable** — a 404 from the provider API, or the run head did not match the submitted commit/repo. The submitter's payload points at a run that does not exist or does not bind. (Operational provider faults — 429/5xx/401/403 — are NOT this class; see `provenance-fault:` below.) |
|
|
98
98
|
| `repo:` | `index.js` cross-field guard | `submission.repo` does not match the owner/repo encoded in `source.run_url` (anti-forgery guard). Emitted as `repo:mismatch: …`. |
|
|
99
99
|
| `submission-contains-verifier-field:` | `index.js` | The submission carried a verifier-owned field (`policy_version`, `verification`, or an object `overall_verdict`) it must not author. |
|
|
100
100
|
| `CONTRACT_SCHEMA_TOO_NEW:` | `validators/schema-version.js` | The submission's `schema_version` declares a MAJOR **above** what this build supports (see `SUPPORTED_SCHEMA_VERSIONS` in `@dogfood-lab/schemas`). This build cannot understand a future contract — **the operator must upgrade testing-os**, but the routing class stays submission-bad (the payload as-shipped cannot be accepted by THIS build). |
|
|
@@ -109,6 +109,7 @@ Discrimination happens by **class**, surfaced by `parseRejectionReason` (below).
|
|
|
109
109
|
| `VALIDATOR_FAULT_STEPS:` | `runValidator('steps', …)` catch | Internal exception inside the steps validator. |
|
|
110
110
|
| `VALIDATOR_FAULT_CONTRACT_SCHEMA_VERSION:` | `runValidator('contract_schema_version', …)` catch | The version gate was called with an unknown contract key (a programmer error at the call site, not a submission fault). |
|
|
111
111
|
| `submission-malformed:` | `index.js` null/non-object early-return | The submission itself was `null` or not an object — a malfunctioning **dispatcher** sent garbage, not a submitter who authored a bad-but-shaped payload. Page ops / inspect the dispatch pipeline; do NOT bounce it to a submitter. |
|
|
112
|
+
| `provenance-fault:` | `index.js` provenance catch | The provenance adapter THREW an operational error confirming the run — a provider **429 rate-limit, 5xx outage, or 401/403 token** fault (`validators/provenance.js` throws these on purpose for non-404 responses). The submitter's payload is fine; the verifier could not reach a verdict. Page ops / retry; do NOT bounce it to a submitter. Distinct from the submission-bad `provenance:` (genuine absence/404). |
|
|
112
113
|
|
|
113
114
|
Any future `VALIDATOR_FAULT_<NEW>:` prefix is classified `operational` by family — `parseRejectionReason` matches the `VALIDATOR_FAULT_` head, so a new validator class needs no parser edit. The `submission-malformed:` prefix is matched literally (it is not part of the `VALIDATOR_FAULT_` family).
|
|
114
115
|
|
|
@@ -153,6 +154,10 @@ for (const r of result.rejection_reasons) {
|
|
|
153
154
|
|
|
154
155
|
Persistence note: every entry above is round-tripped verbatim through `verification.rejection_reasons` in the persisted-record JSON; the schema enforces `array of string` so any consumer of the audit-DB ground truth sees the same prefix vocabulary.
|
|
155
156
|
|
|
157
|
+
### Warnings channel (accepted-with-warning)
|
|
158
|
+
|
|
159
|
+
Not every policy signal is a rejection. A policy rule declared `severity: warn` produces a `policy: <id>: <message>` entry on `verification.warnings` (an optional `array of string` on the persisted record) **without** flipping the verdict to `rejected` — the submission is accepted and recorded, the warning rides alongside it. (`severity: info` rules are logged only and never persisted; `severity: reject` rules go to `rejection_reasons` as above.) Consumers that want advisory signals read `verification.warnings`; the routing decision (`parseRejectionReason`) only concerns `rejection_reasons`. A clean accepted submission carries no `warnings` key at all.
|
|
160
|
+
|
|
156
161
|
## Docs
|
|
157
162
|
|
|
158
163
|
📖 Full handbook: **<https://dogfood-lab.github.io/testing-os/handbook/>**
|
package/cli.js
CHANGED
|
@@ -316,7 +316,10 @@ export function renderExplain(record) {
|
|
|
316
316
|
const accepted = v.status === 'accepted';
|
|
317
317
|
const lines = [];
|
|
318
318
|
|
|
319
|
-
|
|
319
|
+
// Uppercase the rendered state word to match the sibling verdict banners
|
|
320
|
+
// (findings-render.js). This is display-only; the underlying enum
|
|
321
|
+
// (v.status, --json output) stays lowercase.
|
|
322
|
+
lines.push(`VERDICT: ${(v.status || (accepted ? 'accepted' : 'rejected')).toUpperCase()}`);
|
|
320
323
|
lines.push('');
|
|
321
324
|
lines.push(` schema_valid: ${v.schema_valid}`);
|
|
322
325
|
lines.push(` policy_valid: ${v.policy_valid}`);
|
package/index.js
CHANGED
|
@@ -209,9 +209,15 @@ export async function verify(submission, options) {
|
|
|
209
209
|
refCommitSha: submission.ref?.commit_sha
|
|
210
210
|
});
|
|
211
211
|
} catch (err) {
|
|
212
|
-
|
|
212
|
+
// verify-A-002: the adapter THROWS on operational provider faults
|
|
213
|
+
// (429 rate-limit, 5xx outage, 401/403 token) and returns false only for
|
|
214
|
+
// a genuinely-absent run (404/transport). Emit a DISTINCT `provenance-fault:`
|
|
215
|
+
// prefix here so parseRejectionReason routes the incident to ops instead of
|
|
216
|
+
// bouncing an outage back to the submitter as submission-bad. The not-confirmed
|
|
217
|
+
// case below keeps the bare `provenance:` prefix (still submission-bad).
|
|
218
|
+
reasons.push(`provenance-fault: verification failed: ${err.message}`);
|
|
213
219
|
}
|
|
214
|
-
if (!provenanceConfirmed && !reasons.some(r => r.startsWith('provenance
|
|
220
|
+
if (!provenanceConfirmed && !reasons.some(r => r.startsWith('provenance'))) {
|
|
215
221
|
reasons.push('provenance: source run could not be confirmed');
|
|
216
222
|
}
|
|
217
223
|
}
|
|
@@ -243,6 +249,12 @@ export async function verify(submission, options) {
|
|
|
243
249
|
// BEFORE handing the (broken) policy to `validatePolicy`, so the
|
|
244
250
|
// operator sees a real "policy:" rejection in `rejection_reasons`.
|
|
245
251
|
let policyValid = false;
|
|
252
|
+
// VERIFY-F4: severity:warn / severity:info policy rules surface here as an
|
|
253
|
+
// accepted-with-warning channel. Warnings NEVER enter `reasons` (which become
|
|
254
|
+
// rejection_reasons and flip status to 'rejected') — they land on
|
|
255
|
+
// verification.warnings at assembly so an operator sees the advisory without
|
|
256
|
+
// the submission being bounced.
|
|
257
|
+
const warnings = [];
|
|
246
258
|
if (schemaResult.valid) {
|
|
247
259
|
if (repoPolicy && repoPolicy.__torn === true) {
|
|
248
260
|
const detail = repoPolicy.reason || 'repo policy YAML failed to parse';
|
|
@@ -253,6 +265,7 @@ export async function verify(submission, options) {
|
|
|
253
265
|
if (policyRun.ok) {
|
|
254
266
|
policyValid = policyRun.result.valid;
|
|
255
267
|
reasons.push(...policyRun.result.errors.map(e => `policy: ${e}`));
|
|
268
|
+
warnings.push(...(policyRun.result.warnings || []).map(w => `policy: ${w}`));
|
|
256
269
|
} else {
|
|
257
270
|
reasons.push(policyRun.faultReason);
|
|
258
271
|
}
|
|
@@ -300,7 +313,8 @@ export async function verify(submission, options) {
|
|
|
300
313
|
provenance_confirmed: provenanceConfirmed,
|
|
301
314
|
schema_valid: schemaResult.valid,
|
|
302
315
|
policy_valid: policyValid,
|
|
303
|
-
rejection_reasons: reasons
|
|
316
|
+
rejection_reasons: reasons,
|
|
317
|
+
...(warnings.length ? { warnings } : {})
|
|
304
318
|
},
|
|
305
319
|
...(submission.notes ? { notes: submission.notes } : {})
|
|
306
320
|
};
|
package/package.json
CHANGED
package/parse-rejection.js
CHANGED
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
* The prefix vocabulary below is enumerated from the ACTUAL emitters — it is
|
|
25
25
|
* NOT invented:
|
|
26
26
|
* - verify/index.js: schema:, policy:, steps[<id>]:,
|
|
27
|
-
* provenance
|
|
27
|
+
* provenance: (run absent → submission-bad),
|
|
28
|
+
* provenance-fault: (provider 429/5xx/401/403
|
|
29
|
+
* → operational), repo:,
|
|
28
30
|
* submission-contains-verifier-field:,
|
|
29
31
|
* submission-malformed:,
|
|
30
32
|
* VALIDATOR_FAULT_<NAME>:
|
|
@@ -61,6 +63,14 @@ const LITERAL_PREFIXES = [
|
|
|
61
63
|
// submission-bad
|
|
62
64
|
{ match: 'schema:', prefix: 'schema:', class: 'submission-bad' },
|
|
63
65
|
{ match: 'policy:', prefix: 'policy:', class: 'submission-bad' },
|
|
66
|
+
// operational — a provider-side provenance FAULT (429/5xx/401/403). The
|
|
67
|
+
// adapter THROWS these (verify-A-002); index.js catches the throw and emits
|
|
68
|
+
// this distinct `provenance-fault:` prefix so the incident pages ops. Ordered
|
|
69
|
+
// before the bare `provenance:` literal — it is a distinct token, but keeping
|
|
70
|
+
// the longer, more-specific match first preserves the most-specific-first
|
|
71
|
+
// invariant the array is sorted by. The genuine not-confirmed case
|
|
72
|
+
// (`provenance: source run could not be confirmed`) stays submission-bad below.
|
|
73
|
+
{ match: 'provenance-fault:', prefix: 'provenance-fault:', class: 'operational' },
|
|
64
74
|
{ match: 'provenance:', prefix: 'provenance:', class: 'submission-bad' },
|
|
65
75
|
{ match: 'repo:', prefix: 'repo:', class: 'submission-bad' },
|
|
66
76
|
{
|
package/validators/policy.js
CHANGED
|
@@ -29,6 +29,31 @@ function deepMerge(target, source) {
|
|
|
29
29
|
return result;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
/**
|
|
33
|
+
* Global reject-rule ids the build can account for. Two groups:
|
|
34
|
+
* - HANDLED HERE — enforced by the switch below.
|
|
35
|
+
* - ENFORCED ELSEWHERE — owned by another validator or by verify() itself
|
|
36
|
+
* (the switch's `default` arm intentionally no-ops them).
|
|
37
|
+
*
|
|
38
|
+
* PROACT-VERIFY-002: any `severity: reject` rule whose id is OUTSIDE this set is
|
|
39
|
+
* an operator-added gate the build does not enforce. Silently no-op'ing it (the
|
|
40
|
+
* old `default: break`) meant the rule looked active in global-policy.yaml but
|
|
41
|
+
* never ran. We now surface an actionable diagnostic instead. Keep this set in
|
|
42
|
+
* sync when a new reject rule gains real enforcement.
|
|
43
|
+
*/
|
|
44
|
+
const KNOWN_REJECT_RULE_IDS = new Set([
|
|
45
|
+
// handled by the switch in validatePolicy
|
|
46
|
+
'scenario-minimum',
|
|
47
|
+
'attested-if-human',
|
|
48
|
+
'blocked-needs-reason',
|
|
49
|
+
// enforced by other validators or by verify() itself (default-arm no-op is correct)
|
|
50
|
+
'schema-valid',
|
|
51
|
+
'provenance-confirmed',
|
|
52
|
+
'step-results-present',
|
|
53
|
+
'step-verdict-consistent',
|
|
54
|
+
'no-verdict-upgrade',
|
|
55
|
+
]);
|
|
56
|
+
|
|
32
57
|
/**
|
|
33
58
|
* Resolve the effective surface policy for a given product surface.
|
|
34
59
|
* Repo policy overrides global defaults per surface.
|
|
@@ -55,16 +80,32 @@ function resolveSurfacePolicy(surface, globalPolicy, repoPolicy) {
|
|
|
55
80
|
* @param {object} options
|
|
56
81
|
* @param {object} options.globalPolicy
|
|
57
82
|
* @param {object|null} options.repoPolicy
|
|
58
|
-
* @returns {{ valid: boolean, errors: string[] }}
|
|
83
|
+
* @returns {{ valid: boolean, errors: string[], warnings: string[] }}
|
|
59
84
|
*/
|
|
60
85
|
export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
61
86
|
const errors = [];
|
|
87
|
+
// VERIFY-F4: warn-severity rules record an accepted-with-warning note here
|
|
88
|
+
// instead of being silently dropped. A populated warnings[] never affects
|
|
89
|
+
// `valid` — the caller (index.js) routes it to verification.warnings.
|
|
90
|
+
const warnings = [];
|
|
62
91
|
|
|
63
92
|
// --- Global rules (non-overridable) ---
|
|
64
93
|
|
|
65
94
|
const globalRules = globalPolicy.global_rules || [];
|
|
66
95
|
|
|
67
96
|
for (const rule of globalRules) {
|
|
97
|
+
// VERIFY-F4: a global warn-rule is accepted-with-warning; an info-rule logs
|
|
98
|
+
// only. Neither may reject. Mirrors the PROACT-VERIFY-002 discipline of not
|
|
99
|
+
// dropping operator-authored rules on the floor: a declared rule that the
|
|
100
|
+
// build does nothing with is invisible to the operator who wrote it.
|
|
101
|
+
if (rule.severity === 'warn') {
|
|
102
|
+
warnings.push(`${rule.id}: ${rule.description || 'policy warning'}`);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (rule.severity === 'info') {
|
|
106
|
+
// Logged only — info rules are intentionally non-surfacing in the record.
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
68
109
|
if (rule.severity !== 'reject') continue;
|
|
69
110
|
|
|
70
111
|
switch (rule.id) {
|
|
@@ -95,8 +136,20 @@ export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
|
95
136
|
break;
|
|
96
137
|
|
|
97
138
|
// schema-valid, provenance-confirmed, step-results-present, step-verdict-consistent,
|
|
98
|
-
// no-verdict-upgrade are enforced by other validators or the main verify() function
|
|
139
|
+
// no-verdict-upgrade are enforced by other validators or the main verify() function.
|
|
140
|
+
// PROACT-VERIFY-002: a reject rule the build neither handles here NOR enforces
|
|
141
|
+
// elsewhere would otherwise pass SILENTLY — the operator's new gate never runs.
|
|
142
|
+
// Reject the submission with a diagnostic naming the unenforced rule so the gap
|
|
143
|
+
// is visible instead of failing open.
|
|
99
144
|
default:
|
|
145
|
+
if (!KNOWN_REJECT_RULE_IDS.has(rule.id)) {
|
|
146
|
+
// No `policy:` prefix here — index.js prepends it to every policy
|
|
147
|
+
// error (mirrors the `[rule.id]` / `surface[...]` messages above).
|
|
148
|
+
errors.push(
|
|
149
|
+
`rule "${rule.id}" is declared severity:reject but has no enforcement in this build — ` +
|
|
150
|
+
`add an enforcement arm in validators/policy.js or register it in KNOWN_REJECT_RULE_IDS`
|
|
151
|
+
);
|
|
152
|
+
}
|
|
100
153
|
break;
|
|
101
154
|
}
|
|
102
155
|
}
|
|
@@ -134,6 +187,33 @@ export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
|
134
187
|
}
|
|
135
188
|
}
|
|
136
189
|
}
|
|
190
|
+
|
|
191
|
+
// VERIFY-F2: tag gating. forbidden_tags rejects a scenario_result carrying
|
|
192
|
+
// any listed tag; required_tags rejects one missing any listed tag (a
|
|
193
|
+
// tagless scenario fails every required_tags rule). Tags are optional on
|
|
194
|
+
// the scenario_result, so an absent `tags` array trips required_tags but
|
|
195
|
+
// never forbidden_tags.
|
|
196
|
+
const tags = new Set(sr.tags || []);
|
|
197
|
+
|
|
198
|
+
if (evidenceReqs.forbidden_tags) {
|
|
199
|
+
for (const tag of evidenceReqs.forbidden_tags) {
|
|
200
|
+
if (tags.has(tag)) {
|
|
201
|
+
errors.push(
|
|
202
|
+
`surface[${surface}]: scenario "${sr.scenario_id}" carries forbidden tag "${tag}"`
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
if (evidenceReqs.required_tags) {
|
|
209
|
+
for (const tag of evidenceReqs.required_tags) {
|
|
210
|
+
if (!tags.has(tag)) {
|
|
211
|
+
errors.push(
|
|
212
|
+
`surface[${surface}]: scenario "${sr.scenario_id}" is missing required tag "${tag}"`
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
137
217
|
}
|
|
138
218
|
}
|
|
139
219
|
|
|
@@ -176,5 +256,5 @@ export function validatePolicy(submission, { globalPolicy, repoPolicy }) {
|
|
|
176
256
|
}
|
|
177
257
|
}
|
|
178
258
|
|
|
179
|
-
return { valid: errors.length === 0, errors };
|
|
259
|
+
return { valid: errors.length === 0, errors, warnings };
|
|
180
260
|
}
|
|
@@ -212,6 +212,7 @@ describe('gitlabProvenance distinguishes ops failures from missing runs', () =>
|
|
|
212
212
|
it(`throws an operational error on HTTP ${status} (not a submission-bad false)`, async () => {
|
|
213
213
|
const adapter = gitlabProvenance('token', {
|
|
214
214
|
timeoutMs: 1000,
|
|
215
|
+
retries: 0,
|
|
215
216
|
fetchImpl: fetchWithStatus(status)
|
|
216
217
|
});
|
|
217
218
|
await assert.rejects(
|
|
@@ -298,3 +299,77 @@ describe('gitlabProvenance input guards', () => {
|
|
|
298
299
|
assert.equal(ok, false);
|
|
299
300
|
});
|
|
300
301
|
});
|
|
302
|
+
|
|
303
|
+
// ── Bounded retry over transient faults (PROACT-VERIFY-001, mirrors GitHub) ──
|
|
304
|
+
//
|
|
305
|
+
// MEDIUM/resilience: a single 429/5xx blip used to fail the submission as an
|
|
306
|
+
// operational incident. The adapter now retries 429/5xx within a bounded budget
|
|
307
|
+
// (opts.retries, default 2) with exponential backoff (honoring Retry-After), and
|
|
308
|
+
// still THROWS on exhaustion so a genuinely-down provider surfaces as
|
|
309
|
+
// operational — never a false 'confirmed'. 404 is NOT retried.
|
|
310
|
+
|
|
311
|
+
describe('gitlabProvenance bounded retry (PROACT-VERIFY-001)', () => {
|
|
312
|
+
it('retries a 429 then confirms on the following 200 (retry worked)', async () => {
|
|
313
|
+
let calls = 0;
|
|
314
|
+
const fetchImpl = async () => {
|
|
315
|
+
calls++;
|
|
316
|
+
if (calls === 1) {
|
|
317
|
+
return { ok: false, status: 429, headers: { get: () => null }, json: async () => ({}) };
|
|
318
|
+
}
|
|
319
|
+
return { ok: true, status: 200, json: async () => mockPipeline() };
|
|
320
|
+
};
|
|
321
|
+
const adapter = gitlabProvenance('token', { timeoutMs: 1000, backoffMs: 1, fetchImpl });
|
|
322
|
+
const ok = await adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD });
|
|
323
|
+
assert.equal(ok, true, '429-then-200 must confirm — the retry succeeded');
|
|
324
|
+
assert.equal(calls, 2, 'expected exactly one retry (2 total requests)');
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
it('honors Retry-After (numeric seconds) before the retry', async () => {
|
|
328
|
+
let calls = 0;
|
|
329
|
+
const waited = [];
|
|
330
|
+
const fetchImpl = async () => {
|
|
331
|
+
calls++;
|
|
332
|
+
if (calls === 1) {
|
|
333
|
+
return { ok: false, status: 503, headers: { get: h => (h === 'retry-after' ? '2' : null) }, json: async () => ({}) };
|
|
334
|
+
}
|
|
335
|
+
return { ok: true, status: 200, json: async () => mockPipeline() };
|
|
336
|
+
};
|
|
337
|
+
const adapter = gitlabProvenance('token', {
|
|
338
|
+
timeoutMs: 1000,
|
|
339
|
+
sleepImpl: ms => { waited.push(ms); return Promise.resolve(); },
|
|
340
|
+
fetchImpl
|
|
341
|
+
});
|
|
342
|
+
const ok = await adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD });
|
|
343
|
+
assert.equal(ok, true);
|
|
344
|
+
assert.deepEqual(waited, [2000], 'Retry-After: 2 must drive a 2000ms wait');
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
it('THROWS on exhausted 5xx retries (a genuinely-down provider still surfaces)', async () => {
|
|
348
|
+
let calls = 0;
|
|
349
|
+
const fetchImpl = async () => {
|
|
350
|
+
calls++;
|
|
351
|
+
return { ok: false, status: 500, headers: { get: () => null }, json: async () => ({}) };
|
|
352
|
+
};
|
|
353
|
+
const adapter = gitlabProvenance('token', { timeoutMs: 1000, retries: 2, backoffMs: 1, fetchImpl });
|
|
354
|
+
await assert.rejects(
|
|
355
|
+
adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD }),
|
|
356
|
+
err => {
|
|
357
|
+
assert.match(err.message, /provenance: GitLab API returned 500/);
|
|
358
|
+
return true;
|
|
359
|
+
}
|
|
360
|
+
);
|
|
361
|
+
assert.equal(calls, 3, 'expected 1 initial + 2 retries = 3 requests before throwing');
|
|
362
|
+
});
|
|
363
|
+
|
|
364
|
+
it('does NOT retry a 404 (run genuinely absent → single immediate false)', async () => {
|
|
365
|
+
let calls = 0;
|
|
366
|
+
const fetchImpl = async () => {
|
|
367
|
+
calls++;
|
|
368
|
+
return { ok: false, status: 404, headers: { get: () => null }, json: async () => ({}) };
|
|
369
|
+
};
|
|
370
|
+
const adapter = gitlabProvenance('token', { timeoutMs: 1000, retries: 2, backoffMs: 1, fetchImpl });
|
|
371
|
+
const ok = await adapter.confirm(pipelineSource(), { refCommitSha: RUN_HEAD });
|
|
372
|
+
assert.equal(ok, false);
|
|
373
|
+
assert.equal(calls, 1, '404 must not be retried');
|
|
374
|
+
});
|
|
375
|
+
});
|
package/validators/provenance.js
CHANGED
|
@@ -31,6 +31,66 @@ export const GITHUB_PROVENANCE_TIMEOUT_MS = 30000;
|
|
|
31
31
|
*/
|
|
32
32
|
export const GITLAB_PROVENANCE_TIMEOUT_MS = 30000;
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Default RETRY budget for a transient provider fault (HTTP 429 rate-limit or a
|
|
36
|
+
* 5xx provider outage). PROACT-VERIFY-001: a single 429/5xx during a momentary
|
|
37
|
+
* blip used to fail the whole submission as an operational incident even though
|
|
38
|
+
* one retry would have confirmed the run. We retry a BOUNDED number of times
|
|
39
|
+
* with exponential backoff (honoring `Retry-After` when the provider sends it),
|
|
40
|
+
* then THROW on exhaustion so a genuinely-down provider still surfaces as an
|
|
41
|
+
* operational signal — never a false 'confirmed'. 404 is NOT retried (the run is
|
|
42
|
+
* genuinely absent, a single immediate rejection).
|
|
43
|
+
*
|
|
44
|
+
* `retries` is the number of ADDITIONAL attempts after the first, so the default
|
|
45
|
+
* 2 means up to 3 total requests. It is an opts field (like `timeoutMs`) so it
|
|
46
|
+
* stays test-injectable and the offline stub path is unaffected.
|
|
47
|
+
*/
|
|
48
|
+
export const PROVENANCE_RETRIES = 2;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Base backoff between retry attempts, in ms. Attempt N waits
|
|
52
|
+
* `PROVENANCE_BACKOFF_MS * 2^(N-1)` (250 → 500 → …), unless the provider sent a
|
|
53
|
+
* `Retry-After` header, which takes precedence. Injectable via opts so tests run
|
|
54
|
+
* without real delay.
|
|
55
|
+
*/
|
|
56
|
+
export const PROVENANCE_BACKOFF_MS = 250;
|
|
57
|
+
|
|
58
|
+
/** HTTP statuses worth retrying: 429 rate-limit + the 5xx provider-outage band. */
|
|
59
|
+
function isRetryableStatus(status) {
|
|
60
|
+
return status === 429 || (status >= 500 && status <= 599);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Resolve the wait before the next attempt. A `Retry-After` header (delay in
|
|
65
|
+
* seconds, or an HTTP-date) is honored when present and parseable; otherwise
|
|
66
|
+
* fall back to exponential backoff. `attempt` is 1-based (1 = wait before the
|
|
67
|
+
* 2nd request).
|
|
68
|
+
*
|
|
69
|
+
* @param {Response} resp - The non-ok response carrying a possible Retry-After.
|
|
70
|
+
* @param {number} attempt - 1-based retry index.
|
|
71
|
+
* @param {number} backoffMs - Base backoff.
|
|
72
|
+
* @returns {number} Milliseconds to wait (never negative).
|
|
73
|
+
*/
|
|
74
|
+
function nextBackoffMs(resp, attempt, backoffMs) {
|
|
75
|
+
const header = resp?.headers?.get?.('retry-after');
|
|
76
|
+
if (header != null && header !== '') {
|
|
77
|
+
const asSeconds = Number(header);
|
|
78
|
+
if (Number.isFinite(asSeconds) && asSeconds >= 0) {
|
|
79
|
+
return Math.round(asSeconds * 1000);
|
|
80
|
+
}
|
|
81
|
+
const asDate = Date.parse(header);
|
|
82
|
+
if (!Number.isNaN(asDate)) {
|
|
83
|
+
return Math.max(0, asDate - Date.now());
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return backoffMs * 2 ** (attempt - 1);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Default sleep: a real timer. Injectable via `opts.sleepImpl` for tests. */
|
|
90
|
+
function defaultSleep(ms) {
|
|
91
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
92
|
+
}
|
|
93
|
+
|
|
34
94
|
/**
|
|
35
95
|
* Stub provenance adapter. Always confirms.
|
|
36
96
|
* Use in tests and local development.
|
|
@@ -62,6 +122,9 @@ export const rejectingProvenance = {
|
|
|
62
122
|
export function githubProvenance(token, opts = {}) {
|
|
63
123
|
const timeoutMs = opts.timeoutMs ?? GITHUB_PROVENANCE_TIMEOUT_MS;
|
|
64
124
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
125
|
+
const retries = opts.retries ?? PROVENANCE_RETRIES;
|
|
126
|
+
const backoffMs = opts.backoffMs ?? PROVENANCE_BACKOFF_MS;
|
|
127
|
+
const sleep = opts.sleepImpl ?? defaultSleep;
|
|
65
128
|
return {
|
|
66
129
|
/**
|
|
67
130
|
* @param {object} source - submission.source (provider-scoped run claim)
|
|
@@ -96,54 +159,57 @@ export function githubProvenance(token, opts = {}) {
|
|
|
96
159
|
|
|
97
160
|
const apiUrl = `https://api.github.com/repos/${owner}/${repo}/actions/runs/${provider_run_id}`;
|
|
98
161
|
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
// AbortController
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
162
|
+
// PROACT-VERIFY-001: bounded retry over transient provider faults (429
|
|
163
|
+
// rate-limit, 5xx outage). Each attempt carries its own per-request
|
|
164
|
+
// AbortController timeout — without it a hung GitHub API call blocks ingest
|
|
165
|
+
// indefinitely. We retry up to `retries` extra times with exponential
|
|
166
|
+
// backoff (honoring Retry-After), then THROW on exhaustion so a
|
|
167
|
+
// genuinely-down provider surfaces as an operational signal — never a
|
|
168
|
+
// false 'confirmed'. 404 is NOT retried (run genuinely absent). The
|
|
169
|
+
// earlier non-retry behavior (single 429/5xx → immediate throw) failed a
|
|
170
|
+
// submission on a momentary blip that one retry would have confirmed.
|
|
107
171
|
let run;
|
|
108
|
-
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
// (
|
|
127
|
-
|
|
128
|
-
|
|
172
|
+
for (let attempt = 0; ; attempt++) {
|
|
173
|
+
const controller = new AbortController();
|
|
174
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
175
|
+
|
|
176
|
+
let resp;
|
|
177
|
+
try {
|
|
178
|
+
resp = await fetchImpl(apiUrl, {
|
|
179
|
+
headers: {
|
|
180
|
+
Authorization: `Bearer ${token}`,
|
|
181
|
+
Accept: 'application/vnd.github+json',
|
|
182
|
+
'X-GitHub-Api-Version': '2022-11-28'
|
|
183
|
+
},
|
|
184
|
+
signal: controller.signal
|
|
185
|
+
});
|
|
186
|
+
} catch (err) {
|
|
187
|
+
if (err && (err.name === 'AbortError' || err.code === 'ABORT_ERR')) {
|
|
188
|
+
throw new Error(`provenance: GitHub API timeout after ${timeoutMs}ms`);
|
|
189
|
+
}
|
|
190
|
+
// Genuine transport error (DNS, connection refused) — no run to
|
|
191
|
+
// confirm. Reserve `return false` for this case (mirrors GitLab).
|
|
192
|
+
return false;
|
|
193
|
+
} finally {
|
|
194
|
+
clearTimeout(timer);
|
|
129
195
|
}
|
|
130
196
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
throw new Error(`provenance: GitHub API timeout after ${timeoutMs}ms`);
|
|
197
|
+
if (resp.ok) {
|
|
198
|
+
run = await resp.json();
|
|
199
|
+
break;
|
|
135
200
|
}
|
|
136
|
-
|
|
137
|
-
//
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
//
|
|
141
|
-
|
|
142
|
-
|
|
201
|
+
|
|
202
|
+
// verify-A-002: only 404 means the run is genuinely absent — a real
|
|
203
|
+
// submission-bad rejection, never retried. 429/5xx are transient
|
|
204
|
+
// OPERATIONAL signals: retry within budget. 401/403 (expired/insufficient
|
|
205
|
+
// token) are NOT transient — throw immediately. On exhausted retries the
|
|
206
|
+
// last 429/5xx throws so a real outage still surfaces (operational).
|
|
207
|
+
if (resp.status === 404) return false;
|
|
208
|
+
if (isRetryableStatus(resp.status) && attempt < retries) {
|
|
209
|
+
await sleep(nextBackoffMs(resp, attempt + 1, backoffMs));
|
|
210
|
+
continue;
|
|
143
211
|
}
|
|
144
|
-
|
|
145
|
-
} finally {
|
|
146
|
-
clearTimeout(timer);
|
|
212
|
+
throw new Error(`provenance: GitHub API returned ${resp.status}`);
|
|
147
213
|
}
|
|
148
214
|
|
|
149
215
|
if (run.id !== Number(provider_run_id)) return false;
|
|
@@ -211,6 +277,9 @@ export function gitlabProvenance(token, opts = {}) {
|
|
|
211
277
|
const timeoutMs = opts.timeoutMs ?? GITLAB_PROVENANCE_TIMEOUT_MS;
|
|
212
278
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
213
279
|
const apiBase = (opts.apiBase ?? 'https://gitlab.com').replace(/\/+$/, '');
|
|
280
|
+
const retries = opts.retries ?? PROVENANCE_RETRIES;
|
|
281
|
+
const backoffMs = opts.backoffMs ?? PROVENANCE_BACKOFF_MS;
|
|
282
|
+
const sleep = opts.sleepImpl ?? defaultSleep;
|
|
214
283
|
return {
|
|
215
284
|
/**
|
|
216
285
|
* @param {object} source - submission.source (provider-scoped run claim)
|
|
@@ -252,46 +321,50 @@ export function gitlabProvenance(token, opts = {}) {
|
|
|
252
321
|
? `${apiBase}/api/v4/projects/${projectId}/jobs/${provider_run_id}`
|
|
253
322
|
: `${apiBase}/api/v4/projects/${projectId}/pipelines/${provider_run_id}`;
|
|
254
323
|
|
|
255
|
-
//
|
|
256
|
-
//
|
|
257
|
-
//
|
|
258
|
-
//
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
324
|
+
// PROACT-VERIFY-001: bounded retry over transient provider faults — same
|
|
325
|
+
// discipline as githubProvenance. Each attempt carries its own per-request
|
|
326
|
+
// AbortController timeout. 429/5xx retry within budget (exponential backoff,
|
|
327
|
+
// honoring Retry-After), then THROW on exhaustion so a real outage still
|
|
328
|
+
// surfaces as operational — never a false 'confirmed'. 404 is a single
|
|
329
|
+
// immediate rejection; 401/403 throw immediately (not transient).
|
|
262
330
|
let run;
|
|
263
|
-
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
if (
|
|
278
|
-
|
|
331
|
+
for (let attempt = 0; ; attempt++) {
|
|
332
|
+
const controller = new AbortController();
|
|
333
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
334
|
+
|
|
335
|
+
let resp;
|
|
336
|
+
try {
|
|
337
|
+
resp = await fetchImpl(endpoint, {
|
|
338
|
+
headers: {
|
|
339
|
+
'PRIVATE-TOKEN': token,
|
|
340
|
+
Accept: 'application/json'
|
|
341
|
+
},
|
|
342
|
+
signal: controller.signal
|
|
343
|
+
});
|
|
344
|
+
} catch (err) {
|
|
345
|
+
if (err && (err.name === 'AbortError' || err.code === 'ABORT_ERR')) {
|
|
346
|
+
throw new Error(`provenance: GitLab API timeout after ${timeoutMs}ms`);
|
|
347
|
+
}
|
|
348
|
+
// Genuine transport error (DNS, connection refused) — no run to confirm.
|
|
349
|
+
return false;
|
|
350
|
+
} finally {
|
|
351
|
+
clearTimeout(timer);
|
|
279
352
|
}
|
|
280
353
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
throw new Error(`provenance: GitLab API timeout after ${timeoutMs}ms`);
|
|
354
|
+
if (resp.ok) {
|
|
355
|
+
run = await resp.json();
|
|
356
|
+
break;
|
|
285
357
|
}
|
|
286
|
-
|
|
287
|
-
//
|
|
288
|
-
//
|
|
289
|
-
|
|
290
|
-
|
|
358
|
+
|
|
359
|
+
// Mirror verify-A-002: 404 = pipeline/job genuinely absent (submission-bad,
|
|
360
|
+
// never retried). 429/5xx = transient operational fault, retry within
|
|
361
|
+
// budget. 401/403 = non-transient operational, throw immediately.
|
|
362
|
+
if (resp.status === 404) return false;
|
|
363
|
+
if (isRetryableStatus(resp.status) && attempt < retries) {
|
|
364
|
+
await sleep(nextBackoffMs(resp, attempt + 1, backoffMs));
|
|
365
|
+
continue;
|
|
291
366
|
}
|
|
292
|
-
|
|
293
|
-
} finally {
|
|
294
|
-
clearTimeout(timer);
|
|
367
|
+
throw new Error(`provenance: GitLab API returned ${resp.status}`);
|
|
295
368
|
}
|
|
296
369
|
|
|
297
370
|
// id in the response must match the claimed run id.
|