@dogfood-lab/verify 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -17
- package/cli-lint.js +81 -36
- package/cli.js +99 -15
- package/index.js +205 -61
- package/package.json +1 -1
- package/parse-rejection.js +125 -27
- package/validators/f-2a5ddafa-provenance-retry-warn.test.js +226 -0
- package/validators/f-5fd3f832-retry-after-clamp.test.js +96 -0
- package/validators/f-ad98b5ac-required-steps-map-guard.test.js +85 -0
- package/validators/f-dac7e08c-transport-error-operational.test.js +121 -0
- package/validators/f-f50e779b-retry-warn-fallback-guard.test.js +153 -0
- package/validators/lint-scenario.js +141 -0
- package/validators/policy.js +31 -8
- package/validators/provenance-gitlab.test.js +10 -3
- package/validators/provenance-registry.test.js +35 -0
- package/validators/provenance.js +183 -17
- package/validators/repo-binding.js +35 -12
- package/validators/repo-binding.test.js +40 -0
- package/validators/steps.js +116 -9
- package/validators/verdict.js +9 -1
- package/validators/w4-f-8e72d0de-timeout-retry.test.js +118 -0
package/index.js
CHANGED
|
@@ -8,7 +8,10 @@
|
|
|
8
8
|
|
|
9
9
|
import { validateSubmissionSchema as _defaultValidateSubmissionSchema } from './validators/schema.js';
|
|
10
10
|
import { validatePolicy as _defaultValidatePolicy } from './validators/policy.js';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
validateStepResults as _defaultValidateStepResults,
|
|
13
|
+
validateRequiredSteps as _defaultValidateRequiredSteps
|
|
14
|
+
} from './validators/steps.js';
|
|
12
15
|
import { validateSchemaVersion as _defaultValidateSchemaVersion } from './validators/schema-version.js';
|
|
13
16
|
import { computeVerdict } from './validators/verdict.js';
|
|
14
17
|
import { parseRunUrlRepo } from './validators/repo-binding.js';
|
|
@@ -33,23 +36,32 @@ export { provenanceForProvider, PROVENANCE_ADAPTERS } from './validators/provena
|
|
|
33
36
|
const RECORD_SCHEMA_VERSION = SUPPORTED_SCHEMA_VERSIONS.record.current;
|
|
34
37
|
|
|
35
38
|
/**
|
|
36
|
-
* D1B-003 (Stage C humanization): the SOLE catch
|
|
37
|
-
* validator calls. Distinguishes operator-actionable
|
|
39
|
+
* D1B-003 (Stage C humanization) + F-82429f90 (wave 4): the SOLE catch
|
|
40
|
+
* wrapper for synchronous validator calls. Distinguishes operator-actionable
|
|
41
|
+
* signals:
|
|
38
42
|
*
|
|
39
43
|
* - Validator returned a structured result → caller pushes the existing
|
|
40
44
|
* `'<class>: <details>'` prefix into `rejection_reasons` (unchanged
|
|
41
45
|
* submission-bad path; back-compat is preserved).
|
|
42
46
|
* - Validator THREW (operational incident — ajv compile fault, policy
|
|
43
|
-
* merge cycle, out-of-memory) → wrapper
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
+
* merge cycle, out-of-memory) → wrapper RETHROWS a classified error
|
|
48
|
+
* whose message carries the STABLE coded prefix
|
|
49
|
+
* `'VALIDATOR_FAULT_<NAME>: <details>'` and whose `.code` is
|
|
50
|
+
* `VALIDATOR_FAULT_<NAME>`. The prefix is greppable across runner logs:
|
|
51
|
+
* `grep -E 'VALIDATOR_FAULT_' …` → every operational incident
|
|
47
52
|
* `grep -E '^(schema|policy|steps): ' …` → every submission-bad signal
|
|
48
53
|
*
|
|
54
|
+
* F-82429f90 changed the fault path from push-a-rejection-reason to
|
|
55
|
+
* propagate-the-throw: a validator crash is an OPERATIONAL incident, and
|
|
56
|
+
* persisting it as a `_rejected` record permanently poisoned the run_id via
|
|
57
|
+
* ingest's duplicate guard (the exact V2-CROSS-BO-001 pattern the sibling
|
|
58
|
+
* scenario-fetch path already fixed). Both production callers
|
|
59
|
+
* (packages/ingest/run.js's outer catch and cli.js's run() catch) map a
|
|
60
|
+
* verify() throw to exit 2 with NOTHING persisted, so a clean resubmission
|
|
61
|
+
* after recovery is accepted.
|
|
62
|
+
*
|
|
49
63
|
* `name` is upper-cased to match the prefix vocabulary documented in
|
|
50
|
-
* verify/README.md (SCHEMA / POLICY / STEPS).
|
|
51
|
-
* { ok: true, result } when fn() returned cleanly
|
|
52
|
-
* { ok: false, faultReason } when fn() threw
|
|
64
|
+
* verify/README.md (SCHEMA / POLICY / STEPS).
|
|
53
65
|
*
|
|
54
66
|
* The helper is the canonical seam for any new synchronous validator
|
|
55
67
|
* — adding a 4th validator class becomes a one-call site, not a
|
|
@@ -57,17 +69,54 @@ const RECORD_SCHEMA_VERSION = SUPPORTED_SCHEMA_VERSIONS.record.current;
|
|
|
57
69
|
*
|
|
58
70
|
* @param {string} name - Validator class name (will be upper-cased).
|
|
59
71
|
* @param {() => T} fn - The validator call.
|
|
60
|
-
* @returns {
|
|
72
|
+
* @returns {T} fn()'s result when it returned cleanly.
|
|
73
|
+
* @throws {Error} classified `VALIDATOR_FAULT_<NAME>` error when fn() threw.
|
|
61
74
|
* @template T
|
|
62
75
|
*/
|
|
63
76
|
function runValidator(name, fn) {
|
|
64
77
|
try {
|
|
65
|
-
return
|
|
78
|
+
return fn();
|
|
66
79
|
} catch (e) {
|
|
67
80
|
const cls = name.toUpperCase();
|
|
68
81
|
const detail = e && e.message ? e.message : String(e);
|
|
69
|
-
|
|
82
|
+
const fault = new Error(`VALIDATOR_FAULT_${cls}: ${detail}`);
|
|
83
|
+
fault.code = `VALIDATOR_FAULT_${cls}`;
|
|
84
|
+
fault.cause = e;
|
|
85
|
+
throw fault;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* F-4036ae25: does `submission` carry a repo / run_id / timing.finished_at
|
|
91
|
+
* shape that packages/ingest/persist.js's computeRecordPath() could place on
|
|
92
|
+
* disk? Mirrors computeRecordPath's OWN structural preconditions (segment
|
|
93
|
+
* count, run_id charset, a parseable finished_at) — deliberately NOT its
|
|
94
|
+
* stricter isUnsafeSegment traversal guard. That guard rejects strings (e.g.
|
|
95
|
+
* `../etc`) this submission schema's own repo pattern also accepts, so a
|
|
96
|
+
* submission can be schema-VALID and still fail it; that gap belongs to
|
|
97
|
+
* F-4acd28d8, handled downstream in writeRecord()/ingest() once the record
|
|
98
|
+
* already has a verdict. Here we only need "is this filable in principle" —
|
|
99
|
+
* a three-field structural question `verify()` can answer locally without
|
|
100
|
+
* importing `@dogfood-lab/ingest` (which already imports `@dogfood-lab/verify`;
|
|
101
|
+
* importing back would close a new cross-package cycle).
|
|
102
|
+
*
|
|
103
|
+
* @param {object} submission
|
|
104
|
+
* @returns {boolean}
|
|
105
|
+
*/
|
|
106
|
+
function hasFilablePathIdentity(submission) {
|
|
107
|
+
const repo = submission.repo;
|
|
108
|
+
if (typeof repo !== 'string') return false;
|
|
109
|
+
const segments = repo.split('/');
|
|
110
|
+
if (segments.length !== 2 || !segments[0] || !segments[1]) return false;
|
|
111
|
+
|
|
112
|
+
if (typeof submission.run_id !== 'string' || !/^[\w-]+$/.test(submission.run_id)) {
|
|
113
|
+
return false;
|
|
70
114
|
}
|
|
115
|
+
|
|
116
|
+
const finishedAt = submission.timing && typeof submission.timing === 'object'
|
|
117
|
+
? submission.timing.finished_at
|
|
118
|
+
: undefined;
|
|
119
|
+
return typeof finishedAt === 'string' && !isNaN(new Date(finishedAt).getTime());
|
|
71
120
|
}
|
|
72
121
|
|
|
73
122
|
/**
|
|
@@ -79,6 +128,12 @@ function runValidator(name, fn) {
|
|
|
79
128
|
* @param {object|null} options.repoPolicy - Parsed repo policy (null if none)
|
|
80
129
|
* @param {object} options.provenance - Provenance adapter { confirm(source) => Promise<boolean> }
|
|
81
130
|
* @param {string} options.policyVersion - Semver of the policy set being applied
|
|
131
|
+
* @param {Map<string, object>|null} [options.scenarios] - Loaded scenario
|
|
132
|
+
* definitions keyed by scenario_id (from loadScenarios). When present, each
|
|
133
|
+
* scenario_result is checked against its definition's
|
|
134
|
+
* success_criteria.required_steps — the REAL enforcement behind the
|
|
135
|
+
* `step-results-present` / `step-verdict-consistent` global reject rules
|
|
136
|
+
* (F-3bfc2885). Absent/null keeps the legacy structural-only behavior.
|
|
82
137
|
* @param {object} [options.validators] - Test-only override hook (D1B-003).
|
|
83
138
|
* Pass `{ validateSubmissionSchema, validateStepResults, validatePolicy }`
|
|
84
139
|
* to swap in fault-injecting stubs. Production callers leave this unset
|
|
@@ -114,12 +169,13 @@ export async function verify(submission, options) {
|
|
|
114
169
|
};
|
|
115
170
|
}
|
|
116
171
|
|
|
117
|
-
const { globalPolicy, repoPolicy, provenance, policyVersion, validators: validatorOverrides = {} } = options;
|
|
172
|
+
const { globalPolicy, repoPolicy, provenance, policyVersion, scenarios = null, validators: validatorOverrides = {} } = options;
|
|
118
173
|
// Resolve the three validators per-call so tests can inject fault-
|
|
119
174
|
// injecting stubs. Production callers leave `validators` unset and the
|
|
120
175
|
// module-level imports flow through unchanged.
|
|
121
176
|
const validateSubmissionSchema = validatorOverrides.validateSubmissionSchema || _defaultValidateSubmissionSchema;
|
|
122
177
|
const validateStepResults = validatorOverrides.validateStepResults || _defaultValidateStepResults;
|
|
178
|
+
const validateRequiredSteps = validatorOverrides.validateRequiredSteps || _defaultValidateRequiredSteps;
|
|
123
179
|
const validatePolicy = validatorOverrides.validatePolicy || _defaultValidatePolicy;
|
|
124
180
|
const validateSchemaVersion = validatorOverrides.validateSchemaVersion || _defaultValidateSchemaVersion;
|
|
125
181
|
const now = new Date().toISOString();
|
|
@@ -151,18 +207,12 @@ export async function verify(submission, options) {
|
|
|
151
207
|
}
|
|
152
208
|
|
|
153
209
|
// 1. Schema validation
|
|
154
|
-
// D1B-003
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
schemaResult = schemaRun.result;
|
|
161
|
-
if (!schemaResult.valid) {
|
|
162
|
-
reasons.push(...schemaResult.errors.map(e => `schema: ${e}`));
|
|
163
|
-
}
|
|
164
|
-
} else {
|
|
165
|
-
reasons.push(schemaRun.faultReason);
|
|
210
|
+
// D1B-003 / F-82429f90: submission-bad results push the `'schema: …'`
|
|
211
|
+
// prefix; a validator CRASH propagates out of verify() as a classified
|
|
212
|
+
// VALIDATOR_FAULT_SCHEMA throw (operational — exit 2, nothing persisted).
|
|
213
|
+
const schemaResult = runValidator('schema', () => validateSubmissionSchema(submission));
|
|
214
|
+
if (!schemaResult.valid) {
|
|
215
|
+
reasons.push(...schemaResult.errors.map(e => `schema: ${e}`));
|
|
166
216
|
}
|
|
167
217
|
|
|
168
218
|
// 1b. schema_version VALUE gate (F1-CONTRACTS-001)
|
|
@@ -174,15 +224,12 @@ export async function verify(submission, options) {
|
|
|
174
224
|
// a future-major payload may also fail shape, but the version refusal is the
|
|
175
225
|
// operator-actionable signal and must land regardless. The validator emits a
|
|
176
226
|
// fully-prefixed `CONTRACT_SCHEMA_TOO_NEW:` / `CONTRACT_SCHEMA_TOO_OLD:`
|
|
177
|
-
// reason; an unknown-contract throw
|
|
178
|
-
// `VALIDATOR_FAULT_CONTRACT_SCHEMA_VERSION` via the same
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
}
|
|
184
|
-
} else {
|
|
185
|
-
reasons.push(versionRun.faultReason);
|
|
227
|
+
// reason; an unknown-contract throw propagates as a classified
|
|
228
|
+
// `VALIDATOR_FAULT_CONTRACT_SCHEMA_VERSION` fault via the same
|
|
229
|
+
// runValidator seam (F-82429f90: operational, never persisted).
|
|
230
|
+
const versionResult = runValidator('contract_schema_version', () => validateSchemaVersion(submission, 'recordSubmission'));
|
|
231
|
+
if (!versionResult.valid) {
|
|
232
|
+
reasons.push(...versionResult.errors);
|
|
186
233
|
}
|
|
187
234
|
|
|
188
235
|
// 2. Reject if submission includes verifier-owned fields
|
|
@@ -209,15 +256,25 @@ export async function verify(submission, options) {
|
|
|
209
256
|
refCommitSha: submission.ref?.commit_sha
|
|
210
257
|
});
|
|
211
258
|
} catch (err) {
|
|
212
|
-
// verify-A-002: the adapter THROWS on operational provider
|
|
213
|
-
// (429 rate-limit, 5xx outage, 401/403 token
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
|
|
259
|
+
// verify-A-002 + F-82429f90: the adapter THROWS on operational provider
|
|
260
|
+
// faults (429 rate-limit, 5xx outage, 401/403 token, exhausted-retry
|
|
261
|
+
// transport errors — F-dac7e08c) and returns false only for a
|
|
262
|
+
// genuinely-absent run (HTTP 404). RETHROW as a classified
|
|
263
|
+
// PROVENANCE_FAULT so the incident propagates out of verify() —
|
|
264
|
+
// mirroring SCENARIO_FETCH_FAULT — and is NEVER assembled into a
|
|
265
|
+
// persisted `_rejected` record: persisting an outage-window rejection
|
|
266
|
+
// permanently blocked the run_id via ingest's duplicate guard. Both
|
|
267
|
+
// production callers (ingest run.js outer catch; verify cli.js run()
|
|
268
|
+
// catch) map the throw to exit 2 with nothing persisted. The message
|
|
269
|
+
// keeps the `provenance-fault:` prefix parseRejectionReason classifies
|
|
270
|
+
// as operational. The not-confirmed case below keeps the bare
|
|
271
|
+
// `provenance:` prefix (still submission-bad).
|
|
272
|
+
const fault = new Error(`provenance-fault: verification failed: ${err.message}`);
|
|
273
|
+
fault.code = 'PROVENANCE_FAULT';
|
|
274
|
+
fault.cause = err;
|
|
275
|
+
throw fault;
|
|
219
276
|
}
|
|
220
|
-
if (!provenanceConfirmed
|
|
277
|
+
if (!provenanceConfirmed) {
|
|
221
278
|
reasons.push('provenance: source run could not be confirmed');
|
|
222
279
|
}
|
|
223
280
|
}
|
|
@@ -226,11 +283,24 @@ export async function verify(submission, options) {
|
|
|
226
283
|
// D1B-003: same submission-bad-vs-operational-fault split as schema.
|
|
227
284
|
if (schemaResult.valid && submission.scenario_results) {
|
|
228
285
|
for (const scenario of submission.scenario_results) {
|
|
229
|
-
const
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
286
|
+
const stepsErrors = runValidator('steps', () => validateStepResults(scenario));
|
|
287
|
+
reasons.push(...stepsErrors.map(e => `steps[${scenario.scenario_id}]: ${e}`));
|
|
288
|
+
|
|
289
|
+
// F-3bfc2885: enforce success_criteria.required_steps when the caller
|
|
290
|
+
// loaded the scenario definition. This is the enforcement behind the
|
|
291
|
+
// `step-results-present` / `step-verdict-consistent` global reject rules
|
|
292
|
+
// that KNOWN_REJECT_RULE_IDS attributes to "verify() itself" — before
|
|
293
|
+
// this wiring, validateRequiredSteps had no callers and the rules were
|
|
294
|
+
// declared-but-unenforced. A scenario_result whose definition failed to
|
|
295
|
+
// load is NOT checked here; loadScenarios already reports that as a
|
|
296
|
+
// `scenario-load:` rejection at the ingest layer.
|
|
297
|
+
if (scenarios && typeof scenarios.get === 'function') {
|
|
298
|
+
const definition = scenarios.get(scenario.scenario_id);
|
|
299
|
+
const requiredSteps = definition?.success_criteria?.required_steps ?? [];
|
|
300
|
+
if (requiredSteps.length > 0) {
|
|
301
|
+
const requiredErrors = runValidator('steps', () => validateRequiredSteps(scenario, requiredSteps));
|
|
302
|
+
reasons.push(...requiredErrors.map(e => `steps[${scenario.scenario_id}]: ${e}`));
|
|
303
|
+
}
|
|
234
304
|
}
|
|
235
305
|
}
|
|
236
306
|
}
|
|
@@ -261,19 +331,16 @@ export async function verify(submission, options) {
|
|
|
261
331
|
reasons.push(`policy: repo policy unreadable — ${detail}`);
|
|
262
332
|
// policyValid stays false.
|
|
263
333
|
} else {
|
|
264
|
-
const
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
} else {
|
|
275
|
-
reasons.push(policyRun.faultReason);
|
|
276
|
-
}
|
|
334
|
+
const policyResult = runValidator('policy', () => validatePolicy(submission, { globalPolicy, repoPolicy }));
|
|
335
|
+
policyValid = policyResult.valid;
|
|
336
|
+
reasons.push(...policyResult.errors.map(e => `policy: ${e}`));
|
|
337
|
+
// VERIFY-F1: a malformed REPO custom-rule predicate is a distinct
|
|
338
|
+
// `policy-config:` rejection (submission-bad — the repo authored the bad
|
|
339
|
+
// rule). A malformed GLOBAL predicate never reaches here; it throws and
|
|
340
|
+
// propagates as a classified VALIDATOR_FAULT_POLICY (operational,
|
|
341
|
+
// F-82429f90 — exit 2, nothing persisted) via runValidator.
|
|
342
|
+
reasons.push(...(policyResult.configErrors || []).map(e => `policy-config: ${e}`));
|
|
343
|
+
warnings.push(...(policyResult.warnings || []).map(w => `policy: ${w}`));
|
|
277
344
|
}
|
|
278
345
|
}
|
|
279
346
|
|
|
@@ -294,8 +361,85 @@ export async function verify(submission, options) {
|
|
|
294
361
|
});
|
|
295
362
|
|
|
296
363
|
// 7. Assemble persisted record
|
|
364
|
+
//
|
|
365
|
+
// A submission whose repo/run_id/timing.finished_at cannot drive
|
|
366
|
+
// computeRecordPath() is marked `_skipPersist` — the SAME sentinel the
|
|
367
|
+
// null/non-object branch above sets, because it is the same claim: we
|
|
368
|
+
// cannot place this on disk, so there is nothing to file. The record is
|
|
369
|
+
// still assembled and returned (the submitter needs the reasons); it is
|
|
370
|
+
// simply never written.
|
|
371
|
+
//
|
|
372
|
+
// F-4036ae25: this used to fire for EVERY schema-invalid submission, not
|
|
373
|
+
// only this unfilable subset — broader than the rationale below justifies.
|
|
374
|
+
// A submission whose repo/run_id/timing.finished_at are perfectly fine but
|
|
375
|
+
// fails schema on an unrelated field (e.g. an unexpected top-level property)
|
|
376
|
+
// IS filable, and blanket-skipping it left the fleet-wide audit trail with a
|
|
377
|
+
// rejection nobody could see anywhere durable. hasFilablePathIdentity()
|
|
378
|
+
// re-checks the three path-deriving fields directly against `submission`
|
|
379
|
+
// (not against WHICH schema rule fired), so this stays correct regardless of
|
|
380
|
+
// which field the schema violation is actually on.
|
|
381
|
+
//
|
|
382
|
+
// Skipping is not merely an economy. isDuplicate checks the _rejected path,
|
|
383
|
+
// so persisting an UNFILABLE rejection isn't possible anyway (computeRecordPath
|
|
384
|
+
// would throw); persisting a FILABLE one, on the other hand, reopens the
|
|
385
|
+
// V2-CROSS-BO-001 / F-82429f90 run_id-poisoning pathology for the
|
|
386
|
+
// newly-persisted category — a corrected resubmission silently dropped as a
|
|
387
|
+
// duplicate (exit 0). packages/ingest/persist.js's isDuplicate() closes that
|
|
388
|
+
// gap: a prior `_rejected` record whose reasons are ALL retryable-class is
|
|
389
|
+
// treated as non-blocking for a same-run_id retry that goes on to be
|
|
390
|
+
// ACCEPTED, so the audit trail and the resubmission path both hold. (A
|
|
391
|
+
// schema-invalid submission that reaches writeRecord() may still fail
|
|
392
|
+
// dogfood-record.schema.json's own mirrored constraints on the SAME field —
|
|
393
|
+
// ingest() routes that RecordValidationError the same way _skipPersist
|
|
394
|
+
// routes here; see F-4acd28d8.) An UNCORRECTED retry — still rejected,
|
|
395
|
+
// same run_id and date — computes to the exact SAME `_rejected` path
|
|
396
|
+
// attempt 1 already occupies regardless of which violation it reports, so
|
|
397
|
+
// it stays an ordinary duplicate rather than reaching writeRecord()'s
|
|
398
|
+
// exclusive-create at all (F-0f9e4077: the retry carve-out only ever
|
|
399
|
+
// waives the collision for a record now headed to a DIFFERENT, accepted
|
|
400
|
+
// path).
|
|
401
|
+
//
|
|
402
|
+
// The line: the duplicate guard otherwise consumes a run_id when we
|
|
403
|
+
// rendered a VERDICT on a run's own reported content. A policy or
|
|
404
|
+
// provenance rejection is such a verdict — it persists, and PERMANENTLY
|
|
405
|
+
// consuming the run_id is the intended anti-gaming behavior (a submitter
|
|
406
|
+
// must not be able to launder a genuinely-bad run into an accepted one by
|
|
407
|
+
// resubmitting different self-reported content under the same run_id).
|
|
408
|
+
// "We could not read/place/shape your submission" (schema, repo,
|
|
409
|
+
// unsafe-record-path, ...) is not a content verdict — it persists too when
|
|
410
|
+
// filable, but stays reusable for a genuinely corrected resubmission.
|
|
411
|
+
//
|
|
412
|
+
// F-f8952a50 (wave 10): this split is now a PER-PREFIX flag —
|
|
413
|
+
// parse-rejection.js's `retryable` field, consulted by persist.js's
|
|
414
|
+
// isRetryableRejection() — not a blanket "schema: only" allowlist. The
|
|
415
|
+
// shape/addressing subset of `submission-bad` (schema:, repo:,
|
|
416
|
+
// unsafe-record-path:, steps[<id>]:, policy-config:,
|
|
417
|
+
// submission-contains-verifier-field:, CONTRACT_SCHEMA_TOO_OLD:) is
|
|
418
|
+
// `retryable: true`; the content-verdict subset (policy:, provenance:)
|
|
419
|
+
// stays `retryable: false`, exactly as this paragraph describes. See
|
|
420
|
+
// parse-rejection.js's file header for the full rationale and
|
|
421
|
+
// schema-invalid-skip-persist.test.js's "REGRESSION GUARD" /
|
|
422
|
+
// "persist-a-verdict doctrine is preserved" tests for the pinned proof that
|
|
423
|
+
// policy:/provenance: remain terminal.
|
|
424
|
+
//
|
|
425
|
+
// CONTRACT_SCHEMA_TOO_NEW: does NOT belong in the shape/addressing list
|
|
426
|
+
// above — corrected here, wave 22 (F-51780da9's sweep found this comment
|
|
427
|
+
// stale in the same way it found parse-rejection.test.js's F-be0deacd test
|
|
428
|
+
// comment stale). F-be0deacd (wave 20) moved it to class 'operational' with
|
|
429
|
+
// `retryable: false` permanently: only a testing-os upgrade satisfies
|
|
430
|
+
// `major > maxMajor`, never a payload correction. `retryable` is a static
|
|
431
|
+
// per-prefix flag with no visibility into a later upgrade, so
|
|
432
|
+
// persist.js's isRetryableRejection() carries one additional, narrow
|
|
433
|
+
// re-check for this prefix only — re-deriving whether the stored
|
|
434
|
+
// rejection's declared major is still above the CURRENT build's
|
|
435
|
+
// SUPPORTED_SCHEMA_VERSIONS ceiling, rather than trusting the frozen
|
|
436
|
+
// boolean — so a stale TOO_NEW rejection does not permanently poison a
|
|
437
|
+
// run_id once the operator upgrades past the declared major. See
|
|
438
|
+
// isRetryableRejection's own doc comment (packages/ingest/persist.js).
|
|
439
|
+
const canFilePath = schemaResult.valid || hasFilablePathIdentity(submission);
|
|
297
440
|
const persisted = {
|
|
298
441
|
schema_version: RECORD_SCHEMA_VERSION,
|
|
442
|
+
...(canFilePath ? {} : { _skipPersist: true }),
|
|
299
443
|
policy_version: policyVersion,
|
|
300
444
|
run_id: submission.run_id,
|
|
301
445
|
repo: submission.repo,
|
package/package.json
CHANGED
package/parse-rejection.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* a fresh drift source the moment a prefix is added.
|
|
9
9
|
*
|
|
10
10
|
* `parseRejectionReason(reason)` is the single exported classifier. It maps a
|
|
11
|
-
* raw rejection string to `{ class, prefix, detail }`:
|
|
11
|
+
* raw rejection string to `{ class, prefix, detail, retryable }`:
|
|
12
12
|
*
|
|
13
13
|
* - class — the routing decision (who fixes this):
|
|
14
14
|
* 'submission-bad' → the submitter fixes the payload and resubmits
|
|
@@ -20,6 +20,36 @@
|
|
|
20
20
|
* `'steps[<id>]:'`), or `null` for the 'unknown' class.
|
|
21
21
|
* - detail — the human-readable remainder with the matched prefix stripped
|
|
22
22
|
* (for 'unknown', the whole string verbatim).
|
|
23
|
+
* - retryable — (F-f8952a50, wave 10) may a same-run_id resubmission whose
|
|
24
|
+
* ONLY prior rejection carries this prefix still reach an
|
|
25
|
+
* acceptance, once corrected? This is a NARROWER question than
|
|
26
|
+
* `class`. Every 'operational' / 'ingest' / 'unknown' reason is
|
|
27
|
+
* `retryable: false` (only the submitter's own payload earns a
|
|
28
|
+
* retry — an ops fault or an unrecognized signal never does).
|
|
29
|
+
* Within 'submission-bad', the taxonomy splits further:
|
|
30
|
+
* - retryable: true — the prefix means "we could not even
|
|
31
|
+
* read/place/shape your submission" (schema:, repo:,
|
|
32
|
+
* unsafe-record-path:, steps[<id>]:, policy-config:,
|
|
33
|
+
* submission-contains-verifier-field:,
|
|
34
|
+
* CONTRACT_SCHEMA_TOO_OLD:) — a correction changes nothing
|
|
35
|
+
* about what the run actually did, only how it was
|
|
36
|
+
* addressed/formatted/configured. (CONTRACT_SCHEMA_TOO_NEW:
|
|
37
|
+
* moved to class 'operational' / retryable: false in
|
|
38
|
+
* F-be0deacd, wave 20 — see the LITERAL_PREFIXES entry
|
|
39
|
+
* below for why it is NOT the submitter's problem.)
|
|
40
|
+
* - retryable: false — the prefix means "we read your
|
|
41
|
+
* submission and rendered a VERDICT against its content"
|
|
42
|
+
* (policy:, provenance:) — consuming the run_id is the
|
|
43
|
+
* INTENDED anti-gaming behavior: a submitter cannot launder
|
|
44
|
+
* a genuinely-failed run into an accepted one by resubmitting
|
|
45
|
+
* different self-reported content under the same run_id.
|
|
46
|
+
* Pinned end-to-end: schema-invalid-skip-persist.test.js's
|
|
47
|
+
* "REGRESSION GUARD: a resubmission after a persisted
|
|
48
|
+
* NON-schema rejection is STILL blocked" test (provenance:)
|
|
49
|
+
* and f-0f9e4077-retry-collision-duplicate-rejection.test.js's
|
|
50
|
+
* "F-f8952a50" describe block (policy: / provenance: cases).
|
|
51
|
+
* Consumers key off THIS field, never off `prefix` — see
|
|
52
|
+
* packages/ingest/persist.js's isRetryableRejection().
|
|
23
53
|
*
|
|
24
54
|
* The prefix vocabulary below is enumerated from the ACTUAL emitters — it is
|
|
25
55
|
* NOT invented:
|
|
@@ -32,9 +62,19 @@
|
|
|
32
62
|
* submission-contains-verifier-field:,
|
|
33
63
|
* submission-malformed:,
|
|
34
64
|
* VALIDATOR_FAULT_<NAME>:
|
|
35
|
-
* - validators/schema-version.js: CONTRACT_SCHEMA_TOO_NEW
|
|
36
|
-
*
|
|
37
|
-
*
|
|
65
|
+
* - validators/schema-version.js: CONTRACT_SCHEMA_TOO_NEW: (this BUILD is
|
|
66
|
+
* behind a schema major its submitters
|
|
67
|
+
* already adopted → operational, F-be0deacd
|
|
68
|
+
* wave 20), CONTRACT_SCHEMA_TOO_OLD: (the
|
|
69
|
+
* SUBMITTER is behind → submission-bad)
|
|
70
|
+
* - packages/ingest/run.js: scenario-load:,
|
|
71
|
+
* unsafe-record-path: (F-4acd28d8,
|
|
72
|
+
* computeRecordPath's traversal guard
|
|
73
|
+
* rejected a schema-valid repo →
|
|
74
|
+
* submission-bad)
|
|
75
|
+
* - packages/ingest/load-context.js: scenario-fetch-fault: (V2-CROSS-BO-001,
|
|
76
|
+
* scenario-fetch outage/credential fault
|
|
77
|
+
* → operational)
|
|
38
78
|
*
|
|
39
79
|
* `VALIDATOR_FAULT_*` is matched by family, not by an exhaustive name list, so
|
|
40
80
|
* a future `VALIDATOR_FAULT_<NEW>:` (e.g. the runValidator seam adds a 5th
|
|
@@ -49,29 +89,42 @@
|
|
|
49
89
|
|
|
50
90
|
/**
|
|
51
91
|
* @typedef {object} ParsedRejection
|
|
52
|
-
* @property {RejectionClass} class
|
|
53
|
-
* @property {string | null} prefix
|
|
54
|
-
* @property {string} detail
|
|
92
|
+
* @property {RejectionClass} class Routing decision (who must act).
|
|
93
|
+
* @property {string | null} prefix Canonical prefix token, or null when unknown.
|
|
94
|
+
* @property {string} detail Human-readable remainder (prefix stripped).
|
|
95
|
+
* @property {boolean} retryable Whether a same-run_id resubmission whose
|
|
96
|
+
* ONLY prior rejection carries this prefix may still reach an acceptance.
|
|
97
|
+
* See the file header for the full submission-bad split (shape/addressing
|
|
98
|
+
* vs. rendered-verdict).
|
|
55
99
|
*/
|
|
56
100
|
|
|
57
101
|
/**
|
|
58
102
|
* Literal-prefix matchers, ordered most-specific-first. Each carries the
|
|
59
|
-
* canonical prefix token
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
103
|
+
* canonical prefix token, the routing class, and whether a resubmission past
|
|
104
|
+
* this rejection is retryable (F-f8952a50 — see file header). `match` is the
|
|
105
|
+
* exact string the reason must start with; `detail` is whatever follows it
|
|
106
|
+
* (trimmed of one leading space). `repo:` matches the `repo:mismatch: …`
|
|
107
|
+
* family — the canonical token reported is the stable `repo:` head.
|
|
63
108
|
*/
|
|
64
109
|
const LITERAL_PREFIXES = [
|
|
65
|
-
// submission-bad
|
|
66
|
-
{ match: 'schema:', prefix: 'schema:', class: 'submission-bad' },
|
|
110
|
+
// submission-bad, retryable — "we could not even read your submission".
|
|
111
|
+
{ match: 'schema:', prefix: 'schema:', class: 'submission-bad', retryable: true },
|
|
67
112
|
// VERIFY-F1: a malformed REPO custom-rule predicate (eval-time semantic fault the
|
|
68
113
|
// schema could not catch). submission-bad — the repo authored the bad policy YAML,
|
|
69
114
|
// so the fix belongs to the submitter, not studio ops. Ordered BEFORE `policy:`
|
|
70
115
|
// to keep the most-specific-first invariant (the tokens don't overlap — `policy:`
|
|
71
116
|
// ends at the colon, `policy-config:` continues with `-config:` — but the order is
|
|
72
117
|
// explicit). A malformed GLOBAL predicate is operational (VALIDATOR_FAULT_POLICY).
|
|
73
|
-
|
|
74
|
-
|
|
118
|
+
// retryable — the REPO's policy YAML was broken, not a judgment on this run's
|
|
119
|
+
// content; once the repo fixes policies/*.yaml, the identical submission can
|
|
120
|
+
// legitimately evaluate cleanly. Shape/config, not a verdict.
|
|
121
|
+
{ match: 'policy-config:', prefix: 'policy-config:', class: 'submission-bad', retryable: true },
|
|
122
|
+
// NOT retryable — a rendered verdict on the run's own reported content
|
|
123
|
+
// (forbidden tags, missing required evidence, a failed custom_rules
|
|
124
|
+
// predicate). Consuming the run_id here is the deliberate anti-gaming
|
|
125
|
+
// behavior: pinned by schema-invalid-skip-persist.test.js's "the
|
|
126
|
+
// persist-a-verdict doctrine is preserved" describe block.
|
|
127
|
+
{ match: 'policy:', prefix: 'policy:', class: 'submission-bad', retryable: false },
|
|
75
128
|
// operational — a provider-side provenance FAULT (429/5xx/401/403). The
|
|
76
129
|
// adapter THROWS these (verify-A-002); index.js catches the throw and emits
|
|
77
130
|
// this distinct `provenance-fault:` prefix so the incident pages ops. Ordered
|
|
@@ -79,22 +132,57 @@ const LITERAL_PREFIXES = [
|
|
|
79
132
|
// the longer, more-specific match first preserves the most-specific-first
|
|
80
133
|
// invariant the array is sorted by. The genuine not-confirmed case
|
|
81
134
|
// (`provenance: source run could not be confirmed`) stays submission-bad below.
|
|
82
|
-
{ match: 'provenance-fault:', prefix: 'provenance-fault:', class: 'operational' },
|
|
83
|
-
|
|
84
|
-
|
|
135
|
+
{ match: 'provenance-fault:', prefix: 'provenance-fault:', class: 'operational', retryable: false },
|
|
136
|
+
// submission-bad, NOT retryable — a rendered verdict: the provider could not
|
|
137
|
+
// confirm this specific run happened as claimed. Resubmitting different
|
|
138
|
+
// self-reported content under the same run_id to "become confirmable" is
|
|
139
|
+
// exactly the laundering the anti-gaming doctrine blocks. Pinned by
|
|
140
|
+
// schema-invalid-skip-persist.test.js's REGRESSION GUARD test.
|
|
141
|
+
{ match: 'provenance:', prefix: 'provenance:', class: 'submission-bad', retryable: false },
|
|
142
|
+
// submission-bad, retryable — pure identity/addressing: the run happened,
|
|
143
|
+
// only the repo/run_url pairing was mis-stated. Proven live (F-f8952a50): a
|
|
144
|
+
// repo:mismatch correction must reach verify() again, not vanish as a
|
|
145
|
+
// silent duplicate.
|
|
146
|
+
{ match: 'repo:', prefix: 'repo:', class: 'submission-bad', retryable: true },
|
|
85
147
|
{
|
|
86
148
|
match: 'submission-contains-verifier-field:',
|
|
87
149
|
prefix: 'submission-contains-verifier-field:',
|
|
88
150
|
class: 'submission-bad',
|
|
151
|
+
retryable: true,
|
|
89
152
|
},
|
|
90
|
-
|
|
91
|
-
|
|
153
|
+
// F-be0deacd (wave 20): TOO_NEW and TOO_OLD are NOT symmetric, despite the
|
|
154
|
+
// near-identical prefixes and shared emitter (validators/schema-version.js).
|
|
155
|
+
// TOO_OLD means the SUBMITTER is behind — re-emitting against the current
|
|
156
|
+
// contract fixes it, so it stays submission-bad/retryable. TOO_NEW means
|
|
157
|
+
// THIS BUILD is behind a schema major its own submitters have already
|
|
158
|
+
// adopted (schema-version.js's own JSDoc: "operator must upgrade
|
|
159
|
+
// testing-os") — no resubmission, corrected or not, can ever satisfy a
|
|
160
|
+
// `major > maxMajor` comparison until testing-os itself ships an upgrade.
|
|
161
|
+
// That is an ops action, not a submitter action, so TOO_NEW routes
|
|
162
|
+
// operational/not-retryable — the same bucket VALIDATOR_FAULT_* and
|
|
163
|
+
// submission-malformed: use for "page ops, don't bounce it back".
|
|
164
|
+
{ match: 'CONTRACT_SCHEMA_TOO_NEW:', prefix: 'CONTRACT_SCHEMA_TOO_NEW:', class: 'operational', retryable: false },
|
|
165
|
+
{ match: 'CONTRACT_SCHEMA_TOO_OLD:', prefix: 'CONTRACT_SCHEMA_TOO_OLD:', class: 'submission-bad', retryable: true },
|
|
166
|
+
// submission-bad — F-4acd28d8: the record passed schema validation but its
|
|
167
|
+
// OWN repo identifier is unsafe to file under (computeRecordPath's
|
|
168
|
+
// isUnsafeSegment traversal guard, stricter than the schema's repo
|
|
169
|
+
// pattern — see F-bbbe2e1f). The submitter's repo string is the problem,
|
|
170
|
+
// not the verifier/ingest tooling, so this stays submission-bad.
|
|
171
|
+
// retryable — addressing, not a verdict on the run's content.
|
|
172
|
+
{ match: 'unsafe-record-path:', prefix: 'unsafe-record-path:', class: 'submission-bad', retryable: true },
|
|
92
173
|
// operational — a null/non-object submission is a malfunctioning dispatcher
|
|
93
174
|
// (verify-B-003), not a submitter who sent a bad-but-shaped payload. Page ops;
|
|
94
175
|
// do NOT bounce it back to the submitter.
|
|
95
|
-
{ match: 'submission-malformed:', prefix: 'submission-malformed:', class: 'operational' },
|
|
176
|
+
{ match: 'submission-malformed:', prefix: 'submission-malformed:', class: 'operational', retryable: false },
|
|
177
|
+
// operational — the scenario fetcher THREW after exhausting its retry
|
|
178
|
+
// budget (5xx/429 outage, transport reject) or hit a credential fault
|
|
179
|
+
// (401/403). V2-CROSS-BO-001: packages/ingest/load-context.js throws this
|
|
180
|
+
// classified error instead of returning `not_found`, so an outage never
|
|
181
|
+
// rejects a good submission. Ordered before `scenario-load:` to keep the
|
|
182
|
+
// most-specific-first invariant explicit (the tokens do not overlap).
|
|
183
|
+
{ match: 'scenario-fetch-fault:', prefix: 'scenario-fetch-fault:', class: 'operational', retryable: false },
|
|
96
184
|
// ingest
|
|
97
|
-
{ match: 'scenario-load:', prefix: 'scenario-load:', class: 'ingest' },
|
|
185
|
+
{ match: 'scenario-load:', prefix: 'scenario-load:', class: 'ingest', retryable: false },
|
|
98
186
|
];
|
|
99
187
|
|
|
100
188
|
/** Strip a matched literal prefix and one optional leading space from a reason. */
|
|
@@ -111,27 +199,33 @@ function stripLiteral(reason, match) {
|
|
|
111
199
|
*/
|
|
112
200
|
export function parseRejectionReason(reason) {
|
|
113
201
|
if (typeof reason !== 'string') {
|
|
114
|
-
return { class: 'unknown', prefix: null, detail: '' };
|
|
202
|
+
return { class: 'unknown', prefix: null, detail: '', retryable: false };
|
|
115
203
|
}
|
|
116
204
|
|
|
117
205
|
// 1. Operational faults — matched by FAMILY so future VALIDATOR_FAULT_<NEW>
|
|
118
206
|
// classes need no edit here. (runValidator emits `VALIDATOR_FAULT_<CLS>: …`.)
|
|
207
|
+
// Never retryable — an ops fault is not the submitter's to fix.
|
|
119
208
|
const faultMatch = reason.match(/^(VALIDATOR_FAULT_[A-Z0-9_]+):\s*/);
|
|
120
209
|
if (faultMatch) {
|
|
121
210
|
return {
|
|
122
211
|
class: 'operational',
|
|
123
212
|
prefix: `${faultMatch[1]}:`,
|
|
124
213
|
detail: reason.slice(faultMatch[0].length),
|
|
214
|
+
retryable: false,
|
|
125
215
|
};
|
|
126
216
|
}
|
|
127
217
|
|
|
128
218
|
// 2. steps[<id>]: — bracketed id, so matched by regex rather than a literal.
|
|
219
|
+
// retryable — a structural/completeness mismatch between reported step
|
|
220
|
+
// results and the scenario's declared required_steps, not a verdict on
|
|
221
|
+
// whether the run's steps passed.
|
|
129
222
|
const stepsMatch = reason.match(/^steps\[[^\]]*\]:\s*/);
|
|
130
223
|
if (stepsMatch) {
|
|
131
224
|
return {
|
|
132
225
|
class: 'submission-bad',
|
|
133
226
|
prefix: 'steps[<id>]:',
|
|
134
227
|
detail: reason.slice(stepsMatch[0].length),
|
|
228
|
+
retryable: true,
|
|
135
229
|
};
|
|
136
230
|
}
|
|
137
231
|
|
|
@@ -144,12 +238,16 @@ export function parseRejectionReason(reason) {
|
|
|
144
238
|
class: entry.class,
|
|
145
239
|
prefix: entry.prefix,
|
|
146
240
|
detail: stripLiteral(reason, entry.match),
|
|
241
|
+
retryable: entry.retryable,
|
|
147
242
|
};
|
|
148
243
|
}
|
|
149
244
|
}
|
|
150
245
|
|
|
151
|
-
// 4. Unrecognized — log + surface raw
|
|
152
|
-
//
|
|
153
|
-
//
|
|
154
|
-
|
|
246
|
+
// 4. Unrecognized — log + surface raw, and never retryable: an unrecognized
|
|
247
|
+
// prefix is not affirmatively known to be a shape/addressing mistake, so
|
|
248
|
+
// the same "fails closed" discipline the taxonomy applies elsewhere
|
|
249
|
+
// applies here too. (The null/non-object submission reason now carries
|
|
250
|
+
// the typed `submission-malformed:` prefix → 'operational', so it no
|
|
251
|
+
// longer falls through here; see verify-B-003.)
|
|
252
|
+
return { class: 'unknown', prefix: null, detail: reason, retryable: false };
|
|
155
253
|
}
|