@dogfood-lab/verify 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -17
- package/cli-lint.js +81 -36
- package/cli.js +99 -15
- package/index.js +205 -61
- package/package.json +1 -1
- package/parse-rejection.js +125 -27
- package/validators/f-2a5ddafa-provenance-retry-warn.test.js +226 -0
- package/validators/f-5fd3f832-retry-after-clamp.test.js +96 -0
- package/validators/f-ad98b5ac-required-steps-map-guard.test.js +85 -0
- package/validators/f-dac7e08c-transport-error-operational.test.js +121 -0
- package/validators/f-f50e779b-retry-warn-fallback-guard.test.js +153 -0
- package/validators/lint-scenario.js +141 -0
- package/validators/policy.js +31 -8
- package/validators/provenance-gitlab.test.js +10 -3
- package/validators/provenance-registry.test.js +35 -0
- package/validators/provenance.js +183 -17
- package/validators/repo-binding.js +35 -12
- package/validators/repo-binding.test.js +40 -0
- package/validators/steps.js +116 -9
- package/validators/verdict.js +9 -1
- package/validators/w4-f-8e72d0de-timeout-retry.test.js +118 -0
package/README.md
CHANGED
|
@@ -109,21 +109,23 @@ The verifier emits rejection-reason strings under stable prefixes, each mapping
|
|
|
109
109
|
|
|
110
110
|
Discrimination happens by **class**, surfaced by `parseRejectionReason` (below). Every prefix maps to one of four classes: **submission-bad** (the submitter fixes the payload), **operational** (the verifier/tooling faulted), **ingest** (an ingest-side load fault), or **unknown** (unrecognized prefix).
|
|
111
111
|
|
|
112
|
+
**Retryable** (F-f8952a50, wave 10) is a SEPARATE, narrower per-prefix flag, orthogonal to `class`: may a same-`run_id` resubmission whose ONLY prior rejection carries this prefix still reach an acceptance, once corrected? `packages/ingest/persist.js`'s duplicate guard (`isDuplicate` → `isRetryableRejection`) reads this flag — never `class` or `prefix` directly — to decide whether a stale `_rejected` record blocks a same-`run_id` retry that is now headed to acceptance. Every `operational` / `ingest` / `unknown` prefix is `retryable: false` (an ops fault or an unrecognized signal is never the submitter's to retry past). Within `submission-bad`, the taxonomy splits further — **shape/addressing** prefixes ("we could not even read/place/shape your submission") are retryable; **content-verdict** prefixes ("we read your submission and rendered a verdict against its own reported content") are not, so a submitter cannot launder a genuinely-bad run into an accepted one by resubmitting different self-reported content under the same `run_id`. The table below marks each `submission-bad` prefix's retryable value explicitly.
|
|
113
|
+
|
|
112
114
|
**Submission-bad** — `class: 'submission-bad'` (the submitter's payload failed a validator gate; fix the submission and resubmit):
|
|
113
115
|
|
|
114
|
-
| Prefix | Source | Meaning |
|
|
115
|
-
|
|
116
|
-
| `schema:` | `validators/schema.js` | JSON Schema check on the submission/record envelope failed. The rest of the string carries the AJV path + message. |
|
|
117
|
-
| `policy:` | `validators/policy.js` | Per-repo policy gate failed (forbidden tags, missing required fields, surface evidence/CI requirements, or a declarative `when`/`custom_rules` predicate matched — see the [policy DSL](https://dogfood-lab.github.io/testing-os/handbook/policy-dsl/)). |
|
|
118
|
-
| `policy-config:` | `validators/policy.js` | **VERIFY-F1.** A REPO custom-rule predicate hit an eval-time semantic fault the schema could not catch — an unknown leading field, a numeric operator against a non-number, or a depth/width/fan-out budget. The repo authored the bad
|
|
119
|
-
| `steps[<id>]:` | `validators/steps.js` | Step-level contract check failed on a specific step id (gate accumulation, ordering, evidence shape). |
|
|
120
|
-
| `provenance:` | `validators/provenance.js` | The run was genuinely **absent / not confirmable** — a 404 from the provider API, or the run head did not match the submitted commit/repo. The
|
|
121
|
-
| `repo:` | `index.js` cross-field guard | `submission.repo` does not match the owner/repo encoded in `source.run_url` (anti-forgery guard). Emitted as `repo:mismatch: …`. |
|
|
122
|
-
| `submission-contains-verifier-field:` | `index.js` | The submission carried a verifier-owned field (`policy_version`, `verification`, or an object `overall_verdict`) it must not author. |
|
|
123
|
-
| `
|
|
124
|
-
| `
|
|
125
|
-
|
|
126
|
-
**Operational** — `class: 'operational'` (the validator itself threw an internal error; investigate the verifier, do NOT bounce to the submitter):
|
|
116
|
+
| Prefix | Retryable | Source | Meaning |
|
|
117
|
+
|---|---|---|---|
|
|
118
|
+
| `schema:` | Yes — shape | `validators/schema.js` | JSON Schema check on the submission/record envelope failed. The rest of the string carries the AJV path + message. |
|
|
119
|
+
| `policy:` | **No — content verdict** | `validators/policy.js` | Per-repo policy gate failed (forbidden tags, missing required fields, surface evidence/CI requirements, or a declarative `when`/`custom_rules` predicate matched — see the [policy DSL](https://dogfood-lab.github.io/testing-os/handbook/policy-dsl/)). Judges the run's OWN reported content (tags, evidence, scenario results); consuming the run_id is the deliberate anti-gaming behavior. |
|
|
120
|
+
| `policy-config:` | Yes — shape | `validators/policy.js` | **VERIFY-F1.** A REPO custom-rule predicate hit an eval-time semantic fault the schema could not catch — an unknown leading field, a numeric operator against a non-number, or a depth/width/fan-out budget. The repo authored the bad RULE (config), not the run's content, so the fix belongs to the submitter and is retryable. (A malformed GLOBAL predicate is `VALIDATOR_FAULT_POLICY:` operational instead — see below.) |
|
|
121
|
+
| `steps[<id>]:` | Yes — shape | `validators/steps.js` | Step-level contract check failed on a specific step id (gate accumulation, ordering, evidence shape — a completeness/structure mismatch, not a verdict on whether the steps passed). |
|
|
122
|
+
| `provenance:` | **No — content verdict** | `validators/provenance.js` | The run was genuinely **absent / not confirmable** — a 404 from the provider API, or the run head did not match the submitted commit/repo. The provider could not confirm THIS specific run happened as claimed; resubmitting different self-reported content under the same run_id to "become confirmable" is exactly the laundering the anti-gaming doctrine blocks. (Operational provider faults — 429/5xx/401/403 — are NOT this class; see `provenance-fault:` below.) |
|
|
123
|
+
| `repo:` | Yes — shape | `index.js` cross-field guard | `submission.repo` does not match the owner/repo encoded in `source.run_url` (anti-forgery guard). Emitted as `repo:mismatch: …`. Pure identity/addressing — the run happened, only the repo/run_url pairing was mis-stated. |
|
|
124
|
+
| `submission-contains-verifier-field:` | Yes — shape | `index.js` | The submission carried a verifier-owned field (`policy_version`, `verification`, or an object `overall_verdict`) it must not author. |
|
|
125
|
+
| `CONTRACT_SCHEMA_TOO_OLD:` | Yes — shape | `validators/schema-version.js` | The submission's `schema_version` declares a MAJOR **below** the supported floor. **The submitter must re-emit** against the current contract. A patch/minor delta inside the supported major range is NOT rejected. (NOT symmetric with `CONTRACT_SCHEMA_TOO_NEW:` below — see that row for why.) |
|
|
126
|
+
| `unsafe-record-path:` | Yes — shape | `packages/ingest/run.js`, `writeRecord()` catch | The record passed schema validation but `computeRecordPath()`'s traversal guard (`isUnsafeSegment`, stricter than the schema's `repo` pattern — e.g. `../etc`) still refused to place it on disk. The submitter's own `repo` string is the problem; nothing is persisted (there is no safe path to write to). |
|
|
127
|
+
|
|
128
|
+
**Operational** — `class: 'operational'`, always `retryable: false` (the validator itself threw an internal error; investigate the verifier, do NOT bounce to the submitter):
|
|
127
129
|
|
|
128
130
|
| Prefix | Source | Meaning |
|
|
129
131
|
|---|---|---|
|
|
@@ -131,26 +133,28 @@ Discrimination happens by **class**, surfaced by `parseRejectionReason` (below).
|
|
|
131
133
|
| `VALIDATOR_FAULT_POLICY:` | `runValidator('policy', …)` catch | Internal exception inside the policy validator — including a **GLOBAL** declarative-rule predicate fault (VERIFY-F1): a broken `policies/global-policy.yaml` is an ops incident (the studio's own config), so its predicate fault throws here rather than bouncing to the submitter. The repo-authored counterpart is `policy-config:` submission-bad. |
|
|
132
134
|
| `VALIDATOR_FAULT_STEPS:` | `runValidator('steps', …)` catch | Internal exception inside the steps validator. |
|
|
133
135
|
| `VALIDATOR_FAULT_CONTRACT_SCHEMA_VERSION:` | `runValidator('contract_schema_version', …)` catch | The version gate was called with an unknown contract key (a programmer error at the call site, not a submission fault). |
|
|
136
|
+
| `CONTRACT_SCHEMA_TOO_NEW:` | `validators/schema-version.js` | **F-be0deacd (wave 20).** The submission's `schema_version` declares a MAJOR **above** what this build supports (see `SUPPORTED_SCHEMA_VERSIONS` in `@dogfood-lab/schemas`) — THIS BUILD is behind a schema major its own submitters have already adopted. No resubmission, corrected or not, can ever satisfy a `major > maxMajor` comparison until testing-os itself ships an upgrade — **the operator must upgrade testing-os**. Page ops; do NOT bounce it back to the submitter. Unlike every other row in this table, `validators/schema-version.js` RETURNS this as an ordinary rejection string rather than throwing, so (unlike the `VALIDATOR_FAULT_*`/`provenance-fault:`/`scenario-fetch-fault:` rows) it genuinely persists to `records/_rejected/` and is reachable by `packages/ingest/persist.js`'s `isRetryableRejection()` — **F-51780da9 (wave 22):** rather than trusting the frozen `retryable: false` this prefix's own classification carries, `isRetryableRejection()` re-derives retryability for this ONE prefix against the CURRENT build's `SUPPORTED_SCHEMA_VERSIONS` ceiling, so a stale TOO_NEW rejection unblocks once the operator upgrades testing-os past the declared major. Distinct from the submission-bad `CONTRACT_SCHEMA_TOO_OLD:` above — the two prefixes are asymmetric despite the shared emitter. |
|
|
134
137
|
| `submission-malformed:` | `index.js` null/non-object early-return | The submission itself was `null` or not an object — a malfunctioning **dispatcher** sent garbage, not a submitter who authored a bad-but-shaped payload. Page ops / inspect the dispatch pipeline; do NOT bounce it to a submitter. |
|
|
135
138
|
| `provenance-fault:` | `index.js` provenance catch | The provenance adapter THREW an operational error confirming the run — a provider **429 rate-limit, 5xx outage, or 401/403 token** fault (`validators/provenance.js` throws these on purpose for non-404 responses). The submitter's payload is fine; the verifier could not reach a verdict. Page ops / retry; do NOT bounce it to a submitter. Distinct from the submission-bad `provenance:` (genuine absence/404). |
|
|
139
|
+
| `scenario-fetch-fault:` | `packages/ingest/load-context.js` | The scenario fetcher THREW after exhausting its retry budget (**5xx/429 outage, transport reject**) or hit a **401/403 credential** fault loading a scenario definition. The submission may be perfectly good — the fetch infrastructure faulted. The ingest CLI lets this propagate (exit 2, nothing persisted); a true missing file is the ingest-class `scenario-load: … (reason: not_found)` instead. |
|
|
136
140
|
|
|
137
141
|
Any future `VALIDATOR_FAULT_<NEW>:` prefix is classified `operational` by family — `parseRejectionReason` matches the `VALIDATOR_FAULT_` head, so a new validator class needs no parser edit. The `submission-malformed:` prefix is matched literally (it is not part of the `VALIDATOR_FAULT_` family).
|
|
138
142
|
|
|
139
|
-
**Ingest** — `class: 'ingest'` (an ingest-side load fault, not a verifier gate):
|
|
143
|
+
**Ingest** — `class: 'ingest'`, always `retryable: false` (an ingest-side load fault, not a verifier gate):
|
|
140
144
|
|
|
141
145
|
| Prefix | Source | Meaning |
|
|
142
146
|
|---|---|---|
|
|
143
|
-
| `scenario-load:` | `packages/ingest/run.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id`). |
|
|
147
|
+
| `scenario-load:` | `packages/ingest/run.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id` / `too_large` / `schema_invalid`). Outages and credential faults are NOT this class — they throw `scenario-fetch-fault:` (operational, above) instead of rejecting the submission. |
|
|
144
148
|
|
|
145
149
|
### Operator hygiene
|
|
146
150
|
|
|
147
|
-
Discriminate by **class**, not by hand-rolled `.startsWith()` chains. `parseRejectionReason(reason)` returns `{ class, prefix, detail }`:
|
|
151
|
+
Discriminate by **class**, not by hand-rolled `.startsWith()` chains. `parseRejectionReason(reason)` returns `{ class, prefix, detail, retryable }`:
|
|
148
152
|
|
|
149
153
|
```js
|
|
150
154
|
import { parseRejectionReason } from '@dogfood-lab/verify';
|
|
151
155
|
|
|
152
156
|
for (const r of result.rejection_reasons) {
|
|
153
|
-
const { class: cls, prefix, detail } =
|
|
157
|
+
const { class: cls, prefix, detail, retryable } =
|
|
154
158
|
parseRejectionReason(r);
|
|
155
159
|
switch (cls) {
|
|
156
160
|
case 'operational':
|
package/cli-lint.js
CHANGED
|
@@ -1,18 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* cli-lint.js (VERIFY-F3) — the `dogfood-verify lint <
|
|
2
|
+
* cli-lint.js (VERIFY-F3) — the `dogfood-verify lint <file>` subcommand.
|
|
3
3
|
*
|
|
4
|
-
* A SEPARATE parse/render path from the verify CLI (cli.js): it takes a policy YAML
|
|
5
|
-
* (not a submission JSON
|
|
6
|
-
* verify arg parser or output. cli.js's `main`
|
|
7
|
-
* and leaves the verify `run` path untouched. The
|
|
4
|
+
* A SEPARATE parse/render path from the verify CLI (cli.js): it takes a policy YAML or
|
|
5
|
+
* (with --scenario) a scenario YAML — not a submission JSON — and reports static lint
|
|
6
|
+
* findings, so it does not share the verify arg parser or output. cli.js's `main`
|
|
7
|
+
* dispatcher routes the `lint` verb here and leaves the verify `run` path untouched. The
|
|
8
|
+
* exit contract mirrors that path, identically for both modes:
|
|
8
9
|
*
|
|
9
10
|
* 0 — clean, or warnings-only (footgun advisories never block).
|
|
10
|
-
* 1 — one or more errors (schema-invalid, a static
|
|
11
|
+
* 1 — one or more errors (schema-invalid, a static fault, or unparseable YAML).
|
|
11
12
|
* 2 — operator error (file missing/unreadable, or a malformed invocation).
|
|
12
13
|
*
|
|
13
|
-
* YAML that fails to parse is exit 1 (a lint FINDING about the
|
|
14
|
+
* YAML that fails to parse is exit 1 (a lint FINDING about the file the author must fix —
|
|
14
15
|
* surfacing "line 4: bad indentation" is the lint's job), not exit 2. A file that does not
|
|
15
16
|
* exist is exit 2 (the author pointed at the wrong path). See docs/policy-lint.md.
|
|
17
|
+
*
|
|
18
|
+
* Two modes share the whole render/exit machinery (F-BACKEND-003):
|
|
19
|
+
* - default (policy): lintPolicy(doc, { origin }) → origin global|repo|unknown
|
|
20
|
+
* - --scenario: lintScenario(doc, { file }) → origin 'scenario'
|
|
21
|
+
* Both return the same { ok, origin, errors, warnings, coverageNote } shape, so
|
|
22
|
+
* renderLintText / buildLintJson are reused unchanged.
|
|
16
23
|
*/
|
|
17
24
|
|
|
18
25
|
import { readFileSync } from 'node:fs';
|
|
@@ -20,6 +27,7 @@ import { resolve } from 'node:path';
|
|
|
20
27
|
import yaml from 'js-yaml';
|
|
21
28
|
|
|
22
29
|
import { lintPolicy, COVERAGE_NOTE } from './validators/lint-policy.js';
|
|
30
|
+
import { lintScenario, SCENARIO_COVERAGE_NOTE } from './validators/lint-scenario.js';
|
|
23
31
|
|
|
24
32
|
/** Operator-error sentinel → exit 2 (distinct from a lint finding, which is exit 1). */
|
|
25
33
|
class LintOperatorError extends Error {
|
|
@@ -30,56 +38,75 @@ class LintOperatorError extends Error {
|
|
|
30
38
|
}
|
|
31
39
|
}
|
|
32
40
|
|
|
33
|
-
const LINT_USAGE = `dogfood-verify lint — author-time static check for a policy file
|
|
41
|
+
const LINT_USAGE = `dogfood-verify lint — author-time static check for a policy or scenario file
|
|
34
42
|
|
|
35
|
-
|
|
43
|
+
Usage:
|
|
36
44
|
dogfood-verify lint <policy-file> [--json]
|
|
45
|
+
dogfood-verify lint --scenario <scenario-file> [--json]
|
|
37
46
|
|
|
38
|
-
|
|
47
|
+
What it checks — policy mode (default, no submission needed):
|
|
39
48
|
- structural validity against policy.schema.json
|
|
40
49
|
- every predicate's known leading field, combinator depth, and node budget
|
|
41
50
|
- an ADVISORY warning on the [] footgun (a negative op over a [] path fails open)
|
|
42
51
|
|
|
43
|
-
It CANNOT statically catch a type_mismatch or a fanout_budget overrun — those depend
|
|
44
|
-
on submission data. Run \`dogfood-verify --file <submission> --explain\` for that.
|
|
52
|
+
It CANNOT statically catch a type_mismatch or a fanout_budget overrun — those depend
|
|
53
|
+
on submission data. Run \`dogfood-verify --file <submission> --explain\` for that.
|
|
54
|
+
|
|
55
|
+
What it checks — --scenario mode (no submission needed):
|
|
56
|
+
- structural validity against scenario.schema.json
|
|
57
|
+
- every success_criteria.required_steps entry references a declared steps[].id
|
|
58
|
+
- step ids are unique
|
|
59
|
+
- an ADVISORY warning when the file basename does not match scenario_id (the
|
|
60
|
+
receiver fetches dogfood/scenarios/<scenario_id>.yaml, so a mismatch makes the
|
|
61
|
+
committed definition unreachable and required-steps enforcement fails open)
|
|
45
62
|
|
|
46
|
-
|
|
63
|
+
It CANNOT verify that a real submission's step_results satisfy required_steps, nor
|
|
64
|
+
that the receiver can fetch the file at the attested commit — run a real ingest.
|
|
65
|
+
|
|
66
|
+
Options:
|
|
67
|
+
--scenario Lint the file as a scenario definition (default: policy).
|
|
47
68
|
--json Machine-readable result for CI.
|
|
48
69
|
-h, --help Show this help.
|
|
49
70
|
|
|
50
|
-
|
|
71
|
+
Exit codes:
|
|
51
72
|
0 clean or warnings-only 1 errors found 2 operator error (bad flags / IO)`;
|
|
52
73
|
|
|
53
74
|
/**
|
|
54
75
|
* Parse the lint argv (everything AFTER the `lint` verb). Accepts exactly one positional
|
|
55
|
-
*
|
|
56
|
-
* on any malformed invocation.
|
|
76
|
+
* file path plus optional `--scenario` / `--json` / `--help`. Throws LintOperatorError
|
|
77
|
+
* (→ exit 2) on any malformed invocation.
|
|
78
|
+
*
|
|
79
|
+
* `--scenario` is a boolean MODE flag (default: policy mode). It selects which linter runs
|
|
80
|
+
* over the one positional file; it never consumes the path itself.
|
|
57
81
|
*
|
|
58
82
|
* @param {string[]} argv
|
|
59
|
-
* @returns {{ help: boolean, file: string|null, json: boolean }}
|
|
83
|
+
* @returns {{ help: boolean, file: string|null, json: boolean, scenario: boolean }}
|
|
60
84
|
*/
|
|
61
85
|
export function parseLintArgs(argv) {
|
|
62
86
|
let file = null;
|
|
63
87
|
let json = false;
|
|
64
88
|
let help = false;
|
|
89
|
+
let scenario = false;
|
|
65
90
|
|
|
66
91
|
for (const arg of argv) {
|
|
67
92
|
if (arg === '-h' || arg === '--help') { help = true; continue; }
|
|
68
93
|
if (arg === '--json') { json = true; continue; }
|
|
94
|
+
if (arg === '--scenario') { scenario = true; continue; }
|
|
69
95
|
if (arg.startsWith('-')) {
|
|
70
96
|
throw new LintOperatorError(`unknown argument: ${arg}`, 'run `dogfood-verify lint --help` for usage');
|
|
71
97
|
}
|
|
72
98
|
if (file !== null) {
|
|
73
|
-
throw new LintOperatorError('more than one
|
|
99
|
+
throw new LintOperatorError('more than one file given', 'lint one file at a time');
|
|
74
100
|
}
|
|
75
101
|
file = arg;
|
|
76
102
|
}
|
|
77
103
|
|
|
78
|
-
if (help) return { help: true, file: null, json: false };
|
|
104
|
+
if (help) return { help: true, file: null, json: false, scenario: false };
|
|
79
105
|
if (file === null) {
|
|
80
|
-
|
|
106
|
+
const usage = scenario ? 'dogfood-verify lint --scenario <scenario-file>' : 'dogfood-verify lint <policy-file>';
|
|
107
|
+
throw new LintOperatorError(`no ${scenario ? 'scenario' : 'policy'} file provided`, usage);
|
|
81
108
|
}
|
|
82
|
-
return { help: false, file, json };
|
|
109
|
+
return { help: false, file, json, scenario };
|
|
83
110
|
}
|
|
84
111
|
|
|
85
112
|
/**
|
|
@@ -172,11 +199,12 @@ export async function runLint(argv, io = {}) {
|
|
|
172
199
|
}
|
|
173
200
|
|
|
174
201
|
const path = resolve(opts.file);
|
|
202
|
+
const kind = opts.scenario ? 'scenario' : 'policy';
|
|
175
203
|
let raw;
|
|
176
204
|
try {
|
|
177
205
|
raw = readFileSync(path, 'utf-8');
|
|
178
206
|
} catch (e) {
|
|
179
|
-
err(`ERROR: could not read
|
|
207
|
+
err(`ERROR: could not read ${kind} file: ${path} — ${e.message}`);
|
|
180
208
|
err(' hint: check the path exists and is readable');
|
|
181
209
|
return 2;
|
|
182
210
|
}
|
|
@@ -185,25 +213,42 @@ export async function runLint(argv, io = {}) {
|
|
|
185
213
|
try {
|
|
186
214
|
doc = yaml.load(raw);
|
|
187
215
|
} catch (e) {
|
|
188
|
-
// A YAML parse failure is a lint finding about the
|
|
216
|
+
// A YAML parse failure is a lint finding about the file (exit 1), not an operator error.
|
|
217
|
+
// Mirrors the policy path exactly, only differing in the origin/label/coverageNote so the
|
|
218
|
+
// scenario report reads as a scenario report.
|
|
189
219
|
const where = e && e.mark ? ` at line ${e.mark.line + 1}, column ${e.mark.column + 1}` : '';
|
|
190
|
-
const result =
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
220
|
+
const result = opts.scenario
|
|
221
|
+
? {
|
|
222
|
+
ok: false,
|
|
223
|
+
origin: 'scenario',
|
|
224
|
+
errors: [{
|
|
225
|
+
label: 'scenario-schema:',
|
|
226
|
+
code: 'yaml_parse',
|
|
227
|
+
location: '/',
|
|
228
|
+
message: `scenario YAML failed to parse${where} — ${e.message}`,
|
|
229
|
+
}],
|
|
230
|
+
warnings: [],
|
|
231
|
+
coverageNote: SCENARIO_COVERAGE_NOTE,
|
|
232
|
+
}
|
|
233
|
+
: {
|
|
234
|
+
ok: false,
|
|
235
|
+
origin: originForPath(path),
|
|
236
|
+
errors: [{
|
|
237
|
+
label: 'policy-schema:',
|
|
238
|
+
code: 'yaml_parse',
|
|
239
|
+
location: '/',
|
|
240
|
+
message: `policy YAML failed to parse${where} — ${e.message}`,
|
|
241
|
+
}],
|
|
242
|
+
warnings: [],
|
|
243
|
+
coverageNote: COVERAGE_NOTE,
|
|
244
|
+
};
|
|
202
245
|
out(opts.json ? JSON.stringify(buildLintJson(result, path)) : renderLintText(result, path));
|
|
203
246
|
return 1;
|
|
204
247
|
}
|
|
205
248
|
|
|
206
|
-
const result =
|
|
249
|
+
const result = opts.scenario
|
|
250
|
+
? lintScenario(doc, { file: path })
|
|
251
|
+
: lintPolicy(doc, { origin: originForPath(path) });
|
|
207
252
|
out(opts.json ? JSON.stringify(buildLintJson(result, path)) : renderLintText(result, path));
|
|
208
253
|
return result.ok ? 0 : 1;
|
|
209
254
|
}
|
package/cli.js
CHANGED
|
@@ -24,6 +24,15 @@
|
|
|
24
24
|
* (0 accepted / 1 rejected / 2 operator error) so a wrapper can reason about
|
|
25
25
|
* both uniformly.
|
|
26
26
|
*
|
|
27
|
+
* KNOWN PREVIEW GAP (V2-CONTRACT-004, documented — wiring deferred to a
|
|
28
|
+
* feature pass): this CLI loads NO scenario definitions, so
|
|
29
|
+
* success_criteria.required_steps enforcement (F-3bfc2885) runs only in
|
|
30
|
+
* production ingest, which fetches scenarios from the source repo at the
|
|
31
|
+
* persisted commit. A preview VERDICT: ACCEPTED therefore does not cover
|
|
32
|
+
* required_steps — the same discipline run.js uses to document the GitLab
|
|
33
|
+
* scenario-fetcher gap. Both --help and every --explain rendering carry the
|
|
34
|
+
* note so no consumer can read a preview verdict as covering it.
|
|
35
|
+
*
|
|
27
36
|
* Exit codes (consistent with packages/ingest/run.js):
|
|
28
37
|
* 0 — submission accepted
|
|
29
38
|
* 1 — submission rejected (verdict reached; the payload is the problem)
|
|
@@ -32,11 +41,12 @@
|
|
|
32
41
|
* submission.
|
|
33
42
|
*/
|
|
34
43
|
|
|
35
|
-
import { readFileSync } from 'node:fs';
|
|
44
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
36
45
|
import { resolve, dirname, join } from 'node:path';
|
|
37
46
|
import { fileURLToPath } from 'node:url';
|
|
38
47
|
import yaml from 'js-yaml';
|
|
39
48
|
|
|
49
|
+
import { validatePayload } from '@dogfood-lab/schemas';
|
|
40
50
|
import { verify, parseRejectionReason } from './index.js';
|
|
41
51
|
import { stubProvenance, provenanceForProvider } from './validators/provenance.js';
|
|
42
52
|
import { runLint } from './cli-lint.js';
|
|
@@ -57,23 +67,38 @@ class OperatorError extends Error {
|
|
|
57
67
|
}
|
|
58
68
|
}
|
|
59
69
|
|
|
70
|
+
/**
|
|
71
|
+
* V2-CONTRACT-004: appended to EVERY --explain rendering (accepted and
|
|
72
|
+
* rejected alike) so a preview verdict can never be read as covering the
|
|
73
|
+
* required_steps gate, which only production ingest enforces (it fetches the
|
|
74
|
+
* scenario definitions this preview does not have).
|
|
75
|
+
*
|
|
76
|
+
* Declared ahead of USAGE and interpolated directly into its own "Not checked
|
|
77
|
+
* in preview" section below (F-e0c3fa21 — that section used to restate the
|
|
78
|
+
* header as its own redundant lowercase lead-in, worded slightly differently
|
|
79
|
+
* than this constant) so --help and --explain describe the identical gap
|
|
80
|
+
* from one shared string and can never drift apart again.
|
|
81
|
+
*/
|
|
82
|
+
const PREVIEW_GAP_NOTE =
|
|
83
|
+
'Not checked in preview: required_steps (needs scenario definitions — enforced only by real ingest).';
|
|
84
|
+
|
|
60
85
|
const USAGE = `verify — local dry-run / explain for dogfood submissions
|
|
61
86
|
|
|
62
|
-
|
|
87
|
+
Usage:
|
|
63
88
|
verify --file <path> [--explain | --json] [--provenance=stub|github]
|
|
64
89
|
verify --payload '<json>' [--explain | --json] [--provenance=stub|github]
|
|
65
90
|
|
|
66
|
-
|
|
91
|
+
Input (exactly one required):
|
|
67
92
|
--file <path> Read the submission JSON from a file.
|
|
68
93
|
--payload <json> Pass the submission JSON inline.
|
|
69
94
|
|
|
70
|
-
|
|
95
|
+
Output mode (default: --explain):
|
|
71
96
|
--explain Human-readable verdict breakdown with each rejection
|
|
72
97
|
reason classified (who must fix it). [default]
|
|
73
98
|
--json Machine-readable result for tooling. Mutually exclusive
|
|
74
99
|
with --explain.
|
|
75
100
|
|
|
76
|
-
|
|
101
|
+
Provenance (default: stub):
|
|
77
102
|
--provenance=stub No-network local check; provenance is always confirmed.
|
|
78
103
|
This is a LOCAL DRY-RUN — a real ingest re-checks
|
|
79
104
|
provenance against the source run. [default]
|
|
@@ -82,7 +107,11 @@ PROVENANCE (default: stub):
|
|
|
82
107
|
|
|
83
108
|
-h, --help Show this help.
|
|
84
109
|
|
|
85
|
-
|
|
110
|
+
${PREVIEW_GAP_NOTE}
|
|
111
|
+
Scenario definitions live in the source repo; only a real ingest fetches
|
|
112
|
+
them and enforces success_criteria.required_steps.
|
|
113
|
+
|
|
114
|
+
Exit codes:
|
|
86
115
|
0 accepted 1 rejected 2 operator error (bad flags / IO / JSON)`;
|
|
87
116
|
|
|
88
117
|
/**
|
|
@@ -113,7 +142,12 @@ export function parseArgs(argv) {
|
|
|
113
142
|
arg = arg.slice(0, eq);
|
|
114
143
|
}
|
|
115
144
|
}
|
|
116
|
-
|
|
145
|
+
// F-b4dbdc52: a following token that is itself a flag is NOT a value —
|
|
146
|
+
// `verify --file --json` must hit the '--file requires a path' operator
|
|
147
|
+
// error, not consume '--json' as the path. Same guard as run.js
|
|
148
|
+
// (f-ingest-003), which this parser mirrors.
|
|
149
|
+
const nextIsValue = argv[i + 1] !== undefined && !argv[i + 1].startsWith('--');
|
|
150
|
+
const hasValue = inlineValue !== null || nextIsValue;
|
|
117
151
|
const takeValue = () => (inlineValue !== null ? inlineValue : argv[++i]);
|
|
118
152
|
|
|
119
153
|
switch (arg) {
|
|
@@ -225,6 +259,17 @@ function loadSubmission({ file, payload }) {
|
|
|
225
259
|
* @param {string} repoRoot
|
|
226
260
|
* @returns {{ globalPolicy: object, repoPolicy: object|null, policyVersion: string }}
|
|
227
261
|
*/
|
|
262
|
+
/**
|
|
263
|
+
* Collapse validatePayload errors into the same first-3 single-line summary
|
|
264
|
+
* production's loadGlobalPolicy uses (D2B-005), so preview and production
|
|
265
|
+
* name the offending YAML key identically.
|
|
266
|
+
*/
|
|
267
|
+
function summarizePolicySchemaErrors(errors) {
|
|
268
|
+
const trimmed = errors.slice(0, 3).map(e => `${e.path || '/'} ${e.message}`);
|
|
269
|
+
const ellipsis = errors.length > 3 ? `; (+${errors.length - 3} more)` : '';
|
|
270
|
+
return trimmed.join('; ') + ellipsis;
|
|
271
|
+
}
|
|
272
|
+
|
|
228
273
|
function loadPolicies(submission, repoRoot) {
|
|
229
274
|
const globalPath = join(repoRoot, 'policies', 'global-policy.yaml');
|
|
230
275
|
let globalPolicy;
|
|
@@ -235,22 +280,57 @@ function loadPolicies(submission, repoRoot) {
|
|
|
235
280
|
'run from the testing-os repo root, or set VERIFY_REPO_ROOT to it');
|
|
236
281
|
}
|
|
237
282
|
|
|
283
|
+
// F-99aa42bc: mirror production loadGlobalPolicy's fail-loud schema gate.
|
|
284
|
+
// Pre-fix the preview applied a parses-but-schema-invalid (or null/empty)
|
|
285
|
+
// global policy as-is — production ingest would refuse the same file, and
|
|
286
|
+
// a null policy surfaced downstream as a confusing VALIDATOR_FAULT_POLICY.
|
|
287
|
+
// Same divergence class F-65d4d6dd closed for the repo-policy half.
|
|
288
|
+
const globalValidation = validatePayload('policy', globalPolicy);
|
|
289
|
+
if (!globalValidation.valid) {
|
|
290
|
+
throw new OperatorError(
|
|
291
|
+
`global policy schema-invalid: ${globalPath} — ${summarizePolicySchemaErrors(globalValidation.errors)}`,
|
|
292
|
+
'fix the policy to conform to policy.schema.json — production ingest refuses this file too'
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
|
|
238
296
|
let repoPolicy = null;
|
|
239
297
|
const repoSlug = submission && typeof submission === 'object' ? submission.repo : null;
|
|
240
298
|
if (typeof repoSlug === 'string' && repoSlug.includes('/')) {
|
|
241
|
-
|
|
299
|
+
// F-54e5fde7: two-segment contract only (nested GitLab subgroups are
|
|
300
|
+
// unsupported by the submission schema). A 3+-segment slug fails closed —
|
|
301
|
+
// destructuring would silently drop the tail and look up the WRONG policy.
|
|
302
|
+
const segments = repoSlug.split('/');
|
|
303
|
+
const [org, repo] = segments.length === 2 ? segments : [null, null];
|
|
242
304
|
// Reject path-traversal segments before touching the filesystem — a hostile
|
|
243
305
|
// submission.repo like '../../etc' must never escape policies/repos/.
|
|
244
306
|
const safe = (s) => typeof s === 'string' && s.length > 0 && !s.includes('..') && !s.includes('\\') && s !== '.';
|
|
245
307
|
if (safe(org) && safe(repo)) {
|
|
246
308
|
const repoPath = join(repoRoot, 'policies', 'repos', org, `${repo}.yaml`);
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
309
|
+
// F-65d4d6dd: mirror ingest's loadRepoPolicy contract exactly so the
|
|
310
|
+
// preview can never green-light a submission production will reject:
|
|
311
|
+
// - absent file → null (defaults apply)
|
|
312
|
+
// - YAML parse failure → `__torn` sentinel (verify() rejects with
|
|
313
|
+
// `policy: repo policy unreadable — …`)
|
|
314
|
+
// - parses, schema-invalid (D2B-005 class) → `__torn` sentinel too
|
|
315
|
+
// Pre-fix, a parses-but-schema-invalid policy was silently applied as-is
|
|
316
|
+
// and a bad-YAML policy silently became null — both preview/production
|
|
317
|
+
// divergences.
|
|
318
|
+
if (existsSync(repoPath)) {
|
|
319
|
+
try {
|
|
320
|
+
repoPolicy = yaml.load(readFileSync(repoPath, 'utf-8'));
|
|
321
|
+
} catch (e) {
|
|
322
|
+
repoPolicy = { __torn: true, reason: e && e.message ? e.message : String(e), path: repoPath };
|
|
323
|
+
}
|
|
324
|
+
if (repoPolicy && repoPolicy.__torn !== true) {
|
|
325
|
+
const validation = validatePayload('policy', repoPolicy);
|
|
326
|
+
if (!validation.valid) {
|
|
327
|
+
repoPolicy = {
|
|
328
|
+
__torn: true,
|
|
329
|
+
reason: `schema-invalid — ${summarizePolicySchemaErrors(validation.errors)}`,
|
|
330
|
+
path: repoPath
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
}
|
|
254
334
|
}
|
|
255
335
|
}
|
|
256
336
|
}
|
|
@@ -334,6 +414,8 @@ export function renderExplain(record) {
|
|
|
334
414
|
if (accepted) {
|
|
335
415
|
lines.push('');
|
|
336
416
|
lines.push('No rejection reasons. This submission would be accepted.');
|
|
417
|
+
lines.push('');
|
|
418
|
+
lines.push(PREVIEW_GAP_NOTE);
|
|
337
419
|
return lines.join('\n');
|
|
338
420
|
}
|
|
339
421
|
|
|
@@ -361,6 +443,8 @@ export function renderExplain(record) {
|
|
|
361
443
|
lines.push(` - ${prefix}${parsed.detail}`);
|
|
362
444
|
}
|
|
363
445
|
}
|
|
446
|
+
lines.push('');
|
|
447
|
+
lines.push(PREVIEW_GAP_NOTE);
|
|
364
448
|
return lines.join('\n');
|
|
365
449
|
}
|
|
366
450
|
|