@dogfood-lab/verify 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -1
- package/cli-lint.js +254 -0
- package/cli.js +115 -16
- package/index.js +95 -61
- package/package.json +3 -1
- package/parse-rejection.js +10 -0
- package/validators/f-5fd3f832-retry-after-clamp.test.js +96 -0
- package/validators/f-ad98b5ac-required-steps-map-guard.test.js +85 -0
- package/validators/f-dac7e08c-transport-error-operational.test.js +121 -0
- package/validators/lint-policy.js +120 -0
- package/validators/lint-scenario.js +141 -0
- package/validators/policy.js +9 -1
- package/validators/predicate.js +163 -3
- package/validators/provenance-gitlab.test.js +10 -3
- package/validators/provenance.js +60 -13
- package/validators/repo-binding.js +18 -10
- package/validators/steps.js +39 -9
- package/validators/w4-f-8e72d0de-timeout-retry.test.js +118 -0
package/README.md
CHANGED
|
@@ -62,6 +62,28 @@ import { validateSchema } from '@dogfood-lab/verify/validators/schema.js';
|
|
|
62
62
|
import { validateProvenance } from '@dogfood-lab/verify/validators/provenance.js';
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
+
## CLI (`dogfood-verify`)
|
|
66
|
+
|
|
67
|
+
The package ships a `dogfood-verify` bin with two verbs.
|
|
68
|
+
|
|
69
|
+
**Verify** a submission (local dry-run / explain — never writes):
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
dogfood-verify --file submission.json --explain # human verdict, reasons classified by who-fixes-it
|
|
73
|
+
dogfood-verify --file submission.json --json # machine-readable
|
|
74
|
+
# exit: 0 accepted · 1 rejected · 2 operator error
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
**Lint** a policy file (VERIFY-F3, author-time — no submission needed):
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
dogfood-verify lint policies/repos/<org>/<repo>.yaml # or global-policy.yaml
|
|
81
|
+
dogfood-verify lint policies/global-policy.yaml --json # for CI
|
|
82
|
+
# exit: 0 clean or warnings-only · 1 errors · 2 operator error
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`lint` runs the structural schema gate **plus** the data-independent predicate checks (`unknown_field`, `max_depth`, `node_budget`) over every `when` predicate, and emits an **advisory** warning on the `[]` footgun (a negative operator over a `[]` path fails open) with the fail-closed `not(any(...))` rewrite as a suggestion — never auto-applied, never a hard error. It is the `opa check` analogue. It **cannot** statically catch a `type_mismatch` or a fan-out overrun (both are data-dependent) and says so. Full contract + coverage boundary: [`docs/policy-lint.md`](https://github.com/dogfood-lab/testing-os/blob/main/docs/policy-lint.md).
|
|
86
|
+
|
|
65
87
|
## Submission envelope
|
|
66
88
|
|
|
67
89
|
The full envelope shape is defined by `@dogfood-lab/schemas` (`dogfood-record-submission.schema.json`). Minimum required fields:
|
|
@@ -111,6 +133,7 @@ Discrimination happens by **class**, surfaced by `parseRejectionReason` (below).
|
|
|
111
133
|
| `VALIDATOR_FAULT_CONTRACT_SCHEMA_VERSION:` | `runValidator('contract_schema_version', …)` catch | The version gate was called with an unknown contract key (a programmer error at the call site, not a submission fault). |
|
|
112
134
|
| `submission-malformed:` | `index.js` null/non-object early-return | The submission itself was `null` or not an object — a malfunctioning **dispatcher** sent garbage, not a submitter who authored a bad-but-shaped payload. Page ops / inspect the dispatch pipeline; do NOT bounce it to a submitter. |
|
|
113
135
|
| `provenance-fault:` | `index.js` provenance catch | The provenance adapter THREW an operational error confirming the run — a provider **429 rate-limit, 5xx outage, or 401/403 token** fault (`validators/provenance.js` throws these on purpose for non-404 responses). The submitter's payload is fine; the verifier could not reach a verdict. Page ops / retry; do NOT bounce it to a submitter. Distinct from the submission-bad `provenance:` (genuine absence/404). |
|
|
136
|
+
| `scenario-fetch-fault:` | `packages/ingest/load-context.js` | The scenario fetcher THREW after exhausting its retry budget (**5xx/429 outage, transport reject**) or hit a **401/403 credential** fault loading a scenario definition. The submission may be perfectly good — the fetch infrastructure faulted. The ingest CLI lets this propagate (exit 2, nothing persisted); a true missing file is the ingest-class `scenario-load: … (reason: not_found)` instead. |
|
|
114
137
|
|
|
115
138
|
Any future `VALIDATOR_FAULT_<NEW>:` prefix is classified `operational` by family — `parseRejectionReason` matches the `VALIDATOR_FAULT_` head, so a new validator class needs no parser edit. The `submission-malformed:` prefix is matched literally (it is not part of the `VALIDATOR_FAULT_` family).
|
|
116
139
|
|
|
@@ -118,7 +141,7 @@ Any future `VALIDATOR_FAULT_<NEW>:` prefix is classified `operational` by family
|
|
|
118
141
|
|
|
119
142
|
| Prefix | Source | Meaning |
|
|
120
143
|
|---|---|---|
|
|
121
|
-
| `scenario-load:` | `packages/ingest/run.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id`). |
|
|
144
|
+
| `scenario-load:` | `packages/ingest/run.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id` / `too_large` / `schema_invalid`). Outages and credential faults are NOT this class — they throw `scenario-fetch-fault:` (operational, above) instead of rejecting the submission. |
|
|
122
145
|
|
|
123
146
|
### Operator hygiene
|
|
124
147
|
|
package/cli-lint.js
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cli-lint.js (VERIFY-F3) — the `dogfood-verify lint <file>` subcommand.
|
|
3
|
+
*
|
|
4
|
+
* A SEPARATE parse/render path from the verify CLI (cli.js): it takes a policy YAML or
|
|
5
|
+
* (with --scenario) a scenario YAML — not a submission JSON — and reports static lint
|
|
6
|
+
* findings, so it does not share the verify arg parser or output. cli.js's `main`
|
|
7
|
+
* dispatcher routes the `lint` verb here and leaves the verify `run` path untouched. The
|
|
8
|
+
* exit contract mirrors that path, identically for both modes:
|
|
9
|
+
*
|
|
10
|
+
* 0 — clean, or warnings-only (footgun advisories never block).
|
|
11
|
+
* 1 — one or more errors (schema-invalid, a static fault, or unparseable YAML).
|
|
12
|
+
* 2 — operator error (file missing/unreadable, or a malformed invocation).
|
|
13
|
+
*
|
|
14
|
+
* YAML that fails to parse is exit 1 (a lint FINDING about the file the author must fix —
|
|
15
|
+
* surfacing "line 4: bad indentation" is the lint's job), not exit 2. A file that does not
|
|
16
|
+
* exist is exit 2 (the author pointed at the wrong path). See docs/policy-lint.md.
|
|
17
|
+
*
|
|
18
|
+
* Two modes share the whole render/exit machinery (F-BACKEND-003):
|
|
19
|
+
* - default (policy): lintPolicy(doc, { origin }) → origin global|repo|unknown
|
|
20
|
+
* - --scenario: lintScenario(doc, { file }) → origin 'scenario'
|
|
21
|
+
* Both return the same { ok, origin, errors, warnings, coverageNote } shape, so
|
|
22
|
+
* renderLintText / buildLintJson are reused unchanged.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { readFileSync } from 'node:fs';
|
|
26
|
+
import { resolve } from 'node:path';
|
|
27
|
+
import yaml from 'js-yaml';
|
|
28
|
+
|
|
29
|
+
import { lintPolicy, COVERAGE_NOTE } from './validators/lint-policy.js';
|
|
30
|
+
import { lintScenario, SCENARIO_COVERAGE_NOTE } from './validators/lint-scenario.js';
|
|
31
|
+
|
|
32
|
+
/** Operator-error sentinel → exit 2 (distinct from a lint finding, which is exit 1). */
|
|
33
|
+
class LintOperatorError extends Error {
|
|
34
|
+
constructor(message, hint) {
|
|
35
|
+
super(message);
|
|
36
|
+
this.name = 'LintOperatorError';
|
|
37
|
+
this.hint = hint;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const LINT_USAGE = `dogfood-verify lint — author-time static check for a policy or scenario file
|
|
42
|
+
|
|
43
|
+
USAGE:
|
|
44
|
+
dogfood-verify lint <policy-file> [--json]
|
|
45
|
+
dogfood-verify lint --scenario <scenario-file> [--json]
|
|
46
|
+
|
|
47
|
+
WHAT IT CHECKS — policy mode (default, no submission needed):
|
|
48
|
+
- structural validity against policy.schema.json
|
|
49
|
+
- every predicate's known leading field, combinator depth, and node budget
|
|
50
|
+
- an ADVISORY warning on the [] footgun (a negative op over a [] path fails open)
|
|
51
|
+
|
|
52
|
+
It CANNOT statically catch a type_mismatch or a fanout_budget overrun — those depend
|
|
53
|
+
on submission data. Run \`dogfood-verify --file <submission> --explain\` for that.
|
|
54
|
+
|
|
55
|
+
WHAT IT CHECKS — --scenario mode (no submission needed):
|
|
56
|
+
- structural validity against scenario.schema.json
|
|
57
|
+
- every success_criteria.required_steps entry references a declared steps[].id
|
|
58
|
+
- step ids are unique
|
|
59
|
+
- an ADVISORY warning when the file basename does not match scenario_id (the
|
|
60
|
+
receiver fetches dogfood/scenarios/<scenario_id>.yaml, so a mismatch makes the
|
|
61
|
+
committed definition unreachable and required-steps enforcement fails open)
|
|
62
|
+
|
|
63
|
+
It CANNOT verify that a real submission's step_results satisfy required_steps, nor
|
|
64
|
+
that the receiver can fetch the file at the attested commit — run a real ingest.
|
|
65
|
+
|
|
66
|
+
OPTIONS:
|
|
67
|
+
--scenario Lint the file as a scenario definition (default: policy).
|
|
68
|
+
--json Machine-readable result for CI.
|
|
69
|
+
-h, --help Show this help.
|
|
70
|
+
|
|
71
|
+
EXIT CODES:
|
|
72
|
+
0 clean or warnings-only 1 errors found 2 operator error (bad flags / IO)`;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Parse the lint argv (everything AFTER the `lint` verb). Accepts exactly one positional
|
|
76
|
+
* file path plus optional `--scenario` / `--json` / `--help`. Throws LintOperatorError
|
|
77
|
+
* (→ exit 2) on any malformed invocation.
|
|
78
|
+
*
|
|
79
|
+
* `--scenario` is a boolean MODE flag (default: policy mode). It selects which linter runs
|
|
80
|
+
* over the one positional file; it never consumes the path itself.
|
|
81
|
+
*
|
|
82
|
+
* @param {string[]} argv
|
|
83
|
+
* @returns {{ help: boolean, file: string|null, json: boolean, scenario: boolean }}
|
|
84
|
+
*/
|
|
85
|
+
export function parseLintArgs(argv) {
|
|
86
|
+
let file = null;
|
|
87
|
+
let json = false;
|
|
88
|
+
let help = false;
|
|
89
|
+
let scenario = false;
|
|
90
|
+
|
|
91
|
+
for (const arg of argv) {
|
|
92
|
+
if (arg === '-h' || arg === '--help') { help = true; continue; }
|
|
93
|
+
if (arg === '--json') { json = true; continue; }
|
|
94
|
+
if (arg === '--scenario') { scenario = true; continue; }
|
|
95
|
+
if (arg.startsWith('-')) {
|
|
96
|
+
throw new LintOperatorError(`unknown argument: ${arg}`, 'run `dogfood-verify lint --help` for usage');
|
|
97
|
+
}
|
|
98
|
+
if (file !== null) {
|
|
99
|
+
throw new LintOperatorError('more than one file given', 'lint one file at a time');
|
|
100
|
+
}
|
|
101
|
+
file = arg;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (help) return { help: true, file: null, json: false, scenario: false };
|
|
105
|
+
if (file === null) {
|
|
106
|
+
const usage = scenario ? 'dogfood-verify lint --scenario <scenario-file>' : 'dogfood-verify lint <policy-file>';
|
|
107
|
+
throw new LintOperatorError(`no ${scenario ? 'scenario' : 'policy'} file provided`, usage);
|
|
108
|
+
}
|
|
109
|
+
return { help: false, file, json, scenario };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Classify a policy file by its path so the report can name the origin (which decides the
|
|
114
|
+
* runtime fault class: global → operational, repo → submission-bad). Mirrors the
|
|
115
|
+
* `policies/global-policy.yaml` vs `policies/repos/<org>/<repo>.yaml` layout.
|
|
116
|
+
*/
|
|
117
|
+
export function originForPath(p) {
|
|
118
|
+
const norm = String(p).replace(/\\/g, '/');
|
|
119
|
+
if (/\/policies\/repos\//.test(norm)) return 'repo';
|
|
120
|
+
if (/(^|\/)global-policy\.yaml$/.test(norm)) return 'global';
|
|
121
|
+
return 'unknown';
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Render the human (default) view of a lint result — verdict-first, ERROR before WARNING. */
|
|
125
|
+
export function renderLintText(result, file) {
|
|
126
|
+
const lines = [];
|
|
127
|
+
const verdict = !result.ok ? 'ERRORS' : (result.warnings.length ? 'CLEAN (advisory warnings)' : 'CLEAN');
|
|
128
|
+
lines.push(`VERDICT: ${verdict}`);
|
|
129
|
+
lines.push('');
|
|
130
|
+
lines.push(` file: ${file}`);
|
|
131
|
+
lines.push(` origin: ${result.origin}`);
|
|
132
|
+
|
|
133
|
+
if (result.errors.length) {
|
|
134
|
+
lines.push('');
|
|
135
|
+
lines.push(`ERRORS (${result.errors.length}):`);
|
|
136
|
+
for (const e of result.errors) {
|
|
137
|
+
const field = e.field ? ` — field "${e.field}"` : '';
|
|
138
|
+
lines.push(` - [${e.label} ${e.code}] ${e.location}${field}`);
|
|
139
|
+
lines.push(` ${e.message}`);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (result.warnings.length) {
|
|
144
|
+
lines.push('');
|
|
145
|
+
lines.push(`WARNINGS (${result.warnings.length}) — advisory; the author confirms intent, nothing is auto-applied:`);
|
|
146
|
+
for (const w of result.warnings) {
|
|
147
|
+
lines.push(` - [${w.label} ${w.code}] ${w.location} — field "${w.field}"`);
|
|
148
|
+
lines.push(` ${w.message}`);
|
|
149
|
+
lines.push(` ${w.suggestion}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (result.ok && !result.warnings.length) {
|
|
154
|
+
lines.push('');
|
|
155
|
+
lines.push('No findings. The policy passes static lint.');
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
lines.push('');
|
|
159
|
+
lines.push(`note: ${result.coverageNote}`);
|
|
160
|
+
return lines.join('\n');
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Build the machine-readable (--json) result. */
|
|
164
|
+
export function buildLintJson(result, file) {
|
|
165
|
+
return {
|
|
166
|
+
file,
|
|
167
|
+
origin: result.origin,
|
|
168
|
+
ok: result.ok,
|
|
169
|
+
errors: result.errors,
|
|
170
|
+
warnings: result.warnings,
|
|
171
|
+
coverageNote: result.coverageNote,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Run the lint subcommand. Returns the exit code (does not call process.exit) so it is
|
|
177
|
+
* unit-testable with an injected stdout/stderr sink.
|
|
178
|
+
*
|
|
179
|
+
* @param {string[]} argv - args AFTER the `lint` verb
|
|
180
|
+
* @param {{ stdout?: (s: string) => void, stderr?: (s: string) => void }} [io]
|
|
181
|
+
* @returns {Promise<number>}
|
|
182
|
+
*/
|
|
183
|
+
export async function runLint(argv, io = {}) {
|
|
184
|
+
const out = io.stdout ?? ((s) => process.stdout.write(s + '\n'));
|
|
185
|
+
const err = io.stderr ?? ((s) => process.stderr.write(s + '\n'));
|
|
186
|
+
|
|
187
|
+
let opts;
|
|
188
|
+
try {
|
|
189
|
+
opts = parseLintArgs(argv);
|
|
190
|
+
} catch (e) {
|
|
191
|
+
err(`ERROR: ${e.message}`);
|
|
192
|
+
if (e.hint) err(` hint: ${e.hint}`);
|
|
193
|
+
return 2;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (opts.help) {
|
|
197
|
+
out(LINT_USAGE);
|
|
198
|
+
return 0;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const path = resolve(opts.file);
|
|
202
|
+
const kind = opts.scenario ? 'scenario' : 'policy';
|
|
203
|
+
let raw;
|
|
204
|
+
try {
|
|
205
|
+
raw = readFileSync(path, 'utf-8');
|
|
206
|
+
} catch (e) {
|
|
207
|
+
err(`ERROR: could not read ${kind} file: ${path} — ${e.message}`);
|
|
208
|
+
err(' hint: check the path exists and is readable');
|
|
209
|
+
return 2;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
let doc;
|
|
213
|
+
try {
|
|
214
|
+
doc = yaml.load(raw);
|
|
215
|
+
} catch (e) {
|
|
216
|
+
// A YAML parse failure is a lint finding about the file (exit 1), not an operator error.
|
|
217
|
+
// Mirrors the policy path exactly, only differing in the origin/label/coverageNote so the
|
|
218
|
+
// scenario report reads as a scenario report.
|
|
219
|
+
const where = e && e.mark ? ` at line ${e.mark.line + 1}, column ${e.mark.column + 1}` : '';
|
|
220
|
+
const result = opts.scenario
|
|
221
|
+
? {
|
|
222
|
+
ok: false,
|
|
223
|
+
origin: 'scenario',
|
|
224
|
+
errors: [{
|
|
225
|
+
label: 'scenario-schema:',
|
|
226
|
+
code: 'yaml_parse',
|
|
227
|
+
location: '/',
|
|
228
|
+
message: `scenario YAML failed to parse${where} — ${e.message}`,
|
|
229
|
+
}],
|
|
230
|
+
warnings: [],
|
|
231
|
+
coverageNote: SCENARIO_COVERAGE_NOTE,
|
|
232
|
+
}
|
|
233
|
+
: {
|
|
234
|
+
ok: false,
|
|
235
|
+
origin: originForPath(path),
|
|
236
|
+
errors: [{
|
|
237
|
+
label: 'policy-schema:',
|
|
238
|
+
code: 'yaml_parse',
|
|
239
|
+
location: '/',
|
|
240
|
+
message: `policy YAML failed to parse${where} — ${e.message}`,
|
|
241
|
+
}],
|
|
242
|
+
warnings: [],
|
|
243
|
+
coverageNote: COVERAGE_NOTE,
|
|
244
|
+
};
|
|
245
|
+
out(opts.json ? JSON.stringify(buildLintJson(result, path)) : renderLintText(result, path));
|
|
246
|
+
return 1;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const result = opts.scenario
|
|
250
|
+
? lintScenario(doc, { file: path })
|
|
251
|
+
: lintPolicy(doc, { origin: originForPath(path) });
|
|
252
|
+
out(opts.json ? JSON.stringify(buildLintJson(result, path)) : renderLintText(result, path));
|
|
253
|
+
return result.ok ? 0 : 1;
|
|
254
|
+
}
|
package/cli.js
CHANGED
|
@@ -24,6 +24,15 @@
|
|
|
24
24
|
* (0 accepted / 1 rejected / 2 operator error) so a wrapper can reason about
|
|
25
25
|
* both uniformly.
|
|
26
26
|
*
|
|
27
|
+
* KNOWN PREVIEW GAP (V2-CONTRACT-004, documented — wiring deferred to a
|
|
28
|
+
* feature pass): this CLI loads NO scenario definitions, so
|
|
29
|
+
* success_criteria.required_steps enforcement (F-3bfc2885) runs only in
|
|
30
|
+
* production ingest, which fetches scenarios from the source repo at the
|
|
31
|
+
* persisted commit. A preview VERDICT: ACCEPTED therefore does not cover
|
|
32
|
+
* required_steps — the same discipline run.js uses to document the GitLab
|
|
33
|
+
* scenario-fetcher gap. Both --help and every --explain rendering carry the
|
|
34
|
+
* note so no consumer can read a preview verdict as covering it.
|
|
35
|
+
*
|
|
27
36
|
* Exit codes (consistent with packages/ingest/run.js):
|
|
28
37
|
* 0 — submission accepted
|
|
29
38
|
* 1 — submission rejected (verdict reached; the payload is the problem)
|
|
@@ -32,13 +41,15 @@
|
|
|
32
41
|
* submission.
|
|
33
42
|
*/
|
|
34
43
|
|
|
35
|
-
import { readFileSync } from 'node:fs';
|
|
44
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
36
45
|
import { resolve, dirname, join } from 'node:path';
|
|
37
46
|
import { fileURLToPath } from 'node:url';
|
|
38
47
|
import yaml from 'js-yaml';
|
|
39
48
|
|
|
49
|
+
import { validatePayload } from '@dogfood-lab/schemas';
|
|
40
50
|
import { verify, parseRejectionReason } from './index.js';
|
|
41
51
|
import { stubProvenance, provenanceForProvider } from './validators/provenance.js';
|
|
52
|
+
import { runLint } from './cli-lint.js';
|
|
42
53
|
|
|
43
54
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
44
55
|
|
|
@@ -58,21 +69,21 @@ class OperatorError extends Error {
|
|
|
58
69
|
|
|
59
70
|
const USAGE = `verify — local dry-run / explain for dogfood submissions
|
|
60
71
|
|
|
61
|
-
|
|
72
|
+
Usage:
|
|
62
73
|
verify --file <path> [--explain | --json] [--provenance=stub|github]
|
|
63
74
|
verify --payload '<json>' [--explain | --json] [--provenance=stub|github]
|
|
64
75
|
|
|
65
|
-
|
|
76
|
+
Input (exactly one required):
|
|
66
77
|
--file <path> Read the submission JSON from a file.
|
|
67
78
|
--payload <json> Pass the submission JSON inline.
|
|
68
79
|
|
|
69
|
-
|
|
80
|
+
Output mode (default: --explain):
|
|
70
81
|
--explain Human-readable verdict breakdown with each rejection
|
|
71
82
|
reason classified (who must fix it). [default]
|
|
72
83
|
--json Machine-readable result for tooling. Mutually exclusive
|
|
73
84
|
with --explain.
|
|
74
85
|
|
|
75
|
-
|
|
86
|
+
Provenance (default: stub):
|
|
76
87
|
--provenance=stub No-network local check; provenance is always confirmed.
|
|
77
88
|
This is a LOCAL DRY-RUN — a real ingest re-checks
|
|
78
89
|
provenance against the source run. [default]
|
|
@@ -81,7 +92,12 @@ PROVENANCE (default: stub):
|
|
|
81
92
|
|
|
82
93
|
-h, --help Show this help.
|
|
83
94
|
|
|
84
|
-
|
|
95
|
+
Not checked in preview:
|
|
96
|
+
not checked in preview: required_steps (needs scenario definitions).
|
|
97
|
+
Scenario definitions live in the source repo; only a real ingest fetches
|
|
98
|
+
them and enforces success_criteria.required_steps.
|
|
99
|
+
|
|
100
|
+
Exit codes:
|
|
85
101
|
0 accepted 1 rejected 2 operator error (bad flags / IO / JSON)`;
|
|
86
102
|
|
|
87
103
|
/**
|
|
@@ -112,7 +128,12 @@ export function parseArgs(argv) {
|
|
|
112
128
|
arg = arg.slice(0, eq);
|
|
113
129
|
}
|
|
114
130
|
}
|
|
115
|
-
|
|
131
|
+
// F-b4dbdc52: a following token that is itself a flag is NOT a value —
|
|
132
|
+
// `verify --file --json` must hit the '--file requires a path' operator
|
|
133
|
+
// error, not consume '--json' as the path. Same guard as run.js
|
|
134
|
+
// (f-ingest-003), which this parser mirrors.
|
|
135
|
+
const nextIsValue = argv[i + 1] !== undefined && !argv[i + 1].startsWith('--');
|
|
136
|
+
const hasValue = inlineValue !== null || nextIsValue;
|
|
116
137
|
const takeValue = () => (inlineValue !== null ? inlineValue : argv[++i]);
|
|
117
138
|
|
|
118
139
|
switch (arg) {
|
|
@@ -224,6 +245,17 @@ function loadSubmission({ file, payload }) {
|
|
|
224
245
|
* @param {string} repoRoot
|
|
225
246
|
* @returns {{ globalPolicy: object, repoPolicy: object|null, policyVersion: string }}
|
|
226
247
|
*/
|
|
248
|
+
/**
|
|
249
|
+
* Collapse validatePayload errors into the same first-3 single-line summary
|
|
250
|
+
* production's loadGlobalPolicy uses (D2B-005), so preview and production
|
|
251
|
+
* name the offending YAML key identically.
|
|
252
|
+
*/
|
|
253
|
+
function summarizePolicySchemaErrors(errors) {
|
|
254
|
+
const trimmed = errors.slice(0, 3).map(e => `${e.path || '/'} ${e.message}`);
|
|
255
|
+
const ellipsis = errors.length > 3 ? `; (+${errors.length - 3} more)` : '';
|
|
256
|
+
return trimmed.join('; ') + ellipsis;
|
|
257
|
+
}
|
|
258
|
+
|
|
227
259
|
function loadPolicies(submission, repoRoot) {
|
|
228
260
|
const globalPath = join(repoRoot, 'policies', 'global-policy.yaml');
|
|
229
261
|
let globalPolicy;
|
|
@@ -234,22 +266,57 @@ function loadPolicies(submission, repoRoot) {
|
|
|
234
266
|
'run from the testing-os repo root, or set VERIFY_REPO_ROOT to it');
|
|
235
267
|
}
|
|
236
268
|
|
|
269
|
+
// F-99aa42bc: mirror production loadGlobalPolicy's fail-loud schema gate.
|
|
270
|
+
// Pre-fix the preview applied a parses-but-schema-invalid (or null/empty)
|
|
271
|
+
// global policy as-is — production ingest would refuse the same file, and
|
|
272
|
+
// a null policy surfaced downstream as a confusing VALIDATOR_FAULT_POLICY.
|
|
273
|
+
// Same divergence class F-65d4d6dd closed for the repo-policy half.
|
|
274
|
+
const globalValidation = validatePayload('policy', globalPolicy);
|
|
275
|
+
if (!globalValidation.valid) {
|
|
276
|
+
throw new OperatorError(
|
|
277
|
+
`global policy schema-invalid: ${globalPath} — ${summarizePolicySchemaErrors(globalValidation.errors)}`,
|
|
278
|
+
'fix the policy to conform to policy.schema.json — production ingest refuses this file too'
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
|
|
237
282
|
let repoPolicy = null;
|
|
238
283
|
const repoSlug = submission && typeof submission === 'object' ? submission.repo : null;
|
|
239
284
|
if (typeof repoSlug === 'string' && repoSlug.includes('/')) {
|
|
240
|
-
|
|
285
|
+
// F-54e5fde7: two-segment contract only (nested GitLab subgroups are
|
|
286
|
+
// unsupported by the submission schema). A 3+-segment slug fails closed —
|
|
287
|
+
// destructuring would silently drop the tail and look up the WRONG policy.
|
|
288
|
+
const segments = repoSlug.split('/');
|
|
289
|
+
const [org, repo] = segments.length === 2 ? segments : [null, null];
|
|
241
290
|
// Reject path-traversal segments before touching the filesystem — a hostile
|
|
242
291
|
// submission.repo like '../../etc' must never escape policies/repos/.
|
|
243
292
|
const safe = (s) => typeof s === 'string' && s.length > 0 && !s.includes('..') && !s.includes('\\') && s !== '.';
|
|
244
293
|
if (safe(org) && safe(repo)) {
|
|
245
294
|
const repoPath = join(repoRoot, 'policies', 'repos', org, `${repo}.yaml`);
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
295
|
+
// F-65d4d6dd: mirror ingest's loadRepoPolicy contract exactly so the
|
|
296
|
+
// preview can never green-light a submission production will reject:
|
|
297
|
+
// - absent file → null (defaults apply)
|
|
298
|
+
// - YAML parse failure → `__torn` sentinel (verify() rejects with
|
|
299
|
+
// `policy: repo policy unreadable — …`)
|
|
300
|
+
// - parses, schema-invalid (D2B-005 class) → `__torn` sentinel too
|
|
301
|
+
// Pre-fix, a parses-but-schema-invalid policy was silently applied as-is
|
|
302
|
+
// and a bad-YAML policy silently became null — both preview/production
|
|
303
|
+
// divergences.
|
|
304
|
+
if (existsSync(repoPath)) {
|
|
305
|
+
try {
|
|
306
|
+
repoPolicy = yaml.load(readFileSync(repoPath, 'utf-8'));
|
|
307
|
+
} catch (e) {
|
|
308
|
+
repoPolicy = { __torn: true, reason: e && e.message ? e.message : String(e), path: repoPath };
|
|
309
|
+
}
|
|
310
|
+
if (repoPolicy && repoPolicy.__torn !== true) {
|
|
311
|
+
const validation = validatePayload('policy', repoPolicy);
|
|
312
|
+
if (!validation.valid) {
|
|
313
|
+
repoPolicy = {
|
|
314
|
+
__torn: true,
|
|
315
|
+
reason: `schema-invalid — ${summarizePolicySchemaErrors(validation.errors)}`,
|
|
316
|
+
path: repoPath
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
}
|
|
253
320
|
}
|
|
254
321
|
}
|
|
255
322
|
}
|
|
@@ -295,6 +362,15 @@ function resolveProvenance(provenanceMode, submission) {
|
|
|
295
362
|
* the routing decision parseRejectionReason() makes; this maps it to operator
|
|
296
363
|
* language so the consumer knows WHOSE problem each reason is.
|
|
297
364
|
*/
|
|
365
|
+
/**
|
|
366
|
+
* V2-CONTRACT-004: appended to EVERY --explain rendering (accepted and
|
|
367
|
+
* rejected alike) so a preview verdict can never be read as covering the
|
|
368
|
+
* required_steps gate, which only production ingest enforces (it fetches the
|
|
369
|
+
* scenario definitions this preview does not have).
|
|
370
|
+
*/
|
|
371
|
+
const PREVIEW_GAP_NOTE =
|
|
372
|
+
'Not checked in preview: required_steps (needs scenario definitions — enforced only by real ingest).';
|
|
373
|
+
|
|
298
374
|
const CLASS_LABEL = {
|
|
299
375
|
'submission-bad': 'SUBMISSION — fix your payload and resubmit',
|
|
300
376
|
'operational': 'OPERATIONAL — verifier/tooling fault; page ops, do not bounce to submitter',
|
|
@@ -333,6 +409,8 @@ export function renderExplain(record) {
|
|
|
333
409
|
if (accepted) {
|
|
334
410
|
lines.push('');
|
|
335
411
|
lines.push('No rejection reasons. This submission would be accepted.');
|
|
412
|
+
lines.push('');
|
|
413
|
+
lines.push(PREVIEW_GAP_NOTE);
|
|
336
414
|
return lines.join('\n');
|
|
337
415
|
}
|
|
338
416
|
|
|
@@ -360,6 +438,8 @@ export function renderExplain(record) {
|
|
|
360
438
|
lines.push(` - ${prefix}${parsed.detail}`);
|
|
361
439
|
}
|
|
362
440
|
}
|
|
441
|
+
lines.push('');
|
|
442
|
+
lines.push(PREVIEW_GAP_NOTE);
|
|
363
443
|
return lines.join('\n');
|
|
364
444
|
}
|
|
365
445
|
|
|
@@ -440,8 +520,27 @@ export async function run(argv, io = {}) {
|
|
|
440
520
|
return record.verification?.status === 'accepted' ? 0 : 1;
|
|
441
521
|
}
|
|
442
522
|
|
|
523
|
+
/**
|
|
524
|
+
* Top-level dispatcher. The bin has two verbs:
|
|
525
|
+
* - `dogfood-verify lint <policy-file>` → the author-time policy lint (VERIFY-F3, cli-lint.js).
|
|
526
|
+
* - `dogfood-verify <flags>` → the verify dry-run/explain (the original `run` path).
|
|
527
|
+
*
|
|
528
|
+
* Dispatching on `argv[0] === 'lint'` is a purely additive change: that token previously hit the
|
|
529
|
+
* verify parser's default arm and threw `unknown argument: lint`. The verify path is unchanged.
|
|
530
|
+
*
|
|
531
|
+
* @param {string[]} argv - process.argv.slice(2)
|
|
532
|
+
* @param {object} [io] - injected stdout/stderr/repoRoot for testing
|
|
533
|
+
* @returns {Promise<number>} exit code
|
|
534
|
+
*/
|
|
535
|
+
export async function main(argv, io = {}) {
|
|
536
|
+
if (argv[0] === 'lint') {
|
|
537
|
+
return runLint(argv.slice(1), io);
|
|
538
|
+
}
|
|
539
|
+
return run(argv, io);
|
|
540
|
+
}
|
|
541
|
+
|
|
443
542
|
// --- CLI entrypoint ---
|
|
444
543
|
const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(__dirname, 'cli.js');
|
|
445
544
|
if (isMain) {
|
|
446
|
-
|
|
545
|
+
main(process.argv.slice(2)).then((code) => process.exit(code));
|
|
447
546
|
}
|