@dogfood-lab/verify 1.8.0 → 1.9.0

package/README.md

@@ -133,6 +133,7 @@ Discrimination happens by **class**, surfaced by `parseRejectionReason` (below).
 | `VALIDATOR_FAULT_CONTRACT_SCHEMA_VERSION:` | `runValidator('contract_schema_version', …)` catch | The version gate was called with an unknown contract key (a programmer error at the call site, not a submission fault). |
 | `submission-malformed:` | `index.js` null/non-object early-return | The submission itself was `null` or not an object — a malfunctioning **dispatcher** sent garbage, not a submitter who authored a bad-but-shaped payload. Page ops / inspect the dispatch pipeline; do NOT bounce it to a submitter. |
 | `provenance-fault:` | `index.js` provenance catch | The provenance adapter THREW an operational error confirming the run — a provider **429 rate-limit, 5xx outage, or 401/403 token** fault (`validators/provenance.js` throws these on purpose for non-404 responses). The submitter's payload is fine; the verifier could not reach a verdict. Page ops / retry; do NOT bounce it to a submitter. Distinct from the submission-bad `provenance:` (genuine absence/404). |
+| `scenario-fetch-fault:` | `packages/ingest/load-context.js` | The scenario fetcher THREW after exhausting its retry budget (**5xx/429 outage, transport reject**) or hit a **401/403 credential** fault loading a scenario definition. The submission may be perfectly good — the fetch infrastructure faulted. The ingest CLI lets this propagate (exit 2, nothing persisted); a true missing file is the ingest-class `scenario-load: … (reason: not_found)` instead. |
 
 Any future `VALIDATOR_FAULT_<NEW>:` prefix is classified `operational` by family — `parseRejectionReason` matches the `VALIDATOR_FAULT_` head, so a new validator class needs no parser edit. The `submission-malformed:` prefix is matched literally (it is not part of the `VALIDATOR_FAULT_` family).
 
@@ -140,7 +141,7 @@ Any future `VALIDATOR_FAULT_<NEW>:` prefix is classified `operational` by family
 
 | Prefix | Source | Meaning |
 |---|---|---|
-| `scenario-load:` | `packages/ingest/run.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id`). |
+| `scenario-load:` | `packages/ingest/run.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id` / `too_large` / `schema_invalid`). Outages and credential faults are NOT this class — they throw `scenario-fetch-fault:` (operational, above) instead of rejecting the submission. |
 
 ### Operator hygiene
 

package/cli-lint.js

@@ -1,18 +1,25 @@
 /**
- * cli-lint.js (VERIFY-F3) — the `dogfood-verify lint <policy-file>` subcommand.
+ * cli-lint.js (VERIFY-F3) — the `dogfood-verify lint <file>` subcommand.
  *
- * A SEPARATE parse/render path from the verify CLI (cli.js): it takes a policy YAML
- * (not a submission JSON) and reports static lint findings, so it does not share the
- * verify arg parser or output. cli.js's `main` dispatcher routes the `lint` verb here
- * and leaves the verify `run` path untouched. The exit contract mirrors that path:
+ * A SEPARATE parse/render path from the verify CLI (cli.js): it takes a policy YAML or
+ * (with --scenario) a scenario YAML — not a submission JSON — and reports static lint
+ * findings, so it does not share the verify arg parser or output. cli.js's `main`
+ * dispatcher routes the `lint` verb here and leaves the verify `run` path untouched. The
+ * exit contract mirrors that path, identically for both modes:
  *
  *   0 — clean, or warnings-only (footgun advisories never block).
- *   1 — one or more errors (schema-invalid, a static predicate fault, or unparseable YAML).
+ *   1 — one or more errors (schema-invalid, a static fault, or unparseable YAML).
  *   2 — operator error (file missing/unreadable, or a malformed invocation).
  *
- * YAML that fails to parse is exit 1 (a lint FINDING about the policy the author must fix —
+ * YAML that fails to parse is exit 1 (a lint FINDING about the file the author must fix —
  * surfacing "line 4: bad indentation" is the lint's job), not exit 2. A file that does not
  * exist is exit 2 (the author pointed at the wrong path). See docs/policy-lint.md.
+ *
+ * Two modes share the whole render/exit machinery (F-BACKEND-003):
+ *   - default (policy):  lintPolicy(doc, { origin })      → origin global|repo|unknown
+ *   - --scenario:        lintScenario(doc, { file })      → origin 'scenario'
+ * Both return the same { ok, origin, errors, warnings, coverageNote } shape, so
+ * renderLintText / buildLintJson are reused unchanged.
  */
 
 import { readFileSync } from 'node:fs';
@@ -20,6 +27,7 @@ import { resolve } from 'node:path';
 import yaml from 'js-yaml';
 
 import { lintPolicy, COVERAGE_NOTE } from './validators/lint-policy.js';
+import { lintScenario, SCENARIO_COVERAGE_NOTE } from './validators/lint-scenario.js';
 
 /** Operator-error sentinel → exit 2 (distinct from a lint finding, which is exit 1). */
 class LintOperatorError extends Error {
@@ -30,20 +38,33 @@ class LintOperatorError extends Error {
   }
 }
 
-const LINT_USAGE = `dogfood-verify lint — author-time static check for a policy file
+const LINT_USAGE = `dogfood-verify lint — author-time static check for a policy or scenario file
 
 USAGE:
   dogfood-verify lint <policy-file> [--json]
+  dogfood-verify lint --scenario <scenario-file> [--json]
 
-WHAT IT CHECKS (no submission needed):
+WHAT IT CHECKS — policy mode (default, no submission needed):
   - structural validity against policy.schema.json
   - every predicate's known leading field, combinator depth, and node budget
   - an ADVISORY warning on the [] footgun (a negative op over a [] path fails open)
 
-It CANNOT statically catch a type_mismatch or a fanout_budget overrun — those depend
-on submission data. Run \`dogfood-verify --file <submission> --explain\` for that.
+  It CANNOT statically catch a type_mismatch or a fanout_budget overrun — those depend
+  on submission data. Run \`dogfood-verify --file <submission> --explain\` for that.
+
+WHAT IT CHECKS — --scenario mode (no submission needed):
+  - structural validity against scenario.schema.json
+  - every success_criteria.required_steps entry references a declared steps[].id
+  - step ids are unique
+  - an ADVISORY warning when the file basename does not match scenario_id (the
+    receiver fetches dogfood/scenarios/<scenario_id>.yaml, so a mismatch makes the
+    committed definition unreachable and required-steps enforcement fails open)
+
+  It CANNOT verify that a real submission's step_results satisfy required_steps, nor
+  that the receiver can fetch the file at the attested commit — run a real ingest.
 
 OPTIONS:
+  --scenario    Lint the file as a scenario definition (default: policy).
   --json        Machine-readable result for CI.
   -h, --help    Show this help.
 
@@ -52,34 +73,40 @@ EXIT CODES:
 
 /**
  * Parse the lint argv (everything AFTER the `lint` verb). Accepts exactly one positional
- * policy-file path plus optional `--json` / `--help`. Throws LintOperatorError (→ exit 2)
- * on any malformed invocation.
+ * file path plus optional `--scenario` / `--json` / `--help`. Throws LintOperatorError
+ * (→ exit 2) on any malformed invocation.
+ *
+ * `--scenario` is a boolean MODE flag (default: policy mode). It selects which linter runs
+ * over the one positional file; it never consumes the path itself.
  *
  * @param {string[]} argv
- * @returns {{ help: boolean, file: string|null, json: boolean }}
+ * @returns {{ help: boolean, file: string|null, json: boolean, scenario: boolean }}
  */
 export function parseLintArgs(argv) {
   let file = null;
   let json = false;
   let help = false;
+  let scenario = false;
 
   for (const arg of argv) {
     if (arg === '-h' || arg === '--help') { help = true; continue; }
     if (arg === '--json') { json = true; continue; }
+    if (arg === '--scenario') { scenario = true; continue; }
     if (arg.startsWith('-')) {
       throw new LintOperatorError(`unknown argument: ${arg}`, 'run `dogfood-verify lint --help` for usage');
     }
     if (file !== null) {
-      throw new LintOperatorError('more than one policy file given', 'lint one file at a time');
+      throw new LintOperatorError('more than one file given', 'lint one file at a time');
     }
     file = arg;
   }
 
-  if (help) return { help: true, file: null, json: false };
+  if (help) return { help: true, file: null, json: false, scenario: false };
   if (file === null) {
-    throw new LintOperatorError('no policy file provided', 'dogfood-verify lint <policy-file>');
+    const usage = scenario ? 'dogfood-verify lint --scenario <scenario-file>' : 'dogfood-verify lint <policy-file>';
+    throw new LintOperatorError(`no ${scenario ? 'scenario' : 'policy'} file provided`, usage);
   }
-  return { help: false, file, json };
+  return { help: false, file, json, scenario };
 }
 
 /**
@@ -172,11 +199,12 @@ export async function runLint(argv, io = {}) {
   }
 
   const path = resolve(opts.file);
+  const kind = opts.scenario ? 'scenario' : 'policy';
   let raw;
   try {
     raw = readFileSync(path, 'utf-8');
   } catch (e) {
-    err(`ERROR: could not read policy file: ${path} — ${e.message}`);
+    err(`ERROR: could not read ${kind} file: ${path} — ${e.message}`);
     err('  hint: check the path exists and is readable');
     return 2;
   }
@@ -185,25 +213,42 @@ export async function runLint(argv, io = {}) {
   try {
     doc = yaml.load(raw);
   } catch (e) {
-    // A YAML parse failure is a lint finding about the policy (exit 1), not an operator error.
+    // A YAML parse failure is a lint finding about the file (exit 1), not an operator error.
+    // Mirrors the policy path exactly, only differing in the origin/label/coverageNote so the
+    // scenario report reads as a scenario report.
     const where = e && e.mark ? ` at line ${e.mark.line + 1}, column ${e.mark.column + 1}` : '';
-    const result = {
-      ok: false,
-      origin: originForPath(path),
-      errors: [{
-        label: 'policy-schema:',
-        code: 'yaml_parse',
-        location: '/',
-        message: `policy YAML failed to parse${where} — ${e.message}`,
-      }],
-      warnings: [],
-      coverageNote: COVERAGE_NOTE,
-    };
+    const result = opts.scenario
+      ? {
+          ok: false,
+          origin: 'scenario',
+          errors: [{
+            label: 'scenario-schema:',
+            code: 'yaml_parse',
+            location: '/',
+            message: `scenario YAML failed to parse${where} — ${e.message}`,
+          }],
+          warnings: [],
+          coverageNote: SCENARIO_COVERAGE_NOTE,
+        }
+      : {
+          ok: false,
+          origin: originForPath(path),
+          errors: [{
+            label: 'policy-schema:',
+            code: 'yaml_parse',
+            location: '/',
+            message: `policy YAML failed to parse${where} — ${e.message}`,
+          }],
+          warnings: [],
+          coverageNote: COVERAGE_NOTE,
+        };
     out(opts.json ? JSON.stringify(buildLintJson(result, path)) : renderLintText(result, path));
     return 1;
   }
 
-  const result = lintPolicy(doc, { origin: originForPath(path) });
+  const result = opts.scenario
+    ? lintScenario(doc, { file: path })
+    : lintPolicy(doc, { origin: originForPath(path) });
   out(opts.json ? JSON.stringify(buildLintJson(result, path)) : renderLintText(result, path));
   return result.ok ? 0 : 1;
 }

package/cli.js

@@ -24,6 +24,15 @@
  * (0 accepted / 1 rejected / 2 operator error) so a wrapper can reason about
  * both uniformly.
  *
+ * KNOWN PREVIEW GAP (V2-CONTRACT-004, documented — wiring deferred to a
+ * feature pass): this CLI loads NO scenario definitions, so
+ * success_criteria.required_steps enforcement (F-3bfc2885) runs only in
+ * production ingest, which fetches scenarios from the source repo at the
+ * persisted commit. A preview VERDICT: ACCEPTED therefore does not cover
+ * required_steps — the same discipline run.js uses to document the GitLab
+ * scenario-fetcher gap. Both --help and every --explain rendering carry the
+ * note so no consumer can read a preview verdict as covering it.
+ *
  * Exit codes (consistent with packages/ingest/run.js):
  *   0 — submission accepted
  *   1 — submission rejected (verdict reached; the payload is the problem)
@@ -32,11 +41,12 @@
  *       submission.
  */
 
-import { readFileSync } from 'node:fs';
+import { readFileSync, existsSync } from 'node:fs';
 import { resolve, dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import yaml from 'js-yaml';
 
+import { validatePayload } from '@dogfood-lab/schemas';
 import { verify, parseRejectionReason } from './index.js';
 import { stubProvenance, provenanceForProvider } from './validators/provenance.js';
 import { runLint } from './cli-lint.js';
@@ -59,21 +69,21 @@ class OperatorError extends Error {
 
 const USAGE = `verify — local dry-run / explain for dogfood submissions
 
-USAGE:
+Usage:
   verify --file <path> [--explain | --json] [--provenance=stub|github]
   verify --payload '<json>' [--explain | --json] [--provenance=stub|github]
 
-INPUT (exactly one required):
+Input (exactly one required):
   --file <path>        Read the submission JSON from a file.
   --payload <json>     Pass the submission JSON inline.
 
-OUTPUT MODE (default: --explain):
+Output mode (default: --explain):
   --explain            Human-readable verdict breakdown with each rejection
                        reason classified (who must fix it). [default]
   --json               Machine-readable result for tooling. Mutually exclusive
                        with --explain.
 
-PROVENANCE (default: stub):
+Provenance (default: stub):
   --provenance=stub    No-network local check; provenance is always confirmed.
                        This is a LOCAL DRY-RUN — a real ingest re-checks
                        provenance against the source run. [default]
@@ -82,7 +92,12 @@ PROVENANCE (default: stub):
 
   -h, --help           Show this help.
 
-EXIT CODES:
+Not checked in preview:
+  not checked in preview: required_steps (needs scenario definitions).
+  Scenario definitions live in the source repo; only a real ingest fetches
+  them and enforces success_criteria.required_steps.
+
+Exit codes:
   0  accepted     1  rejected     2  operator error (bad flags / IO / JSON)`;
 
 /**
@@ -113,7 +128,12 @@ export function parseArgs(argv) {
         arg = arg.slice(0, eq);
       }
     }
-    const hasValue = inlineValue !== null || argv[i + 1] !== undefined;
+    // F-b4dbdc52: a following token that is itself a flag is NOT a value —
+    // `verify --file --json` must hit the '--file requires a path' operator
+    // error, not consume '--json' as the path. Same guard as run.js
+    // (f-ingest-003), which this parser mirrors.
+    const nextIsValue = argv[i + 1] !== undefined && !argv[i + 1].startsWith('--');
+    const hasValue = inlineValue !== null || nextIsValue;
     const takeValue = () => (inlineValue !== null ? inlineValue : argv[++i]);
 
     switch (arg) {
@@ -225,6 +245,17 @@ function loadSubmission({ file, payload }) {
  * @param {string} repoRoot
  * @returns {{ globalPolicy: object, repoPolicy: object|null, policyVersion: string }}
  */
+/**
+ * Collapse validatePayload errors into the same first-3 single-line summary
+ * production's loadGlobalPolicy uses (D2B-005), so preview and production
+ * name the offending YAML key identically.
+ */
+function summarizePolicySchemaErrors(errors) {
+  const trimmed = errors.slice(0, 3).map(e => `${e.path || '/'} ${e.message}`);
+  const ellipsis = errors.length > 3 ? `; (+${errors.length - 3} more)` : '';
+  return trimmed.join('; ') + ellipsis;
+}
+
 function loadPolicies(submission, repoRoot) {
   const globalPath = join(repoRoot, 'policies', 'global-policy.yaml');
   let globalPolicy;
@@ -235,22 +266,57 @@ function loadPolicies(submission, repoRoot) {
       'run from the testing-os repo root, or set VERIFY_REPO_ROOT to it');
   }
 
+  // F-99aa42bc: mirror production loadGlobalPolicy's fail-loud schema gate.
+  // Pre-fix the preview applied a parses-but-schema-invalid (or null/empty)
+  // global policy as-is — production ingest would refuse the same file, and
+  // a null policy surfaced downstream as a confusing VALIDATOR_FAULT_POLICY.
+  // Same divergence class F-65d4d6dd closed for the repo-policy half.
+  const globalValidation = validatePayload('policy', globalPolicy);
+  if (!globalValidation.valid) {
+    throw new OperatorError(
+      `global policy schema-invalid: ${globalPath} — ${summarizePolicySchemaErrors(globalValidation.errors)}`,
+      'fix the policy to conform to policy.schema.json — production ingest refuses this file too'
+    );
+  }
+
   let repoPolicy = null;
   const repoSlug = submission && typeof submission === 'object' ? submission.repo : null;
   if (typeof repoSlug === 'string' && repoSlug.includes('/')) {
-    const [org, repo] = repoSlug.split('/');
+    // F-54e5fde7: two-segment contract only (nested GitLab subgroups are
+    // unsupported by the submission schema). A 3+-segment slug fails closed —
+    // destructuring would silently drop the tail and look up the WRONG policy.
+    const segments = repoSlug.split('/');
+    const [org, repo] = segments.length === 2 ? segments : [null, null];
     // Reject path-traversal segments before touching the filesystem — a hostile
     // submission.repo like '../../etc' must never escape policies/repos/.
     const safe = (s) => typeof s === 'string' && s.length > 0 && !s.includes('..') && !s.includes('\\') && s !== '.';
     if (safe(org) && safe(repo)) {
       const repoPath = join(repoRoot, 'policies', 'repos', org, `${repo}.yaml`);
-      try {
-        repoPolicy = yaml.load(readFileSync(repoPath, 'utf-8'));
-      } catch {
-        // Absent or unreadable repo policy → null (defaults apply). This CLI is
-        // a preview; it does not reproduce ingest's torn-policy sentinel. A real
-        // ingest is the authority on a corrupt repo-policy file.
-        repoPolicy = null;
+      // F-65d4d6dd: mirror ingest's loadRepoPolicy contract exactly so the
+      // preview can never green-light a submission production will reject:
+      //   - absent file          → null (defaults apply)
+      //   - YAML parse failure   → `__torn` sentinel (verify() rejects with
+      //     `policy: repo policy unreadable — …`)
+      //   - parses, schema-invalid (D2B-005 class) → `__torn` sentinel too
+      // Pre-fix, a parses-but-schema-invalid policy was silently applied as-is
+      // and a bad-YAML policy silently became null — both preview/production
+      // divergences.
+      if (existsSync(repoPath)) {
+        try {
+          repoPolicy = yaml.load(readFileSync(repoPath, 'utf-8'));
+        } catch (e) {
+          repoPolicy = { __torn: true, reason: e && e.message ? e.message : String(e), path: repoPath };
+        }
+        if (repoPolicy && repoPolicy.__torn !== true) {
+          const validation = validatePayload('policy', repoPolicy);
+          if (!validation.valid) {
+            repoPolicy = {
+              __torn: true,
+              reason: `schema-invalid — ${summarizePolicySchemaErrors(validation.errors)}`,
+              path: repoPath
+            };
+          }
+        }
       }
     }
   }
@@ -296,6 +362,15 @@ function resolveProvenance(provenanceMode, submission) {
  * the routing decision parseRejectionReason() makes; this maps it to operator
  * language so the consumer knows WHOSE problem each reason is.
  */
+/**
+ * V2-CONTRACT-004: appended to EVERY --explain rendering (accepted and
+ * rejected alike) so a preview verdict can never be read as covering the
+ * required_steps gate, which only production ingest enforces (it fetches the
+ * scenario definitions this preview does not have).
+ */
+const PREVIEW_GAP_NOTE =
+  'Not checked in preview: required_steps (needs scenario definitions — enforced only by real ingest).';
+
 const CLASS_LABEL = {
   'submission-bad': 'SUBMISSION  — fix your payload and resubmit',
   'operational': 'OPERATIONAL — verifier/tooling fault; page ops, do not bounce to submitter',
@@ -334,6 +409,8 @@ export function renderExplain(record) {
   if (accepted) {
     lines.push('');
     lines.push('No rejection reasons. This submission would be accepted.');
+    lines.push('');
+    lines.push(PREVIEW_GAP_NOTE);
     return lines.join('\n');
   }
 
@@ -361,6 +438,8 @@ export function renderExplain(record) {
       lines.push(`    - ${prefix}${parsed.detail}`);
     }
   }
+  lines.push('');
+  lines.push(PREVIEW_GAP_NOTE);
   return lines.join('\n');
 }