@dogfood-lab/verify 1.2.3 → 1.3.0

package/README.md

@@ -30,9 +30,11 @@ const result = verify(submission, {
 });
 
 if (!result.ok) {
+  // rejection_reasons is an array of STRINGS with stable prefixes (see
+  // "Error shape" below). Operators discriminate failure class via the
+  // prefix; the rest of the string carries the human-readable detail.
   for (const reason of result.rejection_reasons) {
-    console.error(`[${reason.code}] ${reason.message}`);
-    if (reason.hint) console.error(`  hint: ${reason.hint}`);
+    console.error(reason);
   }
   process.exit(1);
 }
@@ -76,17 +78,60 @@ Provenance fields (`github_run_id`, `github_workflow_ref`) are required when `pr
 
 ## Error shape
 
-Each `rejection_reasons[]` entry follows the testing-os structured error shape:
+`rejection_reasons[]` is an array of **strings** — the persisted-record schema (`dogfood-record.schema.json` → `verification.rejection_reasons`) enforces `items: { type: 'string' }`. Machine-readable discrimination happens via **stable string prefixes**.
 
-```ts
-{
-  code: 'POLICY_GATE_FAILED' | 'SCHEMA_MISMATCH' | 'PROVENANCE_UNVERIFIED' | ...,
-  message: string,
-  path: string,        // JSON path to the offending field
-  hint?: string,       // operator-facing remediation hint
+### Prefix taxonomy
+
+The verifier emits two prefix classes:
+
+**Submission-bad** (the submitter's payload failed a validator gate — the operator should fix the submission and resubmit):
+
+| Prefix | Source | Meaning |
+|---|---|---|
+| `schema:` | `validators/schema.js` | JSON Schema check on the submission/record envelope failed. The rest of the string carries the AJV path + message. |
+| `policy:` | `validators/policy.js` | Per-repo policy gate failed (forbidden tags, missing required fields, version-floor violation, etc.). |
+| `steps[<id>]:` | `validators/steps.js` | Step-level contract check failed on a specific step id (gate accumulation, ordering, evidence shape). |
+| `provenance:` | `validators/provenance.js` | The GitHub run-id confirmation could not match the submitted commit/repo at the GitHub API. |
+| `scenario-load:` | `packages/ingest/load-context.js` | A scenario referenced by `scenario_results` could not be loaded from the source repo (typed-reason: `timeout` / `not_found` / `parse_error` / `invalid_id`). |
+
+**Validator-crashed** (the validator itself threw an internal error — this is an operational fault, NOT submission-bad; the operator should investigate the verifier itself):
+
+| Prefix | Source | Meaning |
+|---|---|---|
+| `VALIDATOR_FAULT_SCHEMA:` | `runValidator('schema', …)` catch | Internal exception inside the schema validator. The rest of the string carries the thrown `.message`. |
+| `VALIDATOR_FAULT_POLICY:` | `runValidator('policy', …)` catch | Internal exception inside the policy validator. |
+| `VALIDATOR_FAULT_STEPS:` | `runValidator('steps', …)` catch | Internal exception inside the steps validator. |
+
+### Operator hygiene
+
+```js
+// Discriminate by prefix
+for (const r of result.rejection_reasons) {
+  if (r.startsWith('VALIDATOR_FAULT_')) {
+    // Operational incident — verifier-side. Page someone; do NOT route
+    // back to the submitter as a "fix your payload" message.
+    notifyOps(r);
+  } else if (r.startsWith('schema:') || r.startsWith('policy:') || r.startsWith('steps[')) {
+    // Submission-bad — surface to the submitter.
+    surfaceToSubmitter(r);
+  } else if (r.startsWith('provenance:')) {
+    // May be either class — GitHub API timeouts are operational; a real
+    // commit/repo mismatch is submission-bad. The string detail carries
+    // the discriminator.
+    triageProvenance(r);
+  } else if (r.startsWith('scenario-load:')) {
+    // Ingest-side: scenario fetch reason determines class. timeout is
+    // operational; not_found / parse_error / invalid_id are submission-bad.
+    triageScenarioLoad(r);
+  } else {
+    // Unknown prefix — log and surface as raw text.
+    log.warn('unknown rejection_reason prefix', r);
+  }
 }
 ```
 
+Persistence note: every entry above is round-tripped verbatim through `verification.rejection_reasons` in the persisted-record JSON; the schema enforces `array of string` so any consumer of the audit-DB ground truth sees the same prefix vocabulary.
+
 ## Docs
 
 📖 Full handbook: **<https://dogfood-lab.github.io/testing-os/handbook/>**

package/index.js

@@ -6,11 +6,49 @@
  * Never upgrades a proposed verdict.
  */
 
-import { validateSubmissionSchema } from './validators/schema.js';
-import { validatePolicy } from './validators/policy.js';
-import { validateStepResults } from './validators/steps.js';
+import { validateSubmissionSchema as _defaultValidateSubmissionSchema } from './validators/schema.js';
+import { validatePolicy as _defaultValidatePolicy } from './validators/policy.js';
+import { validateStepResults as _defaultValidateStepResults } from './validators/steps.js';
 import { computeVerdict } from './validators/verdict.js';
 
+/**
+ * D1B-003 (Stage C humanization): the SOLE catch wrapper for synchronous
+ * validator calls. Distinguishes operator-actionable signals:
+ *
+ *   - Validator returned a structured result → caller pushes the existing
+ *     `'<class>: <details>'` prefix into `rejection_reasons` (unchanged
+ *     submission-bad path; back-compat is preserved).
+ *   - Validator THREW (operational incident — ajv compile fault, policy
+ *     merge cycle, out-of-memory) → wrapper synthesizes a STABLE coded
+ *     prefix `'VALIDATOR_FAULT_<NAME>: <details>'` for the caller to push.
+ *     The prefix is greppable across runner logs:
+ *       `grep -E '"VALIDATOR_FAULT_' …`  → every operational incident
+ *       `grep -E '^(schema|policy|steps): ' …` → every submission-bad signal
+ *
+ * `name` is upper-cased to match the prefix vocabulary documented in
+ * verify/README.md (SCHEMA / POLICY / STEPS). Returns:
+ *   { ok: true, result }   when fn() returned cleanly
+ *   { ok: false, faultReason } when fn() threw
+ *
+ * The helper is the canonical seam for any new synchronous validator
+ * — adding a 4th validator class becomes a one-call site, not a
+ * five-line try/catch boilerplate.
+ *
+ * @param {string} name - Validator class name (will be upper-cased).
+ * @param {() => T} fn - The validator call.
+ * @returns {{ ok: true, result: T } | { ok: false, faultReason: string }}
+ * @template T
+ */
+function runValidator(name, fn) {
+  try {
+    return { ok: true, result: fn() };
+  } catch (e) {
+    const cls = name.toUpperCase();
+    const detail = e && e.message ? e.message : String(e);
+    return { ok: false, faultReason: `VALIDATOR_FAULT_${cls}: ${detail}` };
+  }
+}
+
 /**
  * Verify a dogfood submission and produce a persisted record.
  *
@@ -20,6 +58,14 @@ import { computeVerdict } from './validators/verdict.js';
  * @param {object|null} options.repoPolicy - Parsed repo policy (null if none)
  * @param {object} options.provenance - Provenance adapter { confirm(source) => Promise<boolean> }
  * @param {string} options.policyVersion - Semver of the policy set being applied
+ * @param {object} [options.validators] - Test-only override hook (D1B-003).
+ *   Pass `{ validateSubmissionSchema, validateStepResults, validatePolicy }`
+ *   to swap in fault-injecting stubs. Production callers leave this unset
+ *   and the helper falls back to the module-level imports. The override
+ *   exists so the wrapper's catch behaviour can be tested deterministically
+ *   without monkey-patching ESM modules (which is a no-op for live bindings).
+ *   Documented but not part of the public stability contract — the parameter
+ *   may shift shape in future minors.
  * @returns {Promise<object>} Persisted record (accepted or rejected)
  */
 export async function verify(submission, options) {
@@ -42,7 +88,13 @@ export async function verify(submission, options) {
     };
   }
 
-  const { globalPolicy, repoPolicy, provenance, policyVersion } = options;
+  const { globalPolicy, repoPolicy, provenance, policyVersion, validators: validatorOverrides = {} } = options;
+  // Resolve the three validators per-call so tests can inject fault-
+  // injecting stubs. Production callers leave `validators` unset and the
+  // module-level imports flow through unchanged.
+  const validateSubmissionSchema = validatorOverrides.validateSubmissionSchema || _defaultValidateSubmissionSchema;
+  const validateStepResults = validatorOverrides.validateStepResults || _defaultValidateStepResults;
+  const validatePolicy = validatorOverrides.validatePolicy || _defaultValidatePolicy;
   const now = new Date().toISOString();
   const reasons = [];
 
@@ -68,14 +120,18 @@ export async function verify(submission, options) {
   }
 
   // 1. Schema validation
+  // D1B-003: route both happy + fault paths through `runValidator` so
+  // submission-bad (`'schema: …'`) and operational incidents
+  // (`'VALIDATOR_FAULT_SCHEMA: …'`) emit DISTINCT, greppable prefixes.
   let schemaResult = { valid: false, errors: [] };
-  try {
-    schemaResult = validateSubmissionSchema(submission);
+  const schemaRun = runValidator('schema', () => validateSubmissionSchema(submission));
+  if (schemaRun.ok) {
+    schemaResult = schemaRun.result;
     if (!schemaResult.valid) {
       reasons.push(...schemaResult.errors.map(e => `schema: ${e}`));
     }
-  } catch (e) {
-    reasons.push('validator error: ' + e.message);
+  } else {
+    reasons.push(schemaRun.faultReason);
   }
 
   // 2. Reject if submission includes verifier-owned fields
@@ -103,26 +159,45 @@ export async function verify(submission, options) {
   }
 
   // 4. Step results validation (only if schema passed)
+  // D1B-003: same submission-bad-vs-operational-fault split as schema.
   if (schemaResult.valid && submission.scenario_results) {
     for (const scenario of submission.scenario_results) {
-      try {
-        const stepErrors = validateStepResults(scenario);
-        reasons.push(...stepErrors.map(e => `steps[${scenario.scenario_id}]: ${e}`));
-      } catch (e) {
-        reasons.push('validator error: ' + e.message);
+      const stepsRun = runValidator('steps', () => validateStepResults(scenario));
+      if (stepsRun.ok) {
+        reasons.push(...stepsRun.result.map(e => `steps[${scenario.scenario_id}]: ${e}`));
+      } else {
+        reasons.push(stepsRun.faultReason);
       }
     }
   }
 
   // 5. Policy evaluation (only if schema passed)
+  // D1B-003: see runValidator JSDoc for the split. Policy-fault on
+  // `globalPolicy` corruption surfaces as `VALIDATOR_FAULT_POLICY:` and
+  // is the operator's signal that the POLICY FILE (not the submission)
+  // is broken.
+  //
+  // D1B-006: a torn repo-policy file (sentinel `{ __torn: true, reason }`
+  // returned by `loadRepoPolicy`) MUST reject the submission with
+  // `policy_valid=false` + a `policy: repo policy unreadable` rejection
+  // reason. Pre-fix the sentinel was a silent `null` — the verifier
+  // happily ran defaults against a corrupt policy. Catch the sentinel
+  // BEFORE handing the (broken) policy to `validatePolicy`, so the
+  // operator sees a real "policy:" rejection in `rejection_reasons`.
   let policyValid = false;
   if (schemaResult.valid) {
-    try {
-      const policyResult = validatePolicy(submission, { globalPolicy, repoPolicy });
-      policyValid = policyResult.valid;
-      reasons.push(...policyResult.errors.map(e => `policy: ${e}`));
-    } catch (e) {
-      reasons.push('validator error: ' + e.message);
+    if (repoPolicy && repoPolicy.__torn === true) {
+      const detail = repoPolicy.reason || 'repo policy YAML failed to parse';
+      reasons.push(`policy: repo policy unreadable — ${detail}`);
+      // policyValid stays false.
+    } else {
+      const policyRun = runValidator('policy', () => validatePolicy(submission, { globalPolicy, repoPolicy }));
+      if (policyRun.ok) {
+        policyValid = policyRun.result.valid;
+        reasons.push(...policyRun.result.errors.map(e => `policy: ${e}`));
+      } else {
+        reasons.push(policyRun.faultReason);
+      }
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dogfood-lab/verify",
-  "version": "1.2.3",
+  "version": "1.3.0",
   "type": "module",
   "description": "Central verifier for testing-os. Validates submissions against schema and policy, produces persisted records.",
   "main": "index.js",
@@ -24,8 +24,6 @@
   },
   "dependencies": {
     "@dogfood-lab/schemas": "^1.2.0",
-    "ajv": "^8.18.0",
-    "ajv-formats": "^3.0.1",
     "js-yaml": "^4.1.0"
   },
   "engines": {

package/validators/schema.js

@@ -1,37 +1,24 @@
 /**
- * Schema validator — validates submissions against dogfood-record-submission.schema.json
+ * Schema validator — validates submissions against dogfood-record-submission.schema.json.
+ *
+ * H3 hop 3: delegates to the canonical {@link validatePayload} from
+ * `@dogfood-lab/schemas`. Pre-H3 this module compiled its own
+ * Ajv2020 + ajv-formats instance for the submission schema; that
+ * duplicated the verifier's compile path against the inbound payload
+ * AND was the third of four sites contributing to the C1 two-Ajv
+ * structural gap. The migration collapses submission-validation to
+ * the single cached validator the canonical seam shares with the
+ * rest of the workspace.
+ *
+ * Return contract preserved: `{ valid, errors: string[] }`. The
+ * string-prefix shape is consumed by verify/index.js, which prepends
+ * `schema: ` / `policy: ` / `VALIDATOR_FAULT_SCHEMA: ` per the
+ * Stage C D1B-003 operator-legibility cluster. The migration keeps
+ * the `${path} ${message}` projection so downstream prefixes stay
+ * exact.
  */
 
-import Ajv2020 from 'ajv/dist/2020.js';
-import addFormats from 'ajv-formats';
-import { readFileSync } from 'node:fs';
-import { dirname } from 'node:path';
-import { createRequire } from 'node:module';
-
-const require = createRequire(import.meta.url);
-// Resolve the schemas package's json directory via its subpath export.
-const SCHEMA_DIR = dirname(
-  require.resolve('@dogfood-lab/schemas/json/dogfood-record-submission.schema.json')
-);
-
-let _validator = null;
-
-function getValidator() {
-  if (_validator) return _validator;
-
-  try {
-    const ajv = new Ajv2020({ allErrors: true, strict: false });
-    addFormats(ajv);
-
-    const schemaPath = `${SCHEMA_DIR}/dogfood-record-submission.schema.json`;
-    const schema = JSON.parse(readFileSync(schemaPath, 'utf-8'));
-
-    _validator = ajv.compile(schema);
-    return _validator;
-  } catch (e) {
-    return { __loadError: 'Schema loading failed: ' + e.message };
-  }
-}
+import { validatePayload } from '@dogfood-lab/schemas';
 
 /**
  * Validate a submission payload against the submission JSON Schema.
@@ -40,20 +27,32 @@ function getValidator() {
  * @returns {{ valid: boolean, errors: string[] }}
  */
 export function validateSubmissionSchema(submission) {
-  const validate = getValidator();
-  if (validate.__loadError) {
-    return { valid: false, errors: [validate.__loadError] };
+  let result;
+  try {
+    result = validatePayload('recordSubmission', submission);
+  } catch (e) {
+    // The canonical compile path can throw at compileSchema time if the
+    // schema file is unreadable or malformed (Ajv compile fault). Pre-H3
+    // the corresponding fault was surfaced via a `{ __loadError }` sentinel
+    // that the runValidator helper in verify/index.js never actually
+    // routed through `VALIDATOR_FAULT_*` — `getValidator` returned a plain
+    // object and the call site special-cased its `__loadError` key. The
+    // post-H3 path is cleaner: a thrown error here propagates up to
+    // runValidator which already wraps thrown validators in
+    // `VALIDATOR_FAULT_SCHEMA` (D1B-003 humanization). Same operator-
+    // facing prefix, just routed through the lawful seam instead of a
+    // sentinel. No catch — propagate.
+    throw e;
   }
-  const valid = validate(submission);
 
-  if (valid) {
+  if (result.valid) {
     return { valid: true, errors: [] };
   }
 
-  const errors = (validate.errors || []).map(err => {
-    const path = err.instancePath || '/';
-    return `${path} ${err.message}`;
-  });
+  // String-prefix projection: pre-H3 wrote `${path} ${message}` directly
+  // from Ajv.errors. The canonical ValidationError has the same
+  // `{ path, message }` shape, so the formatting is verbatim.
+  const errors = result.errors.map(err => `${err.path} ${err.message}`);
 
   return { valid: false, errors };
 }