@dogfood-lab/ingest 1.2.3 → 1.3.1

package/lib/rename-with-retry.js

@@ -24,6 +24,8 @@
  */
 import { renameSync } from 'node:fs';
 
+import { sleepSync } from './sleep-sync.js';
+
 export function renameWithRetry(tmp, dest, { retries = 10, baseMs = 15, maxMs = 200 } = {}) {
   for (let i = 0; i <= retries; i++) {
     try {
@@ -32,8 +34,12 @@ export function renameWithRetry(tmp, dest, { retries = 10, baseMs = 15, maxMs =
     } catch (err) {
       if ((err.code !== 'EPERM' && err.code !== 'EBUSY') || i === retries) throw err;
       const delay = Math.min(baseMs * (1 << i), maxMs);
-      const until = Date.now() + delay;
-      while (Date.now() < until) { /* spin */ }
+      // D1B-002-ingest (Stage C humanization fold): replaced the
+      // `while (Date.now() < until)` busy-spin with the Atomics.wait-
+      // based sleepSync helper. Same workspace-cycle pattern as
+      // `ingest/lib/atomic-write.js` — sibling helper, not a
+      // cross-package import.
+      sleepSync(delay);
     }
   }
 }

package/lib/sleep-sync.js

@@ -0,0 +1,31 @@
+/**
+ * sleepSync — `Atomics.wait`-based synchronous sleep.
+ *
+ * Sibling of `packages/findings/lib/file-lock.js`'s private `sleepSync`. The
+ * duplication exists because npm workspaces do not allow `ingest → findings`
+ * imports (findings already depends on ingest, and the reverse edge would
+ * create a workspace dependency cycle — same constraint that forced
+ * `packages/ingest/lib/atomic-write.js` to duplicate
+ * `packages/findings/lib/atomic-write.js`).
+ *
+ * The helper exists to replace the `while (Date.now() < until)` busy-spin
+ * loops sprinkled across synchronous backoff paths (notably
+ * `packages/ingest/lib/rename-with-retry.js`'s EPERM/EBUSY retry on Windows
+ * NTFS). Spin loops burn a full CPU core for the entire backoff window
+ * — Atomics.wait cedes the thread.
+ *
+ * Why synchronous: the callsites (atomic-write helpers, rebuild-indexes,
+ * event-log appender, rename-with-retry) are themselves synchronous and
+ * cascading an `await` would leak the boundary into otherwise-deterministic
+ * flush paths — same rationale documented at the findings sibling.
+ *
+ * @param {number} ms - Milliseconds to sleep. Non-finite or non-positive
+ *   values are no-ops (we don't trust caller arithmetic to never produce
+ *   negative deltas if a system clock skews).
+ */
+export function sleepSync(ms) {
+  if (!Number.isFinite(ms) || ms <= 0) return;
+  const sab = new SharedArrayBuffer(4);
+  const view = new Int32Array(sab);
+  Atomics.wait(view, 0, 0, ms);
+}

package/load-context.js

@@ -14,8 +14,39 @@ import { readFileSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
 import yaml from 'js-yaml';
 
+import { validatePayload } from '@dogfood-lab/schemas';
+import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
 import { isUnsafeSegment } from './lib/unsafe-segment.js';
 
+/**
+ * D2B-005 (Phase 10 Step 1): summarise a `validatePayload('policy', …)`
+ * result down to a single-line operator-actionable reason string. The
+ * canonical seam returns `{ path, message, keyword, params }` per error;
+ * we collapse the first 3 violations into a `path message` join so the
+ * sentinel's `reason` field stays grep-friendly without truncating the
+ * structural detail an operator needs to find the offending YAML key.
+ *
+ * Three errors is a deliberate ceiling — a policy with 20 violations
+ * almost certainly has a single root cause (wrong section nesting,
+ * malformed root); flooding the reason string with 20 lines makes the
+ * error harder to read, not easier.
+ */
+function summarizePolicyErrors(errors) {
+  const trimmed = errors.slice(0, 3).map(e => `${e.path || '/'} ${e.message}`);
+  const ellipsis = errors.length > 3 ? `; (+${errors.length - 3} more)` : '';
+  return trimmed.join('; ') + ellipsis;
+}
+
+/**
+ * D1B-006 (Stage C humanization): one structured `logStage(...)` helper,
+ * component-pinned to `ingest` so the NDJSON stream is greppable with the
+ * rest of the pipeline. Same wave-22 wrapper-strip pattern as `run.js`.
+ */
+function logStage(stage, fields = {}) {
+  const { stage: _ignored, ...rest } = fields;
+  sharedLogStage(stage, { component: 'ingest', ...rest });
+}
+
 /**
  * Load the global policy.
  *
@@ -46,8 +77,9 @@ export function loadGlobalPolicy(repoRoot) {
     }
     throw new Error(`Global policy unreadable: ${path} — ${e.message}`);
   }
+  let parsed;
   try {
-    return yaml.load(raw);
+    parsed = yaml.load(raw);
   } catch (e) {
     const where = e.mark ? ` at line ${e.mark.line + 1}, column ${e.mark.column + 1}` : '';
     throw new Error(
@@ -55,14 +87,46 @@ export function loadGlobalPolicy(repoRoot) {
       `Fix the YAML and re-run.`
     );
   }
+
+  // D2B-005: schema-gate the parsed policy against policy.schema.json.
+  // Pre-fix, a YAML that parsed but didn't conform (extra fields, wrong
+  // enums, missing required) silently loaded and the verifier ran with
+  // ad-hoc field probing — a structurally-invalid policy could produce
+  // "verifier accepts everything." Post-H3 the canonical seam is one
+  // call away. Global policy is required + load-once, so a schema fault
+  // here is fail-loud (matches the YAML-invalid sibling above).
+  const validation = validatePayload('policy', parsed);
+  if (!validation.valid) {
+    throw new Error(
+      `Global policy schema-invalid: ${path} — ${summarizePolicyErrors(validation.errors)}\n` +
+      `Fix the policy to conform to policy.schema.json and re-run.`
+    );
+  }
+  return parsed;
 }
 
 /**
- * Load repo-specific policy. Returns null if no policy exists.
+ * Load repo-specific policy.
+ *
+ * Three discriminable return shapes:
+ *   - `null`            → no repo-policy file exists; defaults apply
+ *     (documented design — submission is NOT rejected for absent policy).
+ *   - parsed YAML object → policy loaded cleanly.
+ *   - `{ __torn: true, reason, path }` → D1B-006 sentinel: the policy
+ *     file EXISTS but cannot be parsed. Pre-fix this case returned null,
+ *     which the verifier then treated as "no policy → defaults apply →
+ *     accept". That sentinel inversion silently approved every
+ *     submission against a corrupt policy. The torn sentinel surfaces
+ *     the broken state so the verifier can reject with a
+ *     `'policy: <reason>'` rejection AND `policy_valid=false`.
+ *
+ * The sentinel is intentionally a distinct shape (`__torn: true`) so the
+ * verifier pattern-match needs no schema knowledge — the existence of
+ * `__torn` discriminates without parsing the policy twice.
  *
  * @param {string} repoSlug - e.g. "mcp-tool-shop-org/dogfood-labs"
  * @param {string} repoRoot
- * @returns {object|null}
+ * @returns {object|null|{ __torn: true, reason: string, path: string }}
  */
 export function loadRepoPolicy(repoSlug, repoRoot) {
   const [org, repo] = repoSlug.split('/');
@@ -70,12 +134,43 @@ export function loadRepoPolicy(repoSlug, repoRoot) {
   const path = join(repoRoot, 'policies', 'repos', org, `${repo}.yaml`);
 
   if (!existsSync(path)) return null;
+  let parsed;
   try {
-    return yaml.load(readFileSync(path, 'utf-8'));
-  } catch {
-    console.warn(`load-context: malformed YAML in repo policy for ${repoSlug}`);
-    return null;
+    parsed = yaml.load(readFileSync(path, 'utf-8'));
+  } catch (e) {
+    // D1B-006: torn-policy sentinel + structured warn event for YAML
+    // parse failures. The verifier (verify/index.js:189-192) surfaces
+    // this as a `policy: <reason>` rejection.
+    const reason = e && e.message ? e.message : String(e);
+    logStage('warn', {
+      kind: 'repo_policy_unreadable',
+      repo: repoSlug,
+      path,
+      error: reason
+    });
+    return { __torn: true, reason, path };
   }
+
+  // D2B-005 (Phase 10 Step 1): schema-gate the parsed repo policy.
+  // Reuses the `__torn` sentinel so the verifier's existing single-
+  // branch handler (verify/index.js:189) catches both "YAML failed
+  // to parse" (D1B-006) and "YAML parsed but doesn't conform to
+  // policy.schema.json" (D2B-005). The verify-side reason prefix
+  // (`policy: repo policy unreadable — <reason>`) covers both
+  // classes; the embedded schema-violation detail tells the operator
+  // which YAML key to fix.
+  const validation = validatePayload('policy', parsed);
+  if (!validation.valid) {
+    const reason = `schema-invalid — ${summarizePolicyErrors(validation.errors)}`;
+    logStage('warn', {
+      kind: 'repo_policy_unreadable',
+      repo: repoSlug,
+      path,
+      error: reason
+    });
+    return { __torn: true, reason, path };
+  }
+  return parsed;
 }
 
 /**
@@ -96,64 +191,153 @@ export function localScenarioFetcher(repoRoot) {
   };
 }
 
+/**
+ * Default per-request timeout for the GitHub scenario fetch. Mirrors the
+ * sibling `GITHUB_PROVENANCE_TIMEOUT_MS` constant at
+ * `packages/verify/validators/provenance.js`: a hung GitHub API call
+ * would otherwise stall ingest until the surrounding GitHub Actions
+ * runner timeout fires (default 6h). 30s fails fast with a typed
+ * `'timeout'` reason the operator can pivot on.
+ */
+export const GITHUB_SCENARIO_FETCH_TIMEOUT_MS = 30000;
+
 /**
  * GitHub scenario fetcher. Loads scenario definitions from a source repo
  * via the GitHub API at a specific commit SHA.
  *
+ * Two surfaces on the returned adapter:
+ *   - `fetch(scenarioId)`: legacy contract — returns the scenario object
+ *     on success, `null` on any failure. Preserved for back-compat with
+ *     callers that pattern-match on the truthiness of the result.
+ *   - `fetchWithReason(scenarioId)`: D1B-004 typed contract — always
+ *     returns `{ scenario, reason }` where `scenario` is the loaded
+ *     object on success or `null` on failure, and `reason` is one of
+ *     `'timeout' | 'not_found' | 'parse_error' | 'invalid_id'` on
+ *     failure (absent on success). The reason gives operators a
+ *     pivot key when diagnosing a stale scenario-load chain.
+ *
+ * Both surfaces honour the per-request AbortController timeout
+ * (`GITHUB_SCENARIO_FETCH_TIMEOUT_MS`, overridable via `opts.timeoutMs`).
+ *
  * @param {string} token - GitHub PAT
  * @param {string} repoSlug - e.g. "mcp-tool-shop-org/shipcheck"
  * @param {string} commitSha - Commit to fetch scenarios from
- * @returns {object} Scenario fetch adapter
+ * @param {{ timeoutMs?: number, fetchImpl?: typeof fetch }} [opts]
+ * @returns {{ fetch(scenarioId: string): Promise<object|null>, fetchWithReason(scenarioId: string): Promise<{ scenario: object|null, reason?: string }> }}
  */
-export function githubScenarioFetcher(token, repoSlug, commitSha) {
+export function githubScenarioFetcher(token, repoSlug, commitSha, opts = {}) {
+  const timeoutMs = opts.timeoutMs ?? GITHUB_SCENARIO_FETCH_TIMEOUT_MS;
+  const fetchImpl = opts.fetchImpl ?? ((url, init) => globalThis.fetch(url, init));
+
   const [org, repo] = repoSlug.split('/');
   if (!org || !repo || isUnsafeSegment(org) || isUnsafeSegment(repo)) {
-    return { async fetch() { return null; } };
+    return {
+      async fetch() { return null; },
+      async fetchWithReason() { return { scenario: null, reason: 'invalid_id' }; }
+    };
   }
-  return {
-    async fetch(scenarioId) {
-      if (!/^[\w-]+$/.test(scenarioId)) return null;
-      const path = `dogfood/scenarios/${scenarioId}.yaml`;
-      const url = `https://api.github.com/repos/${repoSlug}/contents/${path}?ref=${commitSha}`;
-
-      try {
-        const resp = await globalThis.fetch(url, {
-          headers: {
-            Authorization: `Bearer ${token}`,
-            Accept: 'application/vnd.github.raw+json',
-            'X-GitHub-Api-Version': '2022-11-28'
-          }
-        });
-        if (!resp.ok) return null;
-        const text = await resp.text();
-        return yaml.load(text);
-      } catch {
-        return null;
+
+  async function fetchWithReason(scenarioId) {
+    if (!/^[\w-]+$/.test(scenarioId)) {
+      return { scenario: null, reason: 'invalid_id' };
+    }
+    const path = `dogfood/scenarios/${scenarioId}.yaml`;
+    const url = `https://api.github.com/repos/${repoSlug}/contents/${path}?ref=${commitSha}`;
+
+    // D1B-004: AbortController-bounded request. Copied from
+    // `packages/verify/validators/provenance.js:80-104`. The 30s default
+    // matches that sibling so a single end-to-end ingest cannot stall
+    // longer than ~60s on cumulative GitHub-API timeouts.
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+    let text;
+    try {
+      const resp = await fetchImpl(url, {
+        headers: {
+          Authorization: `Bearer ${token}`,
+          Accept: 'application/vnd.github.raw+json',
+          'X-GitHub-Api-Version': '2022-11-28'
+        },
+        signal: controller.signal
+      });
+      if (!resp.ok) {
+        return { scenario: null, reason: 'not_found' };
+      }
+      text = await resp.text();
+    } catch (err) {
+      if (err && (err.name === 'AbortError' || err.code === 'ABORT_ERR')) {
+        return { scenario: null, reason: 'timeout' };
       }
+      // Network reject, DNS failure, etc. — surface as not_found for
+      // back-compat with the legacy null contract.
+      return { scenario: null, reason: 'not_found' };
+    } finally {
+      clearTimeout(timer);
     }
+
+    try {
+      const scenario = yaml.load(text);
+      if (!scenario || typeof scenario !== 'object') {
+        return { scenario: null, reason: 'parse_error' };
+      }
+      return { scenario };
+    } catch {
+      return { scenario: null, reason: 'parse_error' };
+    }
+  }
+
+  return {
+    // Legacy: success returns the scenario object, failure returns null
+    // — every existing caller pattern-matches on truthiness.
+    async fetch(scenarioId) {
+      const result = await fetchWithReason(scenarioId);
+      return result.scenario;
+    },
+    fetchWithReason
   };
 }
 
 /**
  * Load all scenario definitions referenced by a submission's scenario_results.
  *
+ * D1B-004 / L1-007 (Wave A2 Stage C amend2): when the fetcher exposes
+ * the typed `fetchWithReason` surface (the canonical
+ * `githubScenarioFetcher` does; the legacy `localScenarioFetcher` and any
+ * legacy test stub may not), the discriminated `reason` (`timeout` /
+ * `not_found` / `parse_error` / `invalid_id`) is propagated into the
+ * error string so the operator can pivot on the failure class. Otherwise
+ * we fall back to the legacy `fetch(id)` truthiness contract.
+ *
  * @param {object} submission
- * @param {object} scenarioFetcher - { fetch(scenarioId) => Promise<object|null> }
+ * @param {object} scenarioFetcher - { fetch(scenarioId), fetchWithReason?(scenarioId) }
  * @returns {Promise<{ scenarios: Map<string, object>, errors: string[] }>}
  */
 export async function loadScenarios(submission, scenarioFetcher) {
   const scenarios = new Map();
   const errors = [];
 
+  const supportsTypedReason = typeof scenarioFetcher.fetchWithReason === 'function';
+
   for (const sr of submission.scenario_results || []) {
     const id = sr.scenario_id;
     if (scenarios.has(id)) continue;
 
-    const definition = await scenarioFetcher.fetch(id);
-    if (definition) {
-      scenarios.set(id, definition);
+    if (supportsTypedReason) {
+      const result = await scenarioFetcher.fetchWithReason(id);
+      if (result && result.scenario) {
+        scenarios.set(id, result.scenario);
+      } else {
+        const reason = result && result.reason ? result.reason : 'unknown';
+        errors.push(`scenario "${id}" could not be loaded from source repo (reason: ${reason})`);
+      }
     } else {
-      errors.push(`scenario "${id}" could not be loaded from source repo`);
+      const definition = await scenarioFetcher.fetch(id);
+      if (definition) {
+        scenarios.set(id, definition);
+      } else {
+        errors.push(`scenario "${id}" could not be loaded from source repo`);
+      }
     }
   }
 

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@dogfood-lab/ingest",
-  "version": "1.2.3",
+  "version": "1.3.1",
   "type": "module",
   "description": "Ingestion pipeline for testing-os. Thin glue: dispatch → verifier → persist → indexes.",
   "main": "run.js",
   "exports": {
     ".": "./run.js",
-    "./lib/*": "./lib/*"
+    "./lib/*": "./lib/*",
+    "./validate-record.js": "./validate-record.js"
   },
   "scripts": {
     "test": "node --test",
@@ -29,8 +30,6 @@
     "@dogfood-lab/dogfood-swarm": "^1.2.0",
     "@dogfood-lab/schemas": "^1.2.0",
     "@dogfood-lab/verify": "^1.2.0",
-    "ajv": "^8.18.0",
-    "ajv-formats": "^3.0.1",
     "js-yaml": "^4.1.0"
   },
   "engines": {

package/rebuild-indexes.js

@@ -28,8 +28,24 @@ import { readdirSync, readFileSync, writeFileSync, mkdirSync, existsSync, unlink
 import { join, relative } from 'node:path';
 import { randomBytes } from 'node:crypto';
 
+import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
 import { stageWriteFileSync, promoteStaged, discardStaged } from './lib/atomic-write.js';
 
+/**
+ * D1B-005 (Stage C humanization): one structured `logStage('warn', ...)`
+ * helper, component-pinned to `ingest` so the NDJSON line is greppable
+ * with the rest of the pipeline. Wave-22 sibling pattern — see
+ * `packages/ingest/run.js`'s private `logStage` wrapper for the same
+ * `component: 'ingest'` pinning convention.
+ */
+function logStage(stage, fields = {}) {
+  // Defensive: strip any caller-supplied `stage:` before delegating so
+  // the outer positional stage always wins. Same shape as run.js (see
+  // wave-22 F-827321-035 hardening rationale).
+  const { stage: _ignored, ...rest } = fields;
+  sharedLogStage(stage, { component: 'ingest', ...rest });
+}
+
 /**
  * Recursively find all .json files under a directory.
  *
@@ -103,12 +119,24 @@ export function rebuildIndexes(repoRoot, options = {}) {
     const { record, error } = loadRecord(f);
     if (error) {
       corrupted.push({ path: relPath, error });
-      console.error(`[rebuild-indexes] corrupted record skipped: ${relPath} — ${error}`);
+      // D1B-005: structured warn event so the NDJSON stream carries the
+      // skip with the same discipline as the ingest pipeline. Greppable
+      // via `"kind":"record_skipped"` + `"reason":"corrupted"`.
+      logStage('warn', {
+        kind: 'record_skipped',
+        reason: 'corrupted',
+        path: relPath,
+        error
+      });
       continue;
     }
     if (!record || !record.run_id) {
       skipped.push({ path: relPath, reason: 'missing run_id' });
-      console.error(`[rebuild-indexes] record skipped (missing run_id): ${relPath}`);
+      logStage('warn', {
+        kind: 'record_skipped',
+        reason: 'missing_run_id',
+        path: relPath
+      });
       continue;
     }
     record._path = relPath;

package/run.js

@@ -100,7 +100,7 @@ function resolveCorrelationId(submission) {
  *
  * @param {object} submission - Source-authored submission payload
  * @param {object} options
- * @param {string} options.repoRoot - Absolute path to dogfood-labs repo root
+ * @param {string} options.repoRoot - Absolute path to the dogfood-lab/testing-os repo root
  * @param {object} options.provenance - Provenance adapter (REQUIRED — no default, no implicit stub)
  * @param {object} [options.scenarioFetcher] - Scenario fetch adapter
  * @returns {Promise<{ record: object, path: string, written: boolean, duplicate: boolean }>}
@@ -435,6 +435,31 @@ export async function verifyOnly(submission, options) {
   return { record, would_persist_to, verify_only: true };
 }
 
+/**
+ * D1B-001: emit a single structured `stage:'error'` NDJSON event for any
+ * CLI-toplevel exit-2 failure, mirroring the shape used by the
+ * `rebuild_indexes` inner catch. Truncates the stack to 20 lines so the
+ * event stays grep-friendly. The human-readable `console.error` line is
+ * preserved so log-only readers continue to get the same actionable hint.
+ *
+ * Keep this hoisted (above the `isMain` block) so it is callable from every
+ * branch inside the CLI body — including the JSON.parse catch which fires
+ * BEFORE the pipeline has assigned a correlation_id from `submission.run_id`.
+ */
+function emitCliErrorEvent({ failedStage, correlationId, submissionId = null, err, humanPrefix }) {
+  const truncatedStack = err && err.stack
+    ? err.stack.split('\n').slice(0, 20).join('\n')
+    : null;
+  logStage('error', {
+    submission_id: submissionId,
+    correlation_id: correlationId,
+    failed_stage: failedStage,
+    message: err && err.message ? err.message : String(err),
+    stack: truncatedStack
+  });
+  console.error(`ERROR: ${humanPrefix}: ${err && err.message ? err.message : String(err)}`);
+}
+
 // --- CLI entrypoint ---
 // When run directly, reads submission from stdin or file argument
 
@@ -455,7 +480,24 @@ if (isMain) {
       provenanceMode = args[++i];
     } else if (args[i] === '--file' && args[i + 1]) {
       const { readFileSync } = await import('node:fs');
-      submissionJson = readFileSync(resolve(args[++i]), 'utf-8');
+      // D1B-001 family (operator-legibility): a --file read failure
+      // (ENOENT/EACCES) routes through the structured error event and exits 2
+      // — pre-fix it propagated as a raw uncaught stack + exit 1, with NO
+      // grep-able `"stage":"error"` NDJSON line. This read runs during
+      // arg-parsing, BEFORE `cliCorrelationId` is seeded below, so synth a
+      // correlation id here (the same pivot the JSON.parse catch uses when
+      // there is no submission to derive a run_id from yet).
+      try {
+        submissionJson = readFileSync(resolve(args[++i]), 'utf-8');
+      } catch (err) {
+        emitCliErrorEvent({
+          failedStage: 'cli_read_file',
+          correlationId: synthCorrelationId(),
+          err,
+          humanPrefix: 'could not read --file payload'
+        });
+        process.exit(2);
+      }
     } else if (args[i] === '--payload' && args[i + 1]) {
       submissionJson = args[++i];
     } else if (args[i] === '--verify-only') {
@@ -476,23 +518,53 @@ if (isMain) {
     submissionJson = Buffer.concat(chunks).toString('utf-8');
   }
 
+  // D1B-001 (Stage C humanization): every CLI exit-2 path emits a structured
+  // `logStage('error', ...)` event before `process.exit(2)` so a grep of
+  // `"stage":"error"` across runner logs surfaces the failure with the same
+  // discipline as the inner `rebuild_indexes` catch. `failed_stage` names
+  // the last-successful pipeline stage; `correlation_id` carries the pivot
+  // key (submission.run_id when available, synth `ing-…` otherwise).
+  let lastSuccessfulStage = 'cli_startup';
+  let cliCorrelationId = synthCorrelationId();
+
   let submission;
   try {
     submission = JSON.parse(submissionJson);
     if (typeof submission === 'string') {
       submission = JSON.parse(submission);
     }
+    lastSuccessfulStage = 'cli_parse_payload';
+    // Promote the synth id to submission.run_id when we have one.
+    cliCorrelationId = resolveCorrelationId(submission);
   } catch (err) {
-    console.error(`ERROR: invalid JSON payload: ${err.message}`);
+    emitCliErrorEvent({
+      failedStage: 'cli_parse_payload',
+      correlationId: cliCorrelationId,
+      err,
+      humanPrefix: 'invalid JSON payload'
+    });
     process.exit(2);
   }
 
-  // Resolve provenance adapter — explicit, never implicit
+  // Resolve provenance adapter — explicit, never implicit.
+  //
+  // L1-001 (Wave A2 amend2): every exit-2 path here routes through
+  // `emitCliErrorEvent` so the D1B-001 documented invariant ("every CLI
+  // exit-2 path emits a structured logStage('error', …) event") holds.
+  // `failed_stage='cli_provenance_resolve'` names the precondition; the
+  // `console.error` line is preserved inside the helper so log-only
+  // readers keep the same actionable hint.
   let provenance;
   if (provenanceMode === 'stub') {
     // Structural anti-misuse: stub only allowed outside CI
     if (process.env.CI || process.env.GITHUB_ACTIONS) {
-      console.error('ERROR: --provenance=stub is not allowed in CI/production. Use --provenance=github.');
+      emitCliErrorEvent({
+        failedStage: 'cli_provenance_resolve',
+        correlationId: cliCorrelationId,
+        submissionId: submission && submission.run_id ? submission.run_id : null,
+        err: new Error('--provenance=stub is not allowed in CI/production. Use --provenance=github.'),
+        humanPrefix: 'provenance precondition unmet'
+      });
       process.exit(2);
     }
     console.error('WARNING: Using stub provenance (test/dev only). Records will NOT have real provenance verification.');
@@ -500,7 +572,13 @@ if (isMain) {
   } else if (provenanceMode === 'github') {
     const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
     if (!token) {
-      console.error('ERROR: --provenance=github requires GITHUB_TOKEN or GH_TOKEN environment variable.');
+      emitCliErrorEvent({
+        failedStage: 'cli_provenance_resolve',
+        correlationId: cliCorrelationId,
+        submissionId: submission && submission.run_id ? submission.run_id : null,
+        err: new Error('--provenance=github requires GITHUB_TOKEN or GH_TOKEN environment variable.'),
+        humanPrefix: 'provenance precondition unmet'
+      });
       process.exit(2);
     }
     provenance = githubProvenance(token);
@@ -508,18 +586,40 @@ if (isMain) {
     // In CI without explicit flag: default to github provenance, fail if no token
     const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
     if (!token) {
-      console.error('ERROR: Running in CI without --provenance flag and no GITHUB_TOKEN. Cannot verify provenance.');
+      emitCliErrorEvent({
+        failedStage: 'cli_provenance_resolve',
+        correlationId: cliCorrelationId,
+        submissionId: submission && submission.run_id ? submission.run_id : null,
+        err: new Error('Running in CI without --provenance flag and no GITHUB_TOKEN. Cannot verify provenance.'),
+        humanPrefix: 'provenance precondition unmet'
+      });
       process.exit(2);
     }
     provenance = githubProvenance(token);
   } else {
-    console.error('ERROR: --provenance flag is required. Use --provenance=github (production) or --provenance=stub (test/dev only).');
+    emitCliErrorEvent({
+      failedStage: 'cli_provenance_resolve',
+      correlationId: cliCorrelationId,
+      submissionId: submission && submission.run_id ? submission.run_id : null,
+      err: new Error('--provenance flag is required. Use --provenance=github (production) or --provenance=stub (test/dev only).'),
+      humanPrefix: 'provenance precondition unmet'
+    });
     process.exit(2);
   }
 
+  // D1B-001: track the last successful pipeline stage so the outer catch
+  // can surface a useful `failed_stage` in its structured error event.
+  // We update it once the verify/ingest call has RETURNED — anything
+  // thrown inside `ingest()` or `verifyOnly()` is, by definition, a
+  // pipeline-runtime failure for which the wrapper itself is the failing
+  // boundary. `cli_pipeline` is the right label there; the inner code
+  // paths that throw have already emitted their own structured rejection
+  // events (`rejected_pre_persist`, `rebuild_indexes_complete` etc.)
+  // when they could.
   try {
     if (verifyOnlyFlag) {
       const result = await verifyOnly(submission, { repoRoot, provenance });
+      lastSuccessfulStage = 'verify_only';
 
       console.log(JSON.stringify({
         status: result.record.verification.status,
@@ -537,6 +637,7 @@ if (isMain) {
     }
 
     const result = await ingest(submission, { repoRoot, provenance });
+    lastSuccessfulStage = 'ingest';
 
     if (result.duplicate) {
       console.log(JSON.stringify({ status: 'duplicate', run_id: submission.run_id }));
@@ -554,7 +655,22 @@ if (isMain) {
 
     process.exit(result.record.verification.status === 'accepted' ? 0 : 1);
   } catch (err) {
-    console.error(`ERROR: ingest failed: ${err.message}`);
+    // D1B-001 (Stage C humanization): emit the structured error event
+    // BEFORE exit 2 so `"stage":"error"` greps land. `failed_stage` is
+    // the last stage that DID complete — anything inside `ingest()` /
+    // `verifyOnly()` that throws has, by definition, blown the boundary
+    // we were about to cross.
+    const submissionId =
+      submission && typeof submission === 'object' && !Array.isArray(submission)
+        ? (submission.run_id || null)
+        : null;
+    emitCliErrorEvent({
+      failedStage: lastSuccessfulStage,
+      correlationId: cliCorrelationId,
+      submissionId,
+      err,
+      humanPrefix: 'ingest failed'
+    });
     process.exit(2);
   }
 }

package/validate-record.js

@@ -6,41 +6,30 @@
  * gate on the outbound payload — the central verifier assembles the record
  * and the persist layer must not write anything that violates the contract.
  *
- * Mirrors the Ajv idiom in @dogfood-lab/verify/validators/schema.js — same
- * Ajv2020 + ajv-formats setup, lazy compile, cached compiled validator.
- * Kept in this package (not extracted to a shared util) because the two
- * call sites have different error shapes and lifecycles: submission validation
- * returns { valid, errors } so the verifier can build a rejection record;
- * record validation throws because a malformed record reaching the write
- * path is a programming error, not user input.
+ * H3 hop 4 (LAST — the C1-sealing hop): delegates to the canonical
+ * {@link validatePayload} from `@dogfood-lab/schemas`. Pre-H3 this module
+ * compiled its own Ajv2020 + ajv-formats instance for the record schema,
+ * which was the SAME schema the verifier-side path (now also canonical)
+ * compiled in a SECOND instance — the structural root of the C1 two-Ajv
+ * gap. After H3, ingest and verify share the single cached validator the
+ * canonical seam holds for `dogfood-record.schema.json`.
+ *
+ * Contract preserved:
+ *   - Throws {@link RecordValidationError} on validation failure (NOT a
+ *     return value — the persist layer treats a malformed record as a
+ *     programming error, not user input).
+ *   - `.code === 'RECORD_SCHEMA_INVALID'` is the structural error-codes
+ *     gate pin (just hardened in A2.1 FX2 — see scripts/doc-drift-patterns.json
+ *     error-codes check); the migration keeps it intact.
+ *   - Each `errors[]` entry carries `{ path, keyword, message, params }` —
+ *     the keyword pin (packages/ingest/ingest.test.js:208-220) asserts
+ *     `typeof e.keyword === 'string'` for every error. The H3 keyword
+ *     extension at the canonical seam (validate.ts ValidationError) is
+ *     what makes this contract preservable without ingest holding its
+ *     own Ajv instance.
  */
 
-import Ajv2020 from 'ajv/dist/2020.js';
-import addFormats from 'ajv-formats';
-import { readFileSync } from 'node:fs';
-import { createRequire } from 'node:module';
-
-const require = createRequire(import.meta.url);
-const SCHEMA_PATH = require.resolve('@dogfood-lab/schemas/json/dogfood-record.schema.json');
-
-let _validator = null;
-let _loadError = null;
-
-function getValidator() {
-  if (_validator) return _validator;
-  if (_loadError) throw _loadError;
-
-  try {
-    const ajv = new Ajv2020({ allErrors: true, strict: false });
-    addFormats(ajv);
-    const schema = JSON.parse(readFileSync(SCHEMA_PATH, 'utf-8'));
-    _validator = ajv.compile(schema);
-    return _validator;
-  } catch (e) {
-    _loadError = new Error(`record schema load failed: ${e.message}`);
-    throw _loadError;
-  }
-}
+import { validatePayload } from '@dogfood-lab/schemas';
 
 /**
  * Structured error thrown when a persisted record violates the schema.
@@ -69,15 +58,20 @@ export class RecordValidationError extends Error {
  * @throws {RecordValidationError}
  */
 export function validateRecord(record) {
-  const validate = getValidator();
-  const valid = validate(record);
-  if (valid) return record;
+  const result = validatePayload('record', record);
+  if (result.valid) return record;
 
-  const errors = (validate.errors || []).map(err => ({
-    path: err.instancePath || '/',
+  // Project canonical ValidationError → ingest's historical error shape:
+  // pre-H3 entries had `{ path, keyword, message, params }`. Canonical
+  // ships the same fields (keyword added in H3 hop 0). Re-construct
+  // explicitly so any future change to the canonical shape (extra
+  // fields, renames) trips an explicit migration here rather than
+  // silently widening RecordValidationError.errors[].
+  const errors = result.errors.map(err => ({
+    path: err.path,
     keyword: err.keyword,
     message: err.message,
-    params: err.params
+    params: err.params,
   }));
   throw new RecordValidationError(errors);
 }