@dogfood-lab/ingest 1.2.1

package/run.js

@@ -0,0 +1,540 @@
+/**
+ * Ingestion orchestrator
+ *
+ * Thin glue: dispatch → load context → verifier → persist → rebuild indexes.
+ *
+ * Does NOT:
+ * - decide verdicts on its own
+ * - enforce policy outside the verifier
+ * - inspect step results beyond passing them through
+ * - mutate source-authored fields except through the verifier result
+ *
+ * Does:
+ * - parse payload
+ * - gather needed inputs
+ * - call verifier
+ * - persist output
+ * - regenerate indexes
+ */
+
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { randomBytes } from 'node:crypto';
+
+import { verify } from '@dogfood-lab/verify';
+import { stubProvenance, githubProvenance } from '@dogfood-lab/verify/validators/provenance.js';
+import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
+import { loadGlobalPolicy, loadRepoPolicy, loadScenarios } from './load-context.js';
+import { isDuplicate, writeRecord, computeRecordPath } from './persist.js';
+import { rebuildIndexes } from './rebuild-indexes.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+/**
+ * Emit a single structured stage-transition log line via the shared helper.
+ *
+ * Pins `component: 'ingest'` so every ingest event is tagged regardless of
+ * caller-supplied fields. Delegates to the canonical helper at
+ * `@dogfood-lab/dogfood-swarm/lib/log-stage.js`, which adds the wave-17
+ * verdict-first human banner (TTY or DOGFOOD_LOG_HUMAN=1) on top of the
+ * NDJSON line that ingest.yml's CI log captures.
+ *
+ * Stages: dispatch_received | context_loaded | verify_complete |
+ * persist_complete | rebuild_indexes_complete | verify_only_complete |
+ * rejected_pre_persist | error.
+ *
+ * F-252714-061 (FT-PIPELINE-004): callers may include `correlation_id` in
+ * `fields` so a downstream log aggregator can pivot a multi-line NDJSON
+ * stream into a per-submission trace. The wrapper passes it through; the
+ * canonical generation site is `ingest()`/`verifyOnly()` (one ID per run).
+ *
+ * @param {string} stage
+ * @param {object} fields - Stage-specific fields. `submission_id` and
+ *   `correlation_id` strongly recommended. Do NOT pass `stage` as an inner
+ *   field — it would collide with the outer stage name and the spread is
+ *   last-wins. For "this stage failed inside that stage" use `failed_stage`
+ *   (e.g. `logStage('error', { failed_stage: 'rebuild_indexes', ... })`).
+ */
+function logStage(stage, fields = {}) {
+  // Defensive against F-827321-035: strip any caller-supplied `stage:`
+  // before spreading, so the positional `stage` always wins. The shared
+  // helper itself spreads fields last; without this strip, an inner
+  // `stage:` would silently overwrite the outer name and a grep of
+  // `"stage":"error"` across runner logs would miss the failure.
+  // `correlation_id` (FT-PIPELINE-004) is destructured-and-passed: it has
+  // no collision with the outer stage name, but naming it explicitly here
+  // documents the wave-22 wrapper-strip pattern's safe-field contract.
+  const { stage: _ignored, correlation_id, ...rest } = fields;
+  sharedLogStage(stage, { component: 'ingest', correlation_id, ...rest });
+}
+
+/**
+ * Generate a synthetic correlation_id for ingests where the submission has
+ * no usable run_id (null/non-object/malformed). Format: `ing-<base36-ts>-<rand4>`.
+ *
+ * Examples: `ing-1abc234d-x7f9` — readable, sortable, distinct from real
+ * `run_id` values (which never start with the `ing-` prefix in practice).
+ */
+function synthCorrelationId() {
+  const ts = Date.now().toString(36);
+  const rand = randomBytes(2).toString('hex');
+  return `ing-${ts}-${rand}`;
+}
+
+/**
+ * Resolve the correlation_id for a single ingest run.
+ * Prefer `submission.run_id` (operator pivots stay on the user-meaningful
+ * key); fall back to a synthetic id for invalid/malformed submissions.
+ */
+function resolveCorrelationId(submission) {
+  if (submission && typeof submission === 'object' && !Array.isArray(submission)) {
+    if (typeof submission.run_id === 'string' && submission.run_id.length > 0) {
+      return submission.run_id;
+    }
+  }
+  return synthCorrelationId();
+}
+
+/**
+ * Run the full ingestion pipeline.
+ *
+ * @param {object} submission - Source-authored submission payload
+ * @param {object} options
+ * @param {string} options.repoRoot - Absolute path to dogfood-labs repo root
+ * @param {object} options.provenance - Provenance adapter (REQUIRED — no default, no implicit stub)
+ * @param {object} [options.scenarioFetcher] - Scenario fetch adapter
+ * @returns {Promise<{ record: object, path: string, written: boolean, duplicate: boolean }>}
+ */
+export async function ingest(submission, options) {
+  const {
+    repoRoot,
+    provenance,
+    scenarioFetcher = null
+  } = options;
+
+  // Provenance adapter is REQUIRED. No implicit stub. Fail closed.
+  if (!provenance || typeof provenance.confirm !== 'function') {
+    throw new Error(
+      'Provenance adapter is required. Use githubProvenance(token) for production ' +
+      'or stubProvenance for tests. No implicit default — fail closed.'
+    );
+  }
+
+  const submissionIsObject = submission && typeof submission === 'object' && !Array.isArray(submission);
+  const submissionId = submissionIsObject ? (submission.run_id || null) : null;
+  const submissionRepo = submissionIsObject ? (submission.repo || null) : null;
+
+  // F-252714-061 (FT-PIPELINE-004): one correlation_id per ingest run, pinned
+  // across every stage. For valid submissions, prefer submission.run_id so
+  // operator pivots stay on the user-meaningful key; for invalid/malformed
+  // submissions (no run_id) generate a synthetic `ing-<base36-ts>-<rand4>`.
+  const correlation_id = resolveCorrelationId(submission);
+
+  logStage('dispatch_received', {
+    submission_id: submissionId,
+    correlation_id,
+    repo: submissionRepo,
+    has_scenario_results: !!(submissionIsObject && submission.scenario_results)
+  });
+
+  // 1. Check for duplicate before doing any work
+  //    We need a minimal record shape to compute the path for duplicate check
+  //    Guard against null/non-object submissions — those flow straight to verify()
+  //    which produces a rejection record marked _skipPersist.
+  if (submissionIsObject && submission.run_id && submission.repo && submission.timing?.finished_at) {
+    const probeRecord = {
+      run_id: submission.run_id,
+      repo: submission.repo,
+      timing: submission.timing,
+      verification: { status: 'accepted' }
+    };
+    if (isDuplicate(submission.run_id, probeRecord, repoRoot)) {
+      logStage('rejected_pre_persist', {
+        submission_id: submissionId,
+        correlation_id,
+        reason: 'duplicate'
+      });
+      return {
+        record: null,
+        path: null,
+        written: false,
+        duplicate: true
+      };
+    }
+  }
+
+  // 2. Load context
+  const globalPolicy = loadGlobalPolicy(repoRoot);
+  const repoPolicy = loadRepoPolicy(submissionIsObject ? (submission.repo || '') : '', repoRoot);
+  const policyVersion = repoPolicy?.policy_version || globalPolicy.policy_version || '1.0.0';
+
+  logStage('context_loaded', {
+    submission_id: submissionId,
+    correlation_id,
+    policy_version: policyVersion,
+    repo_policy_present: !!repoPolicy
+  });
+
+  // 3. Load scenario definitions (non-fatal if missing — becomes rejection reason)
+  let scenarioErrors = [];
+  if (scenarioFetcher && submissionIsObject && submission.scenario_results) {
+    const result = await loadScenarios(submission, scenarioFetcher);
+    scenarioErrors = result.errors;
+  }
+
+  // 4. Call verifier — the law engine makes all decisions
+  const record = await verify(submission, {
+    globalPolicy,
+    repoPolicy,
+    provenance,
+    policyVersion
+  });
+
+  logStage('verify_complete', {
+    submission_id: submissionId,
+    correlation_id,
+    status: record.verification?.status ?? null,
+    rejection_reason_count: record.verification?.rejection_reasons?.length ?? 0,
+    verdict: record.overall_verdict?.verified ?? null
+  });
+
+  // 4b. Append scenario loading errors to rejection reasons if any
+  if (scenarioErrors.length > 0) {
+    record.verification.rejection_reasons.push(
+      ...scenarioErrors.map(e => `scenario-load: ${e}`)
+    );
+    // If scenario loading failed, this is a rejection
+    if (record.verification.status === 'accepted' && scenarioErrors.length > 0) {
+      record.verification.status = 'rejected';
+      record.verification.policy_valid = false;
+      // Downgrade verdict if needed
+      if (record.overall_verdict.verified === 'pass') {
+        record.overall_verdict.verified = 'fail';
+        record.overall_verdict.downgraded = true;
+        if (!record.overall_verdict.downgrade_reasons) {
+          record.overall_verdict.downgrade_reasons = [];
+        }
+        record.overall_verdict.downgrade_reasons.push('scenario definitions could not be loaded');
+      }
+    }
+  }
+
+  // 5. Persist record
+  //    Verifier marks _skipPersist when input was null/non-object — the stub record
+  //    lacks repo/run_id/timing.finished_at and would crash computeRecordPath().
+  //    Surface the structured rejection cleanly without writing.
+  if (record._skipPersist) {
+    delete record._skipPersist;
+    logStage('rejected_pre_persist', {
+      submission_id: submissionId,
+      correlation_id,
+      reason: 'skip_persist',
+      rejection_reasons: record.verification?.rejection_reasons ?? []
+    });
+    return { record, path: null, written: false, duplicate: false };
+  }
+  const persistStart = Date.now();
+  const { path, written } = writeRecord(record, repoRoot);
+  logStage('persist_complete', {
+    submission_id: submissionId,
+    correlation_id,
+    path,
+    written,
+    duplicate: !written,
+    duration_ms: Date.now() - persistStart
+  });
+
+  // 6. Rebuild indexes
+  if (written) {
+    const rebuildStart = Date.now();
+    try {
+      const indexResult = rebuildIndexes(repoRoot);
+      logStage('rebuild_indexes_complete', {
+        submission_id: submissionId,
+        correlation_id,
+        duration_ms: Date.now() - rebuildStart,
+        accepted: indexResult.accepted,
+        rejected: indexResult.rejected,
+        corrupted_count: indexResult.corrupted?.length ?? 0
+      });
+    } catch (err) {
+      // failed_stage (not stage) — outer stage='error' must survive the
+      // spread inside the shared logStage helper. F-827321-035: an inner
+      // `stage:` field overwrites the outer name, hiding the error event
+      // from any `"stage":"error"` grep across the runner log.
+      logStage('error', {
+        submission_id: submissionId,
+        correlation_id,
+        failed_stage: 'rebuild_indexes',
+        message: err.message
+      });
+      console.error(`WARNING: record persisted but index rebuild failed: ${err.message} — indexes may be stale`);
+    }
+  }
+
+  return { record, path, written, duplicate: false };
+}
+
+/**
+ * Run the verify-only pipeline: steps 0-4 (load context + verify), assemble
+ * the would-be record, return it WITHOUT touching the filesystem or rebuilding
+ * indexes. Surfaces what `ingest()` WOULD have persisted plus `would_persist_to`
+ * — the path where the record would have landed.
+ *
+ * F-252714-058 (FT-PIPELINE-001): the verify pipeline already has a
+ * `_skipPersist` internal sentinel for null/non-object inputs; this function
+ * generalizes that path into a public entrypoint operators can use to dry-run
+ * any submission without side effects.
+ *
+ * Same logStage events fire as a real ingest EXCEPT `persist_complete` and
+ * `rebuild_indexes_complete` (which would lie about persistence). A
+ * `verify_only_complete` event takes their place so CI logs read coherently.
+ *
+ * @param {object} submission - Source-authored submission payload
+ * @param {object} options
+ * @param {string} options.repoRoot - Absolute path to repo root (still
+ *   needed for policy + scenario lookup)
+ * @param {object} options.provenance - Provenance adapter (REQUIRED)
+ * @param {object} [options.scenarioFetcher] - Scenario fetch adapter
+ * @returns {Promise<{
+ *   record: object,
+ *   would_persist_to: string|null,
+ *   verify_only: true
+ * }>}
+ */
+export async function verifyOnly(submission, options) {
+  const {
+    repoRoot,
+    provenance,
+    scenarioFetcher = null
+  } = options;
+
+  // Provenance adapter is REQUIRED. Same fail-closed contract as ingest().
+  if (!provenance || typeof provenance.confirm !== 'function') {
+    throw new Error(
+      'Provenance adapter is required. Use githubProvenance(token) for production ' +
+      'or stubProvenance for tests. No implicit default — fail closed.'
+    );
+  }
+
+  const submissionIsObject = submission && typeof submission === 'object' && !Array.isArray(submission);
+  const submissionId = submissionIsObject ? (submission.run_id || null) : null;
+  const submissionRepo = submissionIsObject ? (submission.repo || null) : null;
+  const correlation_id = resolveCorrelationId(submission);
+
+  logStage('dispatch_received', {
+    submission_id: submissionId,
+    correlation_id,
+    repo: submissionRepo,
+    has_scenario_results: !!(submissionIsObject && submission.scenario_results),
+    verify_only: true
+  });
+
+  // 2. Load context (verify-only still needs policy to drive the verifier)
+  const globalPolicy = loadGlobalPolicy(repoRoot);
+  const repoPolicy = loadRepoPolicy(submissionIsObject ? (submission.repo || '') : '', repoRoot);
+  const policyVersion = repoPolicy?.policy_version || globalPolicy.policy_version || '1.0.0';
+
+  logStage('context_loaded', {
+    submission_id: submissionId,
+    correlation_id,
+    policy_version: policyVersion,
+    repo_policy_present: !!repoPolicy
+  });
+
+  // 3. Load scenario definitions (non-fatal — becomes rejection reason)
+  let scenarioErrors = [];
+  if (scenarioFetcher && submissionIsObject && submission.scenario_results) {
+    const result = await loadScenarios(submission, scenarioFetcher);
+    scenarioErrors = result.errors;
+  }
+
+  // 4. Call verifier
+  const record = await verify(submission, {
+    globalPolicy,
+    repoPolicy,
+    provenance,
+    policyVersion
+  });
+
+  logStage('verify_complete', {
+    submission_id: submissionId,
+    correlation_id,
+    status: record.verification?.status ?? null,
+    rejection_reason_count: record.verification?.rejection_reasons?.length ?? 0,
+    verdict: record.overall_verdict?.verified ?? null
+  });
+
+  // 4b. Mirror ingest's scenario-error verdict downgrade so verify-only and
+  //     real ingest produce identical records for the same submission.
+  if (scenarioErrors.length > 0) {
+    record.verification.rejection_reasons.push(
+      ...scenarioErrors.map(e => `scenario-load: ${e}`)
+    );
+    if (record.verification.status === 'accepted' && scenarioErrors.length > 0) {
+      record.verification.status = 'rejected';
+      record.verification.policy_valid = false;
+      if (record.overall_verdict.verified === 'pass') {
+        record.overall_verdict.verified = 'fail';
+        record.overall_verdict.downgraded = true;
+        if (!record.overall_verdict.downgrade_reasons) {
+          record.overall_verdict.downgrade_reasons = [];
+        }
+        record.overall_verdict.downgrade_reasons.push('scenario definitions could not be loaded');
+      }
+    }
+  }
+
+  // 5. Compute would_persist_to without writing.
+  //    `_skipPersist` records lack the fields needed by computeRecordPath()
+  //    (repo, run_id, timing.finished_at). Surface null in that case — same
+  //    semantic as the real-ingest `rejected_pre_persist` branch.
+  let would_persist_to = null;
+  if (record._skipPersist) {
+    delete record._skipPersist;
+  } else {
+    try {
+      would_persist_to = computeRecordPath(record, repoRoot);
+    } catch {
+      // Defensive: if a record passes verify() but still trips path
+      // computation (e.g., a future schema with looser constraints), keep
+      // verify-only side-effect-free. Real ingest would surface the throw
+      // via writeRecord; verify-only just returns null and lets the operator
+      // see the rejection in record.verification.rejection_reasons.
+      would_persist_to = null;
+    }
+  }
+
+  logStage('verify_only_complete', {
+    submission_id: submissionId,
+    correlation_id,
+    status: record.verification?.status ?? null,
+    would_persist_to
+  });
+
+  return { record, would_persist_to, verify_only: true };
+}
+
+// --- CLI entrypoint ---
+// When run directly, reads submission from stdin or file argument
+
+const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(__dirname, 'run.js');
+
+if (isMain) {
+  const args = process.argv.slice(2);
+  const repoRoot = resolve(__dirname, '../..');
+
+  // Parse CLI flags
+  let submissionJson;
+  let provenanceMode = null;
+  let verifyOnlyFlag = false;
+  const positionalArgs = [];
+
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--provenance' && args[i + 1]) {
+      provenanceMode = args[++i];
+    } else if (args[i] === '--file' && args[i + 1]) {
+      const { readFileSync } = await import('node:fs');
+      submissionJson = readFileSync(resolve(args[++i]), 'utf-8');
+    } else if (args[i] === '--payload' && args[i + 1]) {
+      submissionJson = args[++i];
+    } else if (args[i] === '--verify-only') {
+      // F-252714-058: dry-run the pipeline without writing or rebuilding
+      // indexes. CI / operators preview what WOULD have been persisted.
+      verifyOnlyFlag = true;
+    } else {
+      positionalArgs.push(args[i]);
+    }
+  }
+
+  if (!submissionJson) {
+    // Read from stdin
+    const chunks = [];
+    for await (const chunk of process.stdin) {
+      chunks.push(chunk);
+    }
+    submissionJson = Buffer.concat(chunks).toString('utf-8');
+  }
+
+  let submission;
+  try {
+    submission = JSON.parse(submissionJson);
+    if (typeof submission === 'string') {
+      submission = JSON.parse(submission);
+    }
+  } catch (err) {
+    console.error(`ERROR: invalid JSON payload: ${err.message}`);
+    process.exit(2);
+  }
+
+  // Resolve provenance adapter — explicit, never implicit
+  let provenance;
+  if (provenanceMode === 'stub') {
+    // Structural anti-misuse: stub only allowed outside CI
+    if (process.env.CI || process.env.GITHUB_ACTIONS) {
+      console.error('ERROR: --provenance=stub is not allowed in CI/production. Use --provenance=github.');
+      process.exit(2);
+    }
+    console.error('WARNING: Using stub provenance (test/dev only). Records will NOT have real provenance verification.');
+    provenance = stubProvenance;
+  } else if (provenanceMode === 'github') {
+    const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+    if (!token) {
+      console.error('ERROR: --provenance=github requires GITHUB_TOKEN or GH_TOKEN environment variable.');
+      process.exit(2);
+    }
+    provenance = githubProvenance(token);
+  } else if (process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true') {
+    // In CI without explicit flag: default to github provenance, fail if no token
+    const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+    if (!token) {
+      console.error('ERROR: Running in CI without --provenance flag and no GITHUB_TOKEN. Cannot verify provenance.');
+      process.exit(2);
+    }
+    provenance = githubProvenance(token);
+  } else {
+    console.error('ERROR: --provenance flag is required. Use --provenance=github (production) or --provenance=stub (test/dev only).');
+    process.exit(2);
+  }
+
+  try {
+    if (verifyOnlyFlag) {
+      const result = await verifyOnly(submission, { repoRoot, provenance });
+
+      console.log(JSON.stringify({
+        status: result.record.verification.status,
+        run_id: result.record.run_id ?? null,
+        verdict: result.record.overall_verdict?.verified ?? null,
+        would_persist_to: result.would_persist_to,
+        verify_only: true,
+        rejection_reasons: result.record.verification.rejection_reasons ?? []
+      }));
+
+      // Same accepted/rejected exit-code contract as a real ingest so CI
+      // wrappers can swap `--verify-only` in/out without changing their
+      // exit-code handling.
+      process.exit(result.record.verification.status === 'accepted' ? 0 : 1);
+    }
+
+    const result = await ingest(submission, { repoRoot, provenance });
+
+    if (result.duplicate) {
+      console.log(JSON.stringify({ status: 'duplicate', run_id: submission.run_id }));
+      process.exit(0);
+    }
+
+    console.log(JSON.stringify({
+      status: result.record.verification.status,
+      run_id: result.record.run_id ?? null,
+      verdict: result.record.overall_verdict?.verified ?? null,
+      path: result.path,
+      written: result.written,
+      rejection_reasons: result.record.verification.rejection_reasons ?? []
+    }));
+
+    process.exit(result.record.verification.status === 'accepted' ? 0 : 1);
+  } catch (err) {
+    console.error(`ERROR: ingest failed: ${err.message}`);
+    process.exit(2);
+  }
+}

package/validate-record.js

@@ -0,0 +1,83 @@
+/**
+ * Persisted-record schema validator.
+ *
+ * Enforces dogfood-record.schema.json at write time. The submission validator
+ * in @dogfood-lab/verify covers the inbound payload; this is the symmetric
+ * gate on the outbound payload — the central verifier assembles the record
+ * and the persist layer must not write anything that violates the contract.
+ *
+ * Mirrors the Ajv idiom in @dogfood-lab/verify/validators/schema.js — same
+ * Ajv2020 + ajv-formats setup, lazy compile, cached compiled validator.
+ * Kept in this package (not extracted to a shared util) because the two
+ * call sites have different error shapes and lifecycles: submission validation
+ * returns { valid, errors } so the verifier can build a rejection record;
+ * record validation throws because a malformed record reaching the write
+ * path is a programming error, not user input.
+ */
+
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const SCHEMA_PATH = require.resolve('@dogfood-lab/schemas/json/dogfood-record.schema.json');
+
+let _validator = null;
+let _loadError = null;
+
+function getValidator() {
+  if (_validator) return _validator;
+  if (_loadError) throw _loadError;
+
+  try {
+    const ajv = new Ajv2020({ allErrors: true, strict: false });
+    addFormats(ajv);
+    const schema = JSON.parse(readFileSync(SCHEMA_PATH, 'utf-8'));
+    _validator = ajv.compile(schema);
+    return _validator;
+  } catch (e) {
+    _loadError = new Error(`record schema load failed: ${e.message}`);
+    throw _loadError;
+  }
+}
+
+/**
+ * Structured error thrown when a persisted record violates the schema.
+ * Caller code can `instanceof RecordValidationError` to distinguish from
+ * IO/path errors.
+ */
+export class RecordValidationError extends Error {
+  constructor(errors) {
+    const summary = errors
+      .map(e => `${e.path || '/'} ${e.message}`)
+      .join('; ');
+    super(`persisted record failed schema validation: ${summary}`);
+    this.name = 'RecordValidationError';
+    this.code = 'RECORD_SCHEMA_INVALID';
+    this.errors = errors;
+  }
+}
+
+/**
+ * Validate a persisted record against dogfood-record.schema.json.
+ * Throws RecordValidationError on failure; returns the record on success
+ * so callers can chain (`writeFile(validateRecord(r))`).
+ *
+ * @param {object} record
+ * @returns {object} the same record reference, unchanged
+ * @throws {RecordValidationError}
+ */
+export function validateRecord(record) {
+  const validate = getValidator();
+  const valid = validate(record);
+  if (valid) return record;
+
+  const errors = (validate.errors || []).map(err => ({
+    path: err.instancePath || '/',
+    keyword: err.keyword,
+    message: err.message,
+    params: err.params
+  }));
+  throw new RecordValidationError(errors);
+}