@dogfood-lab/ingest 1.8.0 → 1.10.0

package/run.js

@@ -24,8 +24,9 @@ import { randomBytes } from 'node:crypto';
 import { verify } from '@dogfood-lab/verify';
 import { stubProvenance, provenanceForProvider } from '@dogfood-lab/verify/validators/provenance.js';
 import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
-import { loadGlobalPolicy, loadRepoPolicy, loadScenarios } from './load-context.js';
-import { isDuplicate, writeRecord, computeRecordPath } from './persist.js';
+import { loadGlobalPolicy, loadRepoPolicy, loadScenarios, githubScenarioFetcher } from './load-context.js';
+import { isDuplicate, writeRecord, computeRecordPath, UnsafeRecordPathError } from './persist.js';
+import { RecordValidationError } from './validate-record.js';
 import { rebuildIndexes } from './rebuild-indexes.js';
 import { verifyChain, formatChainResult } from './verify-chain.js';
 import { handleAnchorCompute, handleAnchorPost, handleAnchorVerify } from './anchor/cli.js';
@@ -44,6 +45,27 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
  * @param {object} submission
  * @returns {{ provenance: object } | { err: Error }}
  */
+/**
+ * F-a2e40a09: the ONE definition of "running in CI" for the provenance
+ * branches. Truthy check on either conventional variable — CI systems export
+ * CI=1 / CI=yes / CI=true interchangeably. Previously the stub-forbidden
+ * guard used this truthy check while the real-by-default branch demanded the
+ * exact string 'true', so a CI=1 environment was "CI" for one branch and
+ * "not CI" for the other (fail-closed, but with a misleading flag-required
+ * error instead of the token hint).
+ *
+ * F-5ca6c91a: the literal strings 'false' and '0' are explicit OPT-OUTS
+ * (a convention many tools respect — `CI=false npm test` is a real idiom),
+ * not truthy CI signals. Without this carve-out, a CI=false environment was
+ * classified as CI: stub provenance forbidden and the no-flag default
+ * demanding a real token — fail-closed, but against the operator's stated
+ * intent.
+ */
+function isCI() {
+  const truthy = (v) => Boolean(v && v !== 'false' && v !== '0');
+  return truthy(process.env.CI) || truthy(process.env.GITHUB_ACTIONS);
+}
+
 function resolveProviderProvenance(submission) {
   const provider = (submission && submission.source && submission.source.provider) || 'github';
   const factory = provenanceForProvider(provider);
@@ -60,6 +82,71 @@ function resolveProviderProvenance(submission) {
   return { provenance: factory(token) };
 }
 
+/**
+ * V2-INVARIAN-004: the ONE definition of "which submissions get a scenario
+ * fetcher" (and with it required-steps enforcement). Previously this
+ * condition lived inline in the CLI entrypoint where it was untestable
+ * without spawning a process.
+ *
+ * Returns a `githubScenarioFetcher` when ALL hold:
+ *   - provenance is REAL (not stubProvenance — preview/test runs never hit
+ *     the GitHub API);
+ *   - the submission is a plain object with provider `github` (default when
+ *     `source.provider` is absent) — GitLab has no contents-API adapter yet,
+ *     so its required_steps gate remains unenforced (documented gap);
+ *   - `submission.repo` and `submission.ref.commit_sha` are strings (the
+ *     fetcher re-validates their shapes fail-closed);
+ *   - a GitHub token is present in `env` (GITHUB_TOKEN, then GH_TOKEN — the
+ *     same token resolveProviderProvenance validated).
+ *
+ * Otherwise returns null: verification proceeds without scenario
+ * definitions, exactly as before F-3bfc2885 wired the production path.
+ *
+ * @param {object} provenance - resolved provenance adapter
+ * @param {object} submission
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {ReturnType<typeof githubScenarioFetcher> | null}
+ */
+export function resolveScenarioFetcher(provenance, submission, env = process.env) {
+  return resolveScenarioFetcherDecision(provenance, submission, env).fetcher;
+}
+
+/**
+ * F-b04473d5: the reason-carrying form of {@link resolveScenarioFetcher}.
+ * Every ineligible branch previously returned a silent null — an operator
+ * auditing an accepted record could not tell "required_steps enforced and
+ * satisfied" from "enforcement skipped (GitLab gap / missing token)" in the
+ * run log. The CLI logs the decision as a `scenario_enforcement` NDJSON
+ * event; this function is the unit-testable seam behind it.
+ *
+ * @param {object} provenance
+ * @param {object} submission
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {{ fetcher: ReturnType<typeof githubScenarioFetcher> | null,
+ *   active: boolean, reason: string | null }} `reason` names the skip class
+ *   when `active` is false; null when a fetcher was constructed.
+ */
+export function resolveScenarioFetcherDecision(provenance, submission, env = process.env) {
+  const skip = (reason) => ({ fetcher: null, active: false, reason });
+  if (provenance === stubProvenance) return skip('stub-provenance');
+  if (!submission || typeof submission !== 'object' || Array.isArray(submission)) {
+    return skip('submission-malformed');
+  }
+  if (((submission.source && submission.source.provider) || 'github') !== 'github') {
+    // GitLab has no contents-API adapter yet — the documented gap.
+    return skip('provider-unsupported');
+  }
+  if (typeof submission.repo !== 'string') return skip('repo-not-string');
+  if (typeof submission.ref?.commit_sha !== 'string') return skip('commit-sha-not-string');
+  const token = env.GITHUB_TOKEN || env.GH_TOKEN;
+  if (!token) return skip('no-token');
+  return {
+    fetcher: githubScenarioFetcher(token, submission.repo, submission.ref.commit_sha),
+    active: true,
+    reason: null
+  };
+}
+
 /**
  * SEED-1 (d3-ingest-003) — posixify a path-shaped value at the operator/log
  * SERIALIZATION boundary. `computeRecordPath`/`writeRecord` return OS-native
@@ -81,6 +168,50 @@ function posixifyPath(p) {
   return typeof p === 'string' ? p.split(sep).join('/') : p;
 }
 
+/**
+ * F-2750c4e8: ceiling on individually-persisted `scenario-load:` rejection
+ * reasons. A hostile pre-schema-gate payload can produce up to
+ * MAX_DISTINCT_SCENARIO_FETCHES scenario-load errors; persisting one reason
+ * line per error bloats the evidence record without adding operator signal
+ * (a 1000-error submission has one root cause). Everything past the ceiling
+ * collapses into a single count summary.
+ */
+const SCENARIO_LOAD_REASON_CEILING = 20;
+
+/**
+ * Format loadScenarios errors as `scenario-load:` rejection reasons,
+ * collapsing past {@link SCENARIO_LOAD_REASON_CEILING}.
+ *
+ * @param {string[]} scenarioErrors
+ * @returns {string[]}
+ */
+function scenarioLoadReasons(scenarioErrors) {
+  const reasons = scenarioErrors.map(e => `scenario-load: ${e}`);
+  if (reasons.length <= SCENARIO_LOAD_REASON_CEILING) return reasons;
+  return [
+    ...reasons.slice(0, SCENARIO_LOAD_REASON_CEILING),
+    `scenario-load: (+${reasons.length - SCENARIO_LOAD_REASON_CEILING} more scenario-load errors collapsed; ${reasons.length} total)`
+  ];
+}
+
+/**
+ * DESIGN RULING (wave 4): loadScenarios' `warnings` (true-404 scenario
+ * definitions — nothing declared, nothing to enforce) land on the v1.6.0
+ * accepted-with-warning channel (`verification.warnings`), NEVER on
+ * rejection_reasons. verify() only sets the field when policy warnings
+ * exist, so create-or-append here.
+ *
+ * @param {object} record - the record returned by verify()
+ * @param {string[]} scenarioWarnings
+ */
+function appendScenarioWarnings(record, scenarioWarnings) {
+  if (!scenarioWarnings || scenarioWarnings.length === 0) return;
+  record.verification.warnings = [
+    ...(record.verification.warnings || []),
+    ...scenarioWarnings
+  ];
+}
+
 /**
  * Emit a single structured stage-transition log line via the shared helper.
  *
@@ -146,6 +277,27 @@ function resolveCorrelationId(submission) {
   return synthCorrelationId();
 }
 
+/**
+ * F-41872706: count how many loaded scenario definitions actually carry an
+ * enforceable required_steps gate. `scenarios.size` alone answers "how many
+ * definitions were fetched"; this answers "how many of those could fire the
+ * step-results-present / step-verdict-consistent reject rules" — a definition
+ * with no (or empty) success_criteria.required_steps enforces nothing (see
+ * verify/index.js: requiredSteps defaults to [] and the loop no-ops).
+ *
+ * @param {Map|null} scenarios - the Map returned by loadScenarios, or null
+ * @returns {number}
+ */
+function countRequiredStepsGates(scenarios) {
+  if (!scenarios || typeof scenarios.values !== 'function') return 0;
+  let gates = 0;
+  for (const definition of scenarios.values()) {
+    const steps = definition?.success_criteria?.required_steps;
+    if (Array.isArray(steps) && steps.length > 0) gates++;
+  }
+  return gates;
+}
+
 /**
  * Run the full ingestion pipeline.
  *
@@ -199,7 +351,40 @@ export async function ingest(submission, options) {
       timing: submission.timing,
       verification: { status: 'accepted' }
     };
-    if (isDuplicate(submission.run_id, probeRecord, repoRoot)) {
+    // F-f8952a50 (wave 10): this probe's hardcoded `status: 'accepted'` is
+    // ALSO the site the finding proved a corrected repo:mismatch resubmission
+    // was swallowed at — isDuplicate() only asks isRetryableRejection() when
+    // the record it's handed claims 'accepted' (see isDuplicate's own
+    // comment), so this probe is the FIRST place a stale non-schema rejection
+    // could mask a genuine correction, before verify() ever runs. No separate
+    // fix belongs HERE, though: this probe and writeRecord()'s own internal
+    // isDuplicate() call (persist.js) share the exact same isDuplicate() /
+    // isRetryableRejection() code path, so persist.js's per-prefix
+    // `retryable` fix (parse-rejection.js) already reaches both call sites —
+    // widening retryability there is what unblocks this probe too, with
+    // nothing probe-specific to change.
+    // F-a37d36f5: isDuplicate -> computeRecordPath runs against RAW untrusted
+    // submission.repo/run_id, three steps BEFORE verify()'s schema gate. A
+    // malformed repo ('a/b/c', a path-traversal attempt, etc.) makes
+    // computeRecordPath THROW ('invalid repo format' / 'unsafe repo segment'
+    // / 'unsafe run_id') — good, the traversal guards hold — but letting that
+    // throw escape here inverts this repo's submission-bad vs operational
+    // doctrine: it propagates as an uncaught fault (exit 2, "operator error"
+    // per the CLI's own USAGE block) for input that is squarely the
+    // SUBMITTER's to fix, and no `_rejected` evidence record is ever
+    // written. A malformed repo/run_id can never collide with an existing
+    // record anyway, so treating an unpathable submission as "not a
+    // duplicate" is semantically free — verify()'s schema gate is the
+    // authoritative judge of bad input, and writeRecord's own
+    // computeRecordPath (which runs AFTER validateRecord, on the SCHEMA-
+    // VALIDATED persisted record) remains the real enforcement point.
+    let duplicate;
+    try {
+      duplicate = isDuplicate(submission.run_id, probeRecord, repoRoot);
+    } catch {
+      duplicate = false;
+    }
+    if (duplicate) {
       logStage('rejected_pre_persist', {
         submission_id: submissionId,
         correlation_id,
@@ -226,11 +411,26 @@ export async function ingest(submission, options) {
     repo_policy_present: !!repoPolicy
   });
 
-  // 3. Load scenario definitions (non-fatal if missing — becomes rejection reason)
+  // 3. Load scenario definitions.
+  // F-3bfc2885: keep the loaded `scenarios` Map (previously discarded) and hand
+  // it to verify() so success_criteria.required_steps is actually enforced.
+  // F-efe4f893: gate on Array.isArray — this runs BEFORE the schema gate on
+  // untrusted input, and a string/object scenario_results must never reach the
+  // fetch loop (a string iterates CHARACTERS). verify() lands the
+  // authoritative schema rejection downstream.
+  // DESIGN RULING (wave 4): a true-404 definition is a WARNING (nothing
+  // declared, nothing to enforce), a malformed committed file is a
+  // `scenario-load:` rejection, and V2-CROSS-BO-001 outages (incl. exhausted
+  // timeout, F-07ab7f86) throw `scenario-fetch-fault:` PAST this step — the
+  // CLI's outer catch emits a structured operational error and exits 2.
   let scenarioErrors = [];
-  if (scenarioFetcher && submissionIsObject && submission.scenario_results) {
+  let scenarioWarnings = [];
+  let scenarios = null;
+  if (scenarioFetcher && submissionIsObject && Array.isArray(submission.scenario_results)) {
     const result = await loadScenarios(submission, scenarioFetcher);
     scenarioErrors = result.errors;
+    scenarioWarnings = result.warnings || [];
+    scenarios = result.scenarios;
   }
 
   // 4. Call verifier — the law engine makes all decisions
@@ -238,7 +438,8 @@ export async function ingest(submission, options) {
     globalPolicy,
     repoPolicy,
     provenance,
-    policyVersion
+    policyVersion,
+    scenarios
   });
 
   logStage('verify_complete', {
@@ -246,14 +447,25 @@ export async function ingest(submission, options) {
     correlation_id,
     status: record.verification?.status ?? null,
     rejection_reason_count: record.verification?.rejection_reasons?.length ?? 0,
-    verdict: record.overall_verdict?.verified ?? null
+    verdict: record.overall_verdict?.verified ?? null,
+    // F-41872706: the scenario_enforcement event (CLI wrapper, pre-load) logs
+    // scenario_count = how many scenarios the submission DECLARED. These three
+    // are the post-load breakdown so an operator can tell "N gates enforced"
+    // from "N scenarios, all true-404, enforcement skipped". scenarios_loaded
+    // is the Map size (definitions actually fetched); required_steps_gates_fired
+    // is the subset that carried a non-empty required_steps.
+    scenarios_loaded: scenarios ? scenarios.size : 0,
+    required_steps_gates_fired: countRequiredStepsGates(scenarios),
+    scenarios_warned: scenarioWarnings.length,
+    scenarios_errored: scenarioErrors.length
   });
 
-  // 4b. Append scenario loading errors to rejection reasons if any
+  // 4b. Surface scenario loading outcomes: warnings (true-404 definitions) go
+  // to verification.warnings; errors (malformed committed files) become
+  // scenario-load rejections, collapsed past the reason ceiling (F-2750c4e8).
+  appendScenarioWarnings(record, scenarioWarnings);
   if (scenarioErrors.length > 0) {
-    record.verification.rejection_reasons.push(
-      ...scenarioErrors.map(e => `scenario-load: ${e}`)
-    );
+    record.verification.rejection_reasons.push(...scenarioLoadReasons(scenarioErrors));
     // If scenario loading failed, this is a rejection
     if (record.verification.status === 'accepted' && scenarioErrors.length > 0) {
       record.verification.status = 'rejected';
@@ -285,7 +497,72 @@ export async function ingest(submission, options) {
     return { record, path: null, written: false, duplicate: false };
   }
   const persistStart = Date.now();
-  const { path, written } = writeRecord(record, repoRoot);
+  let path, written;
+  try {
+    ({ path, written } = writeRecord(record, repoRoot));
+  } catch (err) {
+    // F-4acd28d8: computeRecordPath()'s traversal guard (isUnsafeSegment) is
+    // STRICTER than the submission schema's repo pattern (F-bbbe2e1f — e.g.
+    // `../etc` is schema-valid but traversal-unsafe), so a record can reach
+    // here without `_skipPersist` ever having been set. The record's own
+    // identifier is what's unfilable — submission-bad, not an operator
+    // incident — so route it like `_skipPersist` above instead of letting the
+    // throw reach the outer CLI catch, which would misreport it as
+    // failed_stage:'cli_parse_payload' and exit 2 ("operator error") for
+    // content that is squarely the submitter's to fix.
+    //
+    // RecordValidationError is the SIBLING gap F-4036ae25's _skipPersist
+    // narrowing opens: dogfood-record.schema.json mirrors the submission
+    // schema's constraints on every source-authored field verify() copies
+    // verbatim (ref, source, timing, scenario_results, overall_verdict.proposed),
+    // so a submission that is schema-invalid on one of THOSE fields (not just
+    // repo/run_id/timing.finished_at) reaches writeRecord() filable-by-identity
+    // and still fails validateRecord() here. Only catch it when the submission
+    // was ALREADY schema-invalid (record.verification.schema_valid === false)
+    // — that is the authoritative signal this is submission-bad fallout, not a
+    // genuine internal defect in what verify() assembled for an
+    // otherwise-valid submission, which must keep crashing loudly.
+    const isUnfilableRecordPath = err instanceof UnsafeRecordPathError;
+    const isSubmissionBadRecordShape =
+      err instanceof RecordValidationError && record.verification.schema_valid === false;
+    if (!isUnfilableRecordPath && !isSubmissionBadRecordShape) {
+      throw err;
+    }
+
+    // verify() already rejected this submission for a real reason whenever
+    // one exists (e.g. repo:mismatch, or the schema violation itself) — those
+    // reasons survive untouched. When verify() had no reason to reject (an
+    // otherwise-accepted record whose ONLY problem is storage-unsafety),
+    // downgrade the verdict here — the same shape used below for a late
+    // scenario-load rejection.
+    if (isUnfilableRecordPath) {
+      record.verification.rejection_reasons.push(`unsafe-record-path: ${err.message}`);
+      if (record.verification.status === 'accepted') {
+        record.verification.status = 'rejected';
+        record.verification.policy_valid = false;
+        if (record.overall_verdict.verified === 'pass') {
+          record.overall_verdict.verified = 'fail';
+          record.overall_verdict.downgraded = true;
+          if (!record.overall_verdict.downgrade_reasons) {
+            record.overall_verdict.downgrade_reasons = [];
+          }
+          record.overall_verdict.downgrade_reasons.push('record path could not be safely computed');
+        }
+      }
+    }
+
+    logStage('rejected_pre_persist', {
+      submission_id: submissionId,
+      correlation_id,
+      reason: isUnfilableRecordPath ? 'unsafe_record_path' : 'record_schema_invalid_from_submission',
+      rejection_reasons: record.verification.rejection_reasons ?? [],
+      // Operator-debug only: validateRecord()'s OWN structured errors, distinct
+      // from (and not merged into) the submitter-facing rejection_reasons above,
+      // which stay exactly as verify() computed them against the submission schema.
+      ...(isSubmissionBadRecordShape ? { record_validation_errors: err.errors } : {})
+    });
+    return { record, path: null, written: false, duplicate: false };
+  }
   logStage('persist_complete', {
     submission_id: submissionId,
     correlation_id,
@@ -347,7 +624,20 @@ export async function ingest(submission, options) {
     }
   }
 
-  return { record, path, written, duplicate: false };
+  // F-7b97fbd4 (wave 10): mirror persist_complete's OWN `duplicate: !written`
+  // (logged a few lines above, off the same `written`) instead of a bare
+  // `false` that silently disagreed with it. By this point `written` can be
+  // `false` for exactly one reason — writeRecord()'s internal isDuplicate()
+  // blocked the write as a collision (persist.js) — every OTHER
+  // non-persisting outcome in this function (record._skipPersist above; the
+  // UnsafeRecordPathError / RecordValidationError catch above that) already
+  // returns its own honest `duplicate: false` earlier and never reaches this
+  // line. The CLI wrapper's `if (result.duplicate)` branch depends on this
+  // being accurate: a blocked resubmission must take the terse exit-0
+  // `{status:'duplicate'}` path, the same one an early-detected duplicate
+  // (the pre-verify probe above) already takes, not fall through to the
+  // full rejected-record exit-1 shape.
+  return { record, path, written, duplicate: !written };
 }
 
 /**
@@ -417,11 +707,18 @@ export async function verifyOnly(submission, options) {
     repo_policy_present: !!repoPolicy
   });
 
-  // 3. Load scenario definitions (non-fatal — becomes rejection reason)
+  // 3. Load scenario definitions — same gates + semantics as ingest() step 3
+  // (F-3bfc2885 Map pass-through, F-efe4f893 Array.isArray gate, wave-4
+  // ruling's warning/rejection/operational split), so verify-only and real
+  // ingest produce identical records for the same submission.
   let scenarioErrors = [];
-  if (scenarioFetcher && submissionIsObject && submission.scenario_results) {
+  let scenarioWarnings = [];
+  let scenarios = null;
+  if (scenarioFetcher && submissionIsObject && Array.isArray(submission.scenario_results)) {
     const result = await loadScenarios(submission, scenarioFetcher);
     scenarioErrors = result.errors;
+    scenarioWarnings = result.warnings || [];
+    scenarios = result.scenarios;
   }
 
   // 4. Call verifier
@@ -429,7 +726,8 @@ export async function verifyOnly(submission, options) {
     globalPolicy,
     repoPolicy,
     provenance,
-    policyVersion
+    policyVersion,
+    scenarios
   });
 
   logStage('verify_complete', {
@@ -437,15 +735,21 @@ export async function verifyOnly(submission, options) {
     correlation_id,
     status: record.verification?.status ?? null,
     rejection_reason_count: record.verification?.rejection_reasons?.length ?? 0,
-    verdict: record.overall_verdict?.verified ?? null
+    verdict: record.overall_verdict?.verified ?? null,
+    // F-41872706: same post-load scenario breakdown as ingest()'s
+    // verify_complete, so verify-only and real ingest produce identical NDJSON
+    // observability for the same submission.
+    scenarios_loaded: scenarios ? scenarios.size : 0,
+    required_steps_gates_fired: countRequiredStepsGates(scenarios),
+    scenarios_warned: scenarioWarnings.length,
+    scenarios_errored: scenarioErrors.length
   });
 
   // 4b. Mirror ingest's scenario-error verdict downgrade so verify-only and
   //     real ingest produce identical records for the same submission.
+  appendScenarioWarnings(record, scenarioWarnings);
   if (scenarioErrors.length > 0) {
-    record.verification.rejection_reasons.push(
-      ...scenarioErrors.map(e => `scenario-load: ${e}`)
-    );
+    record.verification.rejection_reasons.push(...scenarioLoadReasons(scenarioErrors));
     if (record.verification.status === 'accepted' && scenarioErrors.length > 0) {
       record.verification.status = 'rejected';
       record.verification.policy_valid = false;
@@ -472,10 +776,12 @@ export async function verifyOnly(submission, options) {
       would_persist_to = computeRecordPath(record, repoRoot);
     } catch {
       // Defensive: if a record passes verify() but still trips path
-      // computation (e.g., a future schema with looser constraints), keep
-      // verify-only side-effect-free. Real ingest would surface the throw
-      // via writeRecord; verify-only just returns null and lets the operator
-      // see the rejection in record.verification.rejection_reasons.
+      // computation (e.g. F-bbbe2e1f's `../etc` — schema-valid but
+      // traversal-unsafe), keep verify-only side-effect-free. Real ingest
+      // hits the SAME underlying computeRecordPath failure inside
+      // writeRecord() but now catches it too (F-4acd28d8, UnsafeRecordPathError)
+      // rather than letting it escape; verify-only just returns null here and
+      // lets the operator see the rejection in record.verification.rejection_reasons.
       would_persist_to = null;
     }
   }
@@ -521,6 +827,46 @@ function emitCliErrorEvent({ failedStage, correlationId, submissionId = null, er
 // --- CLI entrypoint ---
 // When run directly, reads submission from stdin or file argument
 
+// F-9a65b10c (Stage C humanization): the write-path CLI ships a USAGE block like
+// every other bin (verify/cli.js, report/cli.js, portfolio/generate.js). This is
+// the only operator-facing entry point that writes records, so its reference has
+// to live IN the tool, not scattered across docs.
+const USAGE = `ingest — persist a dogfood submission (verify → policy → provenance → write)
+
+Usage:
+  node packages/ingest/run.js --file <path>   --provenance=github|stub
+  node packages/ingest/run.js --payload '<json>' --provenance=github|stub
+  echo '<json>' | node packages/ingest/run.js --provenance=github|stub
+
+Input (exactly one; stdin used when neither flag is given):
+  --file <path>        Read the submission JSON from a file.
+  --payload <json>     Pass the submission JSON inline.
+  (stdin)              Pipe the submission JSON on stdin.
+
+Provenance (required for an ingest):
+  --provenance=github  Confirm the source run via the GitHub API.
+  --provenance=stub    No-network local confirm (dry-run / dev only).
+
+Modes:
+  --verify-only        Run the full pipeline WITHOUT writing or rebuilding
+                       indexes; report where a real ingest WOULD have landed.
+
+Standalone audit verbs (no submission, no stdin, no --provenance):
+  --verify-chain       Verify the append-only integrity ledger (offline).
+    --reconcile        Also fail on genuine torn-write orphans (on-disk records
+                       missing from the ledger). Records that predate the
+                       integrity chain itself are reported separately and do
+                       not fail the audit.
+    --all              Report every independent break instead of the first.
+  --anchor-compute     Compute + write the next XRPL anchor manifest (offline).
+  --anchor-post        Compute if needed + post the anchor to XRPL (needs XRPL_SEED).
+  --anchor-verify      Verify local manifests + run the truncation check (offline).
+
+  -h, --help           Show this help.
+
+Exit codes:
+  0  success     1  integrity/audit break     2  operator error (flags / IO / JSON)`;
+
 const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(__dirname, 'run.js');
 
 if (isMain) {
@@ -551,6 +897,7 @@ if (isMain) {
   //     per-record-independent break and report every break in one pass.
   let reconcileFlag = false;
   let allBreaksFlag = false;
+  let helpFlag = false;
   // Anchor verbs (optional, off-by-default, operator-run). --anchor-compute and
   // --anchor-verify are fully offline (never import xrpl); --anchor-post lazily
   // loads the optional xrpl package and needs XRPL_SEED.
@@ -590,9 +937,38 @@ if (isMain) {
     const hasValue = inlineValue !== null || nextIsValue;
     const takeValue = () => (inlineValue !== null ? inlineValue : args[++i]);
 
-    if (arg === '--provenance' && hasValue) {
+    // F-e0bcbc47 (amended wave 29, after the first attempt regressed
+    // F-INGEST-003): `flagIs(name)` both TESTS the token and RECORDS that
+    // `name` is a flag this chain knows about, so the catch-all below can tell
+    // "unknown flag" (no branch knows this name) apart from "known flag, no
+    // value" (a branch knows the name but its `&& hasValue` guard failed —
+    // exactly what F-INGEST-003 causes for `--anchor-network --anchor-compute`).
+    // The first attempt at this fix had a bare `else if (arg.startsWith('--'))`
+    // catch-all that could not distinguish the two and so reported a KNOWN flag
+    // as "unknown argument: --anchor-network" — a claim wider than what it
+    // actually checked, which is the exact class this repo keeps paying for.
+    //
+    // `knownFlagName` is DERIVED from the chain itself rather than duplicated
+    // into a second list of flag names: the only way a name enters the known
+    // set is for the chain to literally test for it one line below. A parallel
+    // `KNOWN_FLAGS` enumeration would be a second population to keep in sync
+    // (the same enumeration-vs-property class as CONTROL_CLASS / ZALGO_RUN /
+    // DASH_CONFUSABLES), and it would drift the first time someone adds a flag
+    // and forgets the list. Adding a branch here cannot desync it — the branch
+    // IS the registration. The one residual shape is a future branch written as
+    // a bare `arg === '--new'` instead of `flagIs('--new')`; that would make the
+    // new flag report as unknown when dangling, and is guarded by
+    // f-e0bcbc47-unknown-flag-rejection.test.js's chain-shape assertion.
+    let knownFlagName = false;
+    const flagIs = (name) => {
+      if (arg !== name) return false;
+      knownFlagName = true;
+      return true;
+    };
+
+    if (flagIs('--provenance') && hasValue) {
       provenanceMode = takeValue();
-    } else if (arg === '--file' && hasValue) {
+    } else if (flagIs('--file') && hasValue) {
       const { readFileSync } = await import('node:fs');
       // D1B-001 family (operator-legibility): a --file read failure
       // (ENOENT/EACCES) routes through the structured error event and exits 2
@@ -612,56 +988,115 @@ if (isMain) {
         });
         process.exit(2);
       }
-    } else if (arg === '--payload' && hasValue) {
+    } else if (flagIs('--payload') && hasValue) {
       submissionJson = takeValue();
-    } else if (arg === '--verify-only') {
+    } else if (flagIs('--verify-only')) {
       // F-252714-058: dry-run the pipeline without writing or rebuilding
       // indexes. CI / operators preview what WOULD have been persisted.
       verifyOnlyFlag = true;
-    } else if (arg === '--verify-chain') {
+    } else if (flagIs('--verify-chain')) {
       // Integrity chain v1: verify the append-only tamper-evident ledger at
       // indexes/integrity/chain.jsonl, fully offline. No submission, no stdin,
       // no provenance — a standalone audit command.
       verifyChainFlag = true;
-    } else if (arg === '--reconcile') {
+    } else if (flagIs('--reconcile')) {
       // Modifier for --verify-chain: also reconcile on-disk records against the
       // ledger and fail on any orphan (INGEST-PROACT-001).
       reconcileFlag = true;
-    } else if (arg === '--all') {
+    } else if (flagIs('--all')) {
       // Modifier for --verify-chain: report every per-record-independent break
       // instead of stopping at the first (INGEST-PROACT-004).
       allBreaksFlag = true;
-    } else if (arg === '--anchor-compute') {
+    } else if (flagIs('--anchor-compute')) {
       // Optional XRPL anchor: compute + write the next anchor manifest. Offline.
       anchorComputeFlag = true;
-    } else if (arg === '--anchor-post') {
+    } else if (flagIs('--anchor-post')) {
       // Optional XRPL anchor: compute if needed + post to XRPL. Needs the
       // optional xrpl package (lazily loaded) and XRPL_SEED.
       anchorPostFlag = true;
-    } else if (arg === '--anchor-verify') {
+    } else if (flagIs('--anchor-verify')) {
       // Optional XRPL anchor: verify local manifests + run the truncation check.
       // Offline reports honest NOT-verified for the on-chain leg.
       anchorVerifyFlag = true;
-    } else if (arg === '--anchor-all') {
+    } else if (flagIs('--anchor-all')) {
       // Genesis snapshot mode for compute/post (covers the whole chain).
       anchorMode = 'all';
-    } else if (arg === '--anchor-algo' && hasValue) {
+    } else if (flagIs('--anchor-algo') && hasValue) {
       anchorAlgo = takeValue();
-    } else if (arg === '--anchor-network' && hasValue) {
+    } else if (flagIs('--anchor-network') && hasValue) {
       anchorNetwork = takeValue();
-    } else if (arg === '--anchor-tx' && hasValue) {
+    } else if (flagIs('--anchor-tx') && hasValue) {
       // Path to a JSON file containing a fetched XRPL tx (with Memos) for the
       // on-chain leg of --anchor-verify. Offline-honest: omit it to run the
       // truncation check only.
       anchorTxFile = takeValue();
-    } else if (arg === '--anchor-trusted' && hasValue) {
+    } else if (flagIs('--anchor-trusted') && hasValue) {
       // Comma-separated trusted anchor accounts (UNIONed with the bundled list).
       anchorTrustedAccounts = takeValue().split(',').map((s) => s.trim()).filter(Boolean);
+    } else if (flagIs('-h') || flagIs('--help') || flagIs('--usage')) {
+      helpFlag = true;
+    } else if (arg.startsWith('--') && knownFlagName) {
+      // F-e0bcbc47 / F-INGEST-003 boundary: a KNOWN flag whose `&& hasValue`
+      // guard failed — i.e. a value flag given no value, because the next
+      // token was itself a flag (`--anchor-network --anchor-compute`) or it
+      // ended argv. F-INGEST-003 pins that the FOLLOWING flag is still parsed
+      // on its own rather than swallowed as this one's value, so this token
+      // keeps its historical path (fall through to positionalArgs, continue)
+      // and `--anchor-compute` runs. Rejecting here instead would break that
+      // pinned contract.
+      //
+      // RESIDUAL, STATED (not fixed here): the dangling flag is still
+      // IGNORED — `--anchor-network` with no value does not set the network
+      // and does not fail the run. That is pre-existing behavior and this
+      // amend does not change it. What changes is that it is no longer
+      // SILENT: an operator gets a structured warn naming the flag instead of
+      // the value vanishing with no trace. Making it a hard error is a real
+      // contract change (it would stop `--anchor-compute` from running and
+      // rewrite F-INGEST-003's pin), which is a bigger claim than a Stage C
+      // humanization amend should make unilaterally — filed as a follow-up
+      // rather than landed here.
+      logStage('warn', {
+        kind: 'cli_flag_missing_value',
+        flag: arg,
+        correlation_id: synthCorrelationId(),
+        message: `${arg} was given no value and is being ignored (the next token is a flag, not a value)`
+      });
+      positionalArgs.push(args[i]);
+    } else if (arg.startsWith('--')) {
+      // F-e0bcbc47 (Stage C humanization): a genuinely unrecognized `--flag`
+      // — no branch above knows this name at all. It used to fall through to
+      // the dead `positionalArgs` sink (declared, pushed to, and never read
+      // anywhere else in this file — confirmed by grep) and the CLI proceeded
+      // to read stdin, hit EOF, and crashed with a raw 'Unexpected end of
+      // JSON input' stack that misattributed the failure to the submission
+      // payload rather than the misspelled flag (live-proven trigger:
+      // `--provenance=stub --fiel <path>`, a one-character typo of --file).
+      // Reject at the point of the typo instead — matches the sibling pattern
+      // already correct in this same domain (packages/verify/cli.js:
+      // `unknown argument: --version` -> exit 2, no stack, no fallthrough).
+      // Scoped to `--`-prefixed tokens only (a bare positional stays in the
+      // historical positionalArgs sink) because that is exactly the shape a
+      // mistyped flag takes, and it is the shape the reachable repro above
+      // hits BEFORE any value token is even read.
+      emitCliErrorEvent({
+        failedStage: 'cli_parse_args',
+        correlationId: synthCorrelationId(),
+        err: new Error(`unknown argument: ${arg}`),
+        humanPrefix: 'invalid CLI invocation'
+      });
+      process.exit(2);
     } else {
       positionalArgs.push(args[i]);
     }
   }
 
+  // F-9a65b10c: print USAGE and exit 0 before any input resolution — help must
+  // never block on stdin or demand a --provenance flag.
+  if (helpFlag) {
+    console.log(USAGE);
+    process.exit(0);
+  }
+
   // --verify-chain is a standalone, side-effect-free audit: it reads only the
   // ledger + the record files it references, takes no submission, reads no
   // stdin, and needs no provenance adapter. Handle it BEFORE the stdin read and
@@ -684,7 +1119,12 @@ if (isMain) {
       // operator asked for it, so a grep of the NDJSON shows how many breaks and
       // orphans were found, not just the first break.
       ...(Array.isArray(result.breaks) ? { break_count: result.breaks.length } : {}),
-      ...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {})
+      ...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {}),
+      // F-29134790: pre-adoption records are excluded from chain_ok but still
+      // worth a grep-able count — an operator diffing orphan_count over time
+      // should see the pre-chain figure hold steady while orphan_count reflects
+      // only genuine torn writes.
+      ...(Array.isArray(result.pre_adoption) ? { pre_adoption_count: result.pre_adoption.length } : {})
     });
     const lines = formatChainResult(result);
     if (result.ok) {
@@ -744,6 +1184,19 @@ if (isMain) {
   }
 
   if (!submissionJson) {
+    // F-aa67ea9a (Stage C humanization): guard an interactive stdin. Without a
+    // piped payload, `for await (…process.stdin)` blocks forever at a TTY —
+    // exactly what a first-run operator who forgot to pipe input types — and
+    // looks like a hang. Fail fast with the same empty-state error verify/cli.js
+    // gives, naming the concrete input levers.
+    if (process.stdin.isTTY) {
+      console.error('ERROR: no submission provided');
+      console.error(
+        "  hint: pass --file <path> or --payload '<json>', or pipe JSON on stdin; " +
+        'run --help for usage.'
+      );
+      process.exit(2);
+    }
     // Read from stdin
     const chunks = [];
     for await (const chunk of process.stdin) {
@@ -791,7 +1244,7 @@ if (isMain) {
   let provenance;
   if (provenanceMode === 'stub') {
     // Structural anti-misuse: stub only allowed outside CI
-    if (process.env.CI || process.env.GITHUB_ACTIONS) {
+    if (isCI()) {
       emitCliErrorEvent({
         failedStage: 'cli_provenance_resolve',
         correlationId: cliCorrelationId,
@@ -819,7 +1272,7 @@ if (isMain) {
       process.exit(2);
     }
     provenance = resolved.provenance;
-  } else if (process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true') {
+  } else if (isCI()) {
     // In CI without an explicit flag: default to real provenance, routed by the
     // submission's source.provider (github | gitlab).
     const resolved = resolveProviderProvenance(submission);
@@ -845,6 +1298,34 @@ if (isMain) {
     process.exit(2);
   }
 
+  // F-3bfc2885: construct a scenario fetcher for the production path. Before
+  // this, the CLI never passed one, so scenario definitions — and with them the
+  // `step-results-present` / `step-verdict-consistent` required_steps gates —
+  // were test-only: an operator-declared severity:reject rule silently passed.
+  // Real (non-stub) provenance + a github-provider submission loads scenario
+  // definitions from the source repo at the persisted commit, reusing the token
+  // resolveProviderProvenance already validated. GitLab submissions have no
+  // scenario fetcher yet (no gitlab contents-API adapter exists); their
+  // required_steps gate remains unenforced — a known, documented gap.
+  // V2-INVARIAN-004: the eligibility condition is the exported, unit-tested
+  // resolveScenarioFetcherDecision above.
+  // F-b04473d5: log the decision — an operator auditing an accepted record
+  // must be able to tell "required_steps enforced" from "enforcement skipped
+  // (and why)" straight from the NDJSON stream.
+  const scenarioDecision = resolveScenarioFetcherDecision(provenance, submission, process.env);
+  const scenarioFetcher = scenarioDecision.fetcher;
+  logStage('scenario_enforcement', {
+    submission_id: submission && typeof submission === 'object' && !Array.isArray(submission)
+      ? (submission.run_id || null)
+      : null,
+    correlation_id: cliCorrelationId,
+    active: scenarioDecision.active,
+    ...(scenarioDecision.reason ? { reason: scenarioDecision.reason } : {}),
+    scenario_count: submission && typeof submission === 'object' && Array.isArray(submission.scenario_results)
+      ? submission.scenario_results.length
+      : 0
+  });
+
   // D1B-001: track the last successful pipeline stage so the outer catch
   // can surface a useful `failed_stage` in its structured error event.
   // We update it once the verify/ingest call has RETURNED — anything
@@ -856,7 +1337,7 @@ if (isMain) {
   // when they could.
   try {
     if (verifyOnlyFlag) {
-      const result = await verifyOnly(submission, { repoRoot, provenance });
+      const result = await verifyOnly(submission, { repoRoot, provenance, scenarioFetcher });
       lastSuccessfulStage = 'verify_only';
 
       console.log(JSON.stringify({
@@ -877,7 +1358,7 @@ if (isMain) {
       process.exit(result.record.verification.status === 'accepted' ? 0 : 1);
     }
 
-    const result = await ingest(submission, { repoRoot, provenance });
+    const result = await ingest(submission, { repoRoot, provenance, scenarioFetcher });
     lastSuccessfulStage = 'ingest';
 
     if (result.duplicate) {