@dogfood-lab/ingest 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/run.js CHANGED
@@ -22,14 +22,44 @@ import { fileURLToPath } from 'node:url';
22
22
  import { randomBytes } from 'node:crypto';
23
23
 
24
24
  import { verify } from '@dogfood-lab/verify';
25
- import { stubProvenance, githubProvenance } from '@dogfood-lab/verify/validators/provenance.js';
25
+ import { stubProvenance, provenanceForProvider } from '@dogfood-lab/verify/validators/provenance.js';
26
26
  import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
27
27
  import { loadGlobalPolicy, loadRepoPolicy, loadScenarios } from './load-context.js';
28
28
  import { isDuplicate, writeRecord, computeRecordPath } from './persist.js';
29
29
  import { rebuildIndexes } from './rebuild-indexes.js';
30
+ import { verifyChain, formatChainResult } from './verify-chain.js';
31
+ import { handleAnchorCompute, handleAnchorPost, handleAnchorVerify } from './anchor/cli.js';
30
32
 
31
33
  const __dirname = dirname(fileURLToPath(import.meta.url));
32
34
 
35
+ /**
36
+ * Resolve the REAL provenance adapter for a submission, routed by
37
+ * `submission.source.provider` (github | gitlab), sourcing the provider's token
38
+ * from the environment. Returns `{ provenance }` on success or `{ err }` (a
39
+ * structured, operator-legible Error the caller surfaces via emitCliErrorEvent
40
+ * + exit 2). The adapter registry (`provenanceForProvider`) is the single
41
+ * provider-keyed seam; a provider in the schema enum without a registered
42
+ * adapter fails here loudly rather than silently skipping verification.
43
+ *
44
+ * @param {object} submission
45
+ * @returns {{ provenance: object } | { err: Error }}
46
+ */
47
+ function resolveProviderProvenance(submission) {
48
+ const provider = (submission && submission.source && submission.source.provider) || 'github';
49
+ const factory = provenanceForProvider(provider);
50
+ if (!factory) {
51
+ return { err: new Error(`unknown provenance provider '${provider}' — no adapter registered (supported: github, gitlab).`) };
52
+ }
53
+ const token = provider === 'gitlab'
54
+ ? (process.env.GITLAB_TOKEN || process.env.CI_JOB_TOKEN)
55
+ : (process.env.GITHUB_TOKEN || process.env.GH_TOKEN);
56
+ if (!token) {
57
+ const need = provider === 'gitlab' ? 'GITLAB_TOKEN or CI_JOB_TOKEN' : 'GITHUB_TOKEN or GH_TOKEN';
58
+ return { err: new Error(`real provenance for provider '${provider}' requires ${need} in the environment.`) };
59
+ }
60
+ return { provenance: factory(token) };
61
+ }
62
+
33
63
  /**
34
64
  * SEED-1 (d3-ingest-003) — posixify a path-shaped value at the operator/log
35
65
  * SERIALIZATION boundary. `computeRecordPath`/`writeRecord` return OS-native
@@ -513,6 +543,25 @@ if (isMain) {
513
543
  let submissionJson;
514
544
  let provenanceMode = null;
515
545
  let verifyOnlyFlag = false;
546
+ let verifyChainFlag = false;
547
+ // --verify-chain modifiers (only meaningful alongside it):
548
+ // --reconcile → collectOrphans (INGEST-PROACT-001): also flag on-disk records
549
+ // absent from the ledger (a torn persist). Makes the audit fail on orphans.
550
+ // --all → collectAllBreaks (INGEST-PROACT-004): continue past the first
551
+ // per-record-independent break and report every break in one pass.
552
+ let reconcileFlag = false;
553
+ let allBreaksFlag = false;
554
+ // Anchor verbs (optional, off-by-default, operator-run). --anchor-compute and
555
+ // --anchor-verify are fully offline (never import xrpl); --anchor-post lazily
556
+ // loads the optional xrpl package and needs XRPL_SEED.
557
+ let anchorComputeFlag = false;
558
+ let anchorPostFlag = false;
559
+ let anchorVerifyFlag = false;
560
+ let anchorMode = 'since-last';
561
+ let anchorAlgo = null;
562
+ let anchorNetwork = null;
563
+ let anchorTxFile = null;
564
+ let anchorTrustedAccounts = [];
516
565
  const positionalArgs = [];
517
566
 
518
567
  for (let i = 0; i < args.length; i++) {
@@ -532,7 +581,13 @@ if (isMain) {
532
581
  arg = arg.slice(0, eq);
533
582
  }
534
583
  }
535
- const hasValue = inlineValue !== null || args[i + 1] !== undefined;
584
+ // A space-form value is the NEXT token only when it is not itself a flag —
585
+ // otherwise `--flag --next` would swallow `--next` as `--flag`'s value and
586
+ // silently drop it. A `--`-prefixed next token means this flag has no value,
587
+ // so it falls through to its "requires a value" path (for `--provenance`,
588
+ // the downstream "--provenance flag is required" error).
589
+ const nextIsValue = args[i + 1] !== undefined && !args[i + 1].startsWith('--');
590
+ const hasValue = inlineValue !== null || nextIsValue;
536
591
  const takeValue = () => (inlineValue !== null ? inlineValue : args[++i]);
537
592
 
538
593
  if (arg === '--provenance' && hasValue) {
@@ -563,11 +618,131 @@ if (isMain) {
563
618
  // F-252714-058: dry-run the pipeline without writing or rebuilding
564
619
  // indexes. CI / operators preview what WOULD have been persisted.
565
620
  verifyOnlyFlag = true;
621
+ } else if (arg === '--verify-chain') {
622
+ // Integrity chain v1: verify the append-only tamper-evident ledger at
623
+ // indexes/integrity/chain.jsonl, fully offline. No submission, no stdin,
624
+ // no provenance — a standalone audit command.
625
+ verifyChainFlag = true;
626
+ } else if (arg === '--reconcile') {
627
+ // Modifier for --verify-chain: also reconcile on-disk records against the
628
+ // ledger and fail on any orphan (INGEST-PROACT-001).
629
+ reconcileFlag = true;
630
+ } else if (arg === '--all') {
631
+ // Modifier for --verify-chain: report every per-record-independent break
632
+ // instead of stopping at the first (INGEST-PROACT-004).
633
+ allBreaksFlag = true;
634
+ } else if (arg === '--anchor-compute') {
635
+ // Optional XRPL anchor: compute + write the next anchor manifest. Offline.
636
+ anchorComputeFlag = true;
637
+ } else if (arg === '--anchor-post') {
638
+ // Optional XRPL anchor: compute if needed + post to XRPL. Needs the
639
+ // optional xrpl package (lazily loaded) and XRPL_SEED.
640
+ anchorPostFlag = true;
641
+ } else if (arg === '--anchor-verify') {
642
+ // Optional XRPL anchor: verify local manifests + run the truncation check.
643
+ // Offline reports honest NOT-verified for the on-chain leg.
644
+ anchorVerifyFlag = true;
645
+ } else if (arg === '--anchor-all') {
646
+ // Genesis snapshot mode for compute/post (covers the whole chain).
647
+ anchorMode = 'all';
648
+ } else if (arg === '--anchor-algo' && hasValue) {
649
+ anchorAlgo = takeValue();
650
+ } else if (arg === '--anchor-network' && hasValue) {
651
+ anchorNetwork = takeValue();
652
+ } else if (arg === '--anchor-tx' && hasValue) {
653
+ // Path to a JSON file containing a fetched XRPL tx (with Memos) for the
654
+ // on-chain leg of --anchor-verify. Offline-honest: omit it to run the
655
+ // truncation check only.
656
+ anchorTxFile = takeValue();
657
+ } else if (arg === '--anchor-trusted' && hasValue) {
658
+ // Comma-separated trusted anchor accounts (UNIONed with the bundled list).
659
+ anchorTrustedAccounts = takeValue().split(',').map((s) => s.trim()).filter(Boolean);
566
660
  } else {
567
661
  positionalArgs.push(args[i]);
568
662
  }
569
663
  }
570
664
 
665
+ // --verify-chain is a standalone, side-effect-free audit: it reads only the
666
+ // ledger + the record files it references, takes no submission, reads no
667
+ // stdin, and needs no provenance adapter. Handle it BEFORE the stdin read and
668
+ // provenance resolution so `node run.js --verify-chain` does not block on
669
+ // stdin or demand a --provenance flag. Exit 0 when the chain verifies, 1 on
670
+ // the first break (operator-legible output, no raw stack traces).
671
+ if (verifyChainFlag) {
672
+ const result = verifyChain(repoRoot, {
673
+ collectOrphans: reconcileFlag,
674
+ collectAllBreaks: allBreaksFlag,
675
+ });
676
+ logStage(result.ok ? 'verify_chain_complete' : 'error', {
677
+ correlation_id: synthCorrelationId(),
678
+ ...(result.ok ? {} : { failed_stage: 'verify_chain' }),
679
+ verified: result.count,
680
+ head_digest: result.head_digest,
681
+ chain_ok: result.ok,
682
+ ...(result.break ? { break_seq: result.break.seq, break_reason: result.break.reason } : {}),
683
+ // INGEST-PROACT-004 / -001: surface the full corruption scope when the
684
+ // operator asked for it, so a grep of the NDJSON shows how many breaks and
685
+ // orphans were found, not just the first break.
686
+ ...(Array.isArray(result.breaks) ? { break_count: result.breaks.length } : {}),
687
+ ...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {})
688
+ });
689
+ const lines = formatChainResult(result);
690
+ if (result.ok) {
691
+ for (const line of lines) console.log(line);
692
+ } else {
693
+ for (const line of lines) console.error(line);
694
+ }
695
+ process.exit(result.ok ? 0 : 1);
696
+ }
697
+
698
+ // Optional XRPL anchor verbs — operator-run, off by default, NOT in the normal
699
+ // ingest/CI path. Like --verify-chain these are standalone audit/operations:
700
+ // no submission, no stdin, no provenance adapter. --anchor-compute and
701
+ // --anchor-verify are fully offline (never import xrpl); --anchor-post lazily
702
+ // loads the optional xrpl package and needs XRPL_SEED. Each handler returns
703
+ // { ok, exitCode, lines, event } and run.js owns the console + logStage + exit.
704
+ if (anchorComputeFlag || anchorPostFlag || anchorVerifyFlag) {
705
+ const correlation_id = synthCorrelationId();
706
+ let result;
707
+ if (anchorComputeFlag) {
708
+ result = handleAnchorCompute(repoRoot, {
709
+ mode: anchorMode,
710
+ ...(anchorAlgo ? { algo: anchorAlgo } : {}),
711
+ ...(anchorNetwork ? { network: anchorNetwork } : {}),
712
+ });
713
+ } else if (anchorPostFlag) {
714
+ result = await handleAnchorPost(repoRoot, {
715
+ mode: anchorMode,
716
+ ...(anchorNetwork ? { network: anchorNetwork } : {}),
717
+ });
718
+ } else {
719
+ // --anchor-verify: optionally load a fetched tx JSON for the on-chain leg.
720
+ let tx;
721
+ if (anchorTxFile) {
722
+ const { readFileSync } = await import('node:fs');
723
+ try {
724
+ tx = JSON.parse(readFileSync(resolve(anchorTxFile), 'utf-8'));
725
+ } catch (err) {
726
+ emitCliErrorEvent({
727
+ failedStage: 'anchor_verify_read_tx',
728
+ correlationId: correlation_id,
729
+ err,
730
+ humanPrefix: 'could not read --anchor-tx file'
731
+ });
732
+ process.exit(2);
733
+ }
734
+ }
735
+ result = handleAnchorVerify(repoRoot, { tx, trustedAnchorAccounts: anchorTrustedAccounts });
736
+ }
737
+
738
+ // logStage strips any inner `stage:` field (the positional name wins), so
739
+ // spreading result.event — which carries its own `stage` — is safe.
740
+ logStage(result.event.stage, { correlation_id, ...result.event });
741
+ const sink = result.exitCode === 0 ? console.log : console.error;
742
+ for (const line of result.lines) sink(line);
743
+ process.exit(result.exitCode);
744
+ }
745
+
571
746
  if (!submissionJson) {
572
747
  // Read from stdin
573
748
  const chunks = [];
@@ -629,32 +804,36 @@ if (isMain) {
629
804
  console.error('WARNING: Using stub provenance (test/dev only). Records will NOT have real provenance verification.');
630
805
  provenance = stubProvenance;
631
806
  } else if (provenanceMode === 'github') {
632
- const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
633
- if (!token) {
807
+ // --provenance=github selects REAL provenance; the actual provider is taken
808
+ // from submission.source.provider, so a GitLab submission is confirmed via
809
+ // gitlabProvenance end-to-end (the adapter registry keys on the provider).
810
+ const resolved = resolveProviderProvenance(submission);
811
+ if (resolved.err) {
634
812
  emitCliErrorEvent({
635
813
  failedStage: 'cli_provenance_resolve',
636
814
  correlationId: cliCorrelationId,
637
815
  submissionId: submission && submission.run_id ? submission.run_id : null,
638
- err: new Error('--provenance=github requires GITHUB_TOKEN or GH_TOKEN environment variable.'),
816
+ err: resolved.err,
639
817
  humanPrefix: 'provenance precondition unmet'
640
818
  });
641
819
  process.exit(2);
642
820
  }
643
- provenance = githubProvenance(token);
821
+ provenance = resolved.provenance;
644
822
  } else if (process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true') {
645
- // In CI without explicit flag: default to github provenance, fail if no token
646
- const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
647
- if (!token) {
823
+ // In CI without an explicit flag: default to real provenance, routed by the
824
+ // submission's source.provider (github | gitlab).
825
+ const resolved = resolveProviderProvenance(submission);
826
+ if (resolved.err) {
648
827
  emitCliErrorEvent({
649
828
  failedStage: 'cli_provenance_resolve',
650
829
  correlationId: cliCorrelationId,
651
830
  submissionId: submission && submission.run_id ? submission.run_id : null,
652
- err: new Error('Running in CI without --provenance flag and no GITHUB_TOKEN. Cannot verify provenance.'),
831
+ err: resolved.err,
653
832
  humanPrefix: 'provenance precondition unmet'
654
833
  });
655
834
  process.exit(2);
656
835
  }
657
- provenance = githubProvenance(token);
836
+ provenance = resolved.provenance;
658
837
  } else {
659
838
  emitCliErrorEvent({
660
839
  failedStage: 'cli_provenance_resolve',
@@ -0,0 +1,316 @@
1
+ /**
2
+ * Chain verifier — the offline tamper-evidence gate.
3
+ *
4
+ * `verifyChain(repoRoot)` reads the append-only ledger at
5
+ * `indexes/integrity/chain.jsonl` and, FULLY OFFLINE (no network, no GitHub
6
+ * API), proves the chain is internally consistent:
7
+ *
8
+ * 1. For each entry, load the record file at `entry.path` and recompute its
9
+ * digest with the SAME `submissionDigest` helper persist used. Assert it
10
+ * equals BOTH `entry.submission_digest` (the ledger's claim) AND
11
+ * `record.integrity.submission_digest` (the record's self-claim). All three
12
+ * must agree — a tamper that touches the record but not the ledger, or the
13
+ * ledger but not the record, breaks one of these equalities.
14
+ * 2. Assert `entry.prev_digest` equals the PREVIOUS entry's
15
+ * `submission_digest` (GENESIS_DIGEST for seq 0) — the chain link.
16
+ * 3. Assert `seq` is 0,1,2,… strictly monotonic from 0.
17
+ *
18
+ * On the FIRST break it stops and reports `{ seq, run_id, reason }`; the caller
19
+ * (`--verify-chain` in run.js, or any consumer) exits non-zero. On success it
20
+ * returns the record count and head digest.
21
+ *
22
+ * Honesty (threat model): this is tamper-EVIDENT, not tamper-PROOF. An actor
23
+ * with the ingest write credential can rewrite a record, recompute its digest,
24
+ * and rewrite the matching ledger line — that re-verifies clean. The verifier
25
+ * catches any mutation to a record or ledger entry that is NOT consistently
26
+ * reflected in BOTH (an out-of-band edit, disk corruption, a partial restore, a
27
+ * push that touched a record but not the chain), and it catches middle-deletion,
28
+ * reorder, and forged-insertion (they break the seq run or the prev-link).
29
+ *
30
+ * KNOWN LIMITATION — tail truncation. Removing the most-recent entries (and
31
+ * their record files) leaves a SHORTER but internally-consistent chain, which
32
+ * verifies OK: the offline verifier has no external record of the expected head
33
+ * or leaf-count, so it can prove what remains is consistent but NOT that the
34
+ * chain is COMPLETE. Detecting truncation requires an external anchor of the
35
+ * head/count outside the writer's control — exactly what the optional XRPL
36
+ * anchor provides (its on-chain Merkle root + leaf count make any truncation
37
+ * BELOW an anchored point detectable). The offline chain alone is a completeness
38
+ * floor for in-place tampering, not a defense against tail truncation.
39
+ */
40
+
41
+ import { existsSync, readFileSync } from 'node:fs';
42
+ import { join, relative, sep } from 'node:path';
43
+
44
+ import { submissionDigest, GENESIS_DIGEST } from './lib/integrity.js';
45
+ import { readChainManifest } from './lib/chain-manifest.js';
46
+ import { findJsonFiles } from './rebuild-indexes.js';
47
+
48
+ /**
49
+ * @typedef {object} ChainBreak
50
+ * @property {number} seq - The seq at which verification failed.
51
+ * @property {string|null} run_id - The run_id of the failing entry, if known.
52
+ * @property {string} reason - Operator-legible description of what failed.
53
+ */
54
+
55
+ /**
56
+ * @typedef {object} ChainOrphan
57
+ * @property {string|null} run_id - The orphan record's run_id (from its
58
+ * integrity block or its body), if known.
59
+ * @property {number|null} seq - The orphan record's self-claimed integrity.seq.
60
+ * @property {string} path - Repo-relative, forward-slashed path to the orphan file.
61
+ * @property {string} reason - Operator-legible description of why it is an orphan.
62
+ */
63
+
64
+ /**
65
+ * @typedef {object} ChainVerifyResult
66
+ * @property {boolean} ok - True when the whole chain verifies (and, when
67
+ * collectOrphans is set, no orphan records exist).
68
+ * @property {number} count - Number of entries verified (0 for an empty chain).
69
+ * @property {string} head_digest - submission_digest of the last entry, or
70
+ * GENESIS_DIGEST for an empty chain.
71
+ * @property {ChainBreak|null} break - First break, or null when ok.
72
+ * @property {ChainBreak[]} [breaks] - All independent breaks (only present when
73
+ * `collectAllBreaks` is set). The first element equals `break`.
74
+ * @property {ChainOrphan[]} [orphans] - Records on disk but absent from the
75
+ * ledger (only present when `collectOrphans` is set).
76
+ */
77
+
78
+ /**
79
+ * Verify the integrity chain at `<repoRoot>/indexes/integrity/chain.jsonl`.
80
+ *
81
+ * @param {string} repoRoot - Absolute path to the testing-os repo root.
82
+ * @param {object} [opts]
83
+ * @param {boolean} [opts.collectAllBreaks=false] - INGEST-PROACT-004: continue
84
+ * past the FIRST break and return a `breaks[]` array of every
85
+ * per-record-independent break (digest-mismatch, missing-file). The default
86
+ * (false) stops at the first break — the fail-fast CI gate semantics. A
87
+ * structural break (non-monotonic seq, broken prev-link) still stops the
88
+ * walk even under this flag, because everything after it is unverifiable
89
+ * relative to a now-untrustworthy chain order.
90
+ * @param {boolean} [opts.collectOrphans=false] - INGEST-PROACT-001: after the
91
+ * chain walk, reconcile the on-disk records against the ledger and return an
92
+ * `orphans[]` array of any record file whose seq/run_id is absent from the
93
+ * ledger (a torn write between record-write and ledger-append). An orphan
94
+ * makes the result `ok: false` — the audit DETECTS the orphan instead of
95
+ * silently passing while a real record lives outside the tamper-evident chain.
96
+ * @returns {ChainVerifyResult}
97
+ */
98
+ export function verifyChain(repoRoot, opts = {}) {
99
+ const collectAllBreaks = opts.collectAllBreaks === true;
100
+ const collectOrphans = opts.collectOrphans === true;
101
+
102
+ let entries;
103
+ try {
104
+ entries = readChainManifest(repoRoot);
105
+ } catch (err) {
106
+ // A corrupt (non-JSON) manifest line is itself a tamper signal. We cannot
107
+ // trust the ledger at all here, so orphan reconciliation is not meaningful.
108
+ const brk = { seq: -1, run_id: null, reason: err.message };
109
+ return {
110
+ ok: false,
111
+ count: 0,
112
+ head_digest: GENESIS_DIGEST,
113
+ break: brk,
114
+ ...(collectAllBreaks ? { breaks: [brk] } : {}),
115
+ ...(collectOrphans ? { orphans: [] } : {}),
116
+ };
117
+ }
118
+
119
+ const headDigest = entries.length === 0
120
+ ? GENESIS_DIGEST
121
+ : (entries[entries.length - 1].submission_digest ?? GENESIS_DIGEST);
122
+
123
+ const breaks = [];
124
+ let prevDigest = GENESIS_DIGEST;
125
+
126
+ for (let i = 0; i < entries.length; i++) {
127
+ const entry = entries[i];
128
+ const seq = entry.seq;
129
+ const runId = entry.run_id ?? null;
130
+ const record = (reason) => breaks.push({ seq, run_id: runId, reason });
131
+
132
+ // (3) Monotonic seq and (2) chain-link are STRUCTURAL: a break here means
133
+ // the chain order itself is untrustworthy, so everything downstream is
134
+ // unverifiable relative to it. Record the break and stop — even under
135
+ // collectAllBreaks — rather than emitting a cascade of meaningless
136
+ // downstream prev-link errors.
137
+ if (seq !== i) {
138
+ record(`seq is not monotonic: expected ${i}, found ${seq}`);
139
+ break;
140
+ }
141
+ if (entry.prev_digest !== prevDigest) {
142
+ record(`broken prev_digest link: expected ${prevDigest}, found ${entry.prev_digest}`);
143
+ break;
144
+ }
145
+
146
+ // (1) Per-record-INDEPENDENT checks: missing-file and digest-mismatch.
147
+ // Each is judged on this record alone, so under collectAllBreaks we record
148
+ // the break and CONTINUE to surface the full corruption scope. Under the
149
+ // default we record the first and stop.
150
+ const recordPath = join(repoRoot, entry.path);
151
+ if (!existsSync(recordPath)) {
152
+ record(`record file missing at ${entry.path} (could not read to recompute digest)`);
153
+ if (!collectAllBreaks) break;
154
+ prevDigest = entry.submission_digest;
155
+ continue;
156
+ }
157
+
158
+ let parsed;
159
+ try {
160
+ parsed = JSON.parse(readFileSync(recordPath, 'utf-8'));
161
+ } catch (err) {
162
+ record(`record file at ${entry.path} is not valid JSON: ${err.message}`);
163
+ if (!collectAllBreaks) break;
164
+ prevDigest = entry.submission_digest;
165
+ continue;
166
+ }
167
+
168
+ const recomputed = submissionDigest(parsed);
169
+ if (recomputed !== entry.submission_digest) {
170
+ record(
171
+ `digest mismatch: recomputed ${recomputed} but manifest claims ${entry.submission_digest} ` +
172
+ `— record at ${entry.path} was modified after persist`
173
+ );
174
+ if (!collectAllBreaks) break;
175
+ prevDigest = entry.submission_digest;
176
+ continue;
177
+ }
178
+
179
+ const selfDigest = parsed.integrity?.submission_digest;
180
+ if (selfDigest !== recomputed) {
181
+ record(
182
+ `record self-digest mismatch: record.integrity.submission_digest is ` +
183
+ `${selfDigest ?? '(absent)'} but recomputed ${recomputed} — record at ${entry.path} ` +
184
+ `was modified after persist`
185
+ );
186
+ if (!collectAllBreaks) break;
187
+ prevDigest = entry.submission_digest;
188
+ continue;
189
+ }
190
+
191
+ prevDigest = entry.submission_digest;
192
+ }
193
+
194
+ const orphans = collectOrphans ? collectOrphanRecords(repoRoot, entries) : null;
195
+
196
+ const ok = breaks.length === 0 && (orphans === null || orphans.length === 0);
197
+ return {
198
+ ok,
199
+ count: entries.length,
200
+ head_digest: headDigest,
201
+ break: breaks.length > 0 ? breaks[0] : null,
202
+ ...(collectAllBreaks ? { breaks } : {}),
203
+ ...(collectOrphans ? { orphans } : {}),
204
+ };
205
+ }
206
+
207
+ /**
208
+ * INGEST-PROACT-001 — reconcile on-disk records against the ledger.
209
+ *
210
+ * Walks every `*.json` under `records/` (accepted AND `_rejected/`) and flags
211
+ * any record file whose `(run_id, seq)` identity is absent from the ledger.
212
+ * Such a file is an ORPHAN: persist wrote the record but the ledger append did
213
+ * not happen (a crash in the torn window between the two steps), so the record
214
+ * exists OUTSIDE the tamper-evident chain and the plain chain-walk is blind to
215
+ * it. The chain-manifest's own ledger lines are NOT under `records/`, so they
216
+ * are never mistaken for orphans.
217
+ *
218
+ * @param {string} repoRoot
219
+ * @param {Array<object>} entries - The ledger entries (already read).
220
+ * @returns {ChainOrphan[]}
221
+ */
222
+ function collectOrphanRecords(repoRoot, entries) {
223
+ // Index the ledger by run_id and by repo-relative path so a record matches if
224
+ // EITHER identity is present — a record is ledgered as long as its line exists.
225
+ const ledgerRunIds = new Set();
226
+ const ledgerPaths = new Set();
227
+ for (const e of entries) {
228
+ if (e.run_id) ledgerRunIds.add(e.run_id);
229
+ if (e.path) ledgerPaths.add(e.path);
230
+ }
231
+
232
+ const orphans = [];
233
+ const recordsDir = join(repoRoot, 'records');
234
+ for (const absPath of findJsonFiles(recordsDir)) {
235
+ const relPath = relative(repoRoot, absPath).split(sep).join('/');
236
+ if (ledgerPaths.has(relPath)) continue;
237
+
238
+ let record;
239
+ try {
240
+ record = JSON.parse(readFileSync(absPath, 'utf-8'));
241
+ } catch {
242
+ // A record file we cannot even parse, that is also absent from the ledger,
243
+ // is still an orphan — report it with what little identity we have.
244
+ orphans.push({
245
+ run_id: null,
246
+ seq: null,
247
+ path: relPath,
248
+ reason: `record file present on disk but absent from the ledger, and not valid JSON`,
249
+ });
250
+ continue;
251
+ }
252
+
253
+ const runId = record.run_id ?? null;
254
+ if (runId && ledgerRunIds.has(runId)) continue;
255
+
256
+ orphans.push({
257
+ run_id: runId,
258
+ seq: record.integrity?.seq ?? null,
259
+ path: relPath,
260
+ reason:
261
+ `record present on disk but absent from the ledger — a torn persist ` +
262
+ `(record written, ledger append missed). Re-run ingest for this record ` +
263
+ `or rebuild the chain to admit it into the tamper-evident ledger.`,
264
+ });
265
+ }
266
+
267
+ return orphans;
268
+ }
269
+
270
+ /**
271
+ * Render a verifyChain result as operator-legible lines (no raw stack traces).
272
+ * Returned as an array so callers can choose the sink (console.log/console.error).
273
+ *
274
+ * @param {ChainVerifyResult} result
275
+ * @returns {string[]}
276
+ */
277
+ export function formatChainResult(result) {
278
+ if (result.ok) {
279
+ const lines = [
280
+ `integrity chain OK: ${result.count} record(s) verified`,
281
+ `head digest: ${result.head_digest}`,
282
+ ];
283
+ if (Array.isArray(result.orphans)) {
284
+ lines.push(`reconciliation: 0 orphan record(s) on disk`);
285
+ }
286
+ return lines;
287
+ }
288
+
289
+ const lines = [];
290
+
291
+ // INGEST-PROACT-004: when collectAllBreaks populated breaks[], list every
292
+ // independent break so the operator sees the full corruption scope in one
293
+ // pass. Otherwise report the single first break (the CI-gate default).
294
+ const allBreaks = Array.isArray(result.breaks) ? result.breaks : (result.break ? [result.break] : []);
295
+ if (allBreaks.length > 1) {
296
+ lines.push(`integrity chain BROKEN: ${allBreaks.length} break(s) found`);
297
+ for (const b of allBreaks) {
298
+ lines.push(` - seq ${b.seq}${b.run_id ? ` (run_id: ${b.run_id})` : ''}: ${b.reason}`);
299
+ }
300
+ } else if (allBreaks.length === 1) {
301
+ const b = allBreaks[0];
302
+ lines.push(`integrity chain BROKEN at seq ${b.seq}${b.run_id ? ` (run_id: ${b.run_id})` : ''}`);
303
+ lines.push(`reason: ${b.reason}`);
304
+ }
305
+
306
+ // INGEST-PROACT-001: an orphan makes the result not-ok even with zero chain
307
+ // breaks — name each orphan file + run_id and the recovery action.
308
+ if (Array.isArray(result.orphans) && result.orphans.length > 0) {
309
+ lines.push(`reconciliation: ${result.orphans.length} orphan record(s) on disk but absent from the ledger`);
310
+ for (const o of result.orphans) {
311
+ lines.push(` - ${o.path}${o.run_id ? ` (run_id: ${o.run_id})` : ''}: ${o.reason}`);
312
+ }
313
+ }
314
+
315
+ return lines;
316
+ }