@dogfood-lab/ingest 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -22
- package/anchor/cli.js +215 -0
- package/anchor/compute-root.js +302 -0
- package/anchor/config.js +44 -0
- package/anchor/merkle.js +123 -0
- package/anchor/post-anchor.js +328 -0
- package/anchor/verify-anchor.js +358 -0
- package/lib/chain-manifest.js +106 -0
- package/lib/integrity.js +98 -0
- package/load-context.js +69 -13
- package/package.json +6 -2
- package/persist.js +41 -1
- package/rebuild-indexes.js +218 -16
- package/run.js +190 -11
- package/verify-chain.js +316 -0
package/run.js
CHANGED
|
@@ -22,14 +22,44 @@ import { fileURLToPath } from 'node:url';
|
|
|
22
22
|
import { randomBytes } from 'node:crypto';
|
|
23
23
|
|
|
24
24
|
import { verify } from '@dogfood-lab/verify';
|
|
25
|
-
import { stubProvenance,
|
|
25
|
+
import { stubProvenance, provenanceForProvider } from '@dogfood-lab/verify/validators/provenance.js';
|
|
26
26
|
import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
|
|
27
27
|
import { loadGlobalPolicy, loadRepoPolicy, loadScenarios } from './load-context.js';
|
|
28
28
|
import { isDuplicate, writeRecord, computeRecordPath } from './persist.js';
|
|
29
29
|
import { rebuildIndexes } from './rebuild-indexes.js';
|
|
30
|
+
import { verifyChain, formatChainResult } from './verify-chain.js';
|
|
31
|
+
import { handleAnchorCompute, handleAnchorPost, handleAnchorVerify } from './anchor/cli.js';
|
|
30
32
|
|
|
31
33
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
32
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Resolve the REAL provenance adapter for a submission, routed by
|
|
37
|
+
* `submission.source.provider` (github | gitlab), sourcing the provider's token
|
|
38
|
+
* from the environment. Returns `{ provenance }` on success or `{ err }` (a
|
|
39
|
+
* structured, operator-legible Error the caller surfaces via emitCliErrorEvent
|
|
40
|
+
* + exit 2). The adapter registry (`provenanceForProvider`) is the single
|
|
41
|
+
* provider-keyed seam; a provider in the schema enum without a registered
|
|
42
|
+
* adapter fails here loudly rather than silently skipping verification.
|
|
43
|
+
*
|
|
44
|
+
* @param {object} submission
|
|
45
|
+
* @returns {{ provenance: object } | { err: Error }}
|
|
46
|
+
*/
|
|
47
|
+
function resolveProviderProvenance(submission) {
|
|
48
|
+
const provider = (submission && submission.source && submission.source.provider) || 'github';
|
|
49
|
+
const factory = provenanceForProvider(provider);
|
|
50
|
+
if (!factory) {
|
|
51
|
+
return { err: new Error(`unknown provenance provider '${provider}' — no adapter registered (supported: github, gitlab).`) };
|
|
52
|
+
}
|
|
53
|
+
const token = provider === 'gitlab'
|
|
54
|
+
? (process.env.GITLAB_TOKEN || process.env.CI_JOB_TOKEN)
|
|
55
|
+
: (process.env.GITHUB_TOKEN || process.env.GH_TOKEN);
|
|
56
|
+
if (!token) {
|
|
57
|
+
const need = provider === 'gitlab' ? 'GITLAB_TOKEN or CI_JOB_TOKEN' : 'GITHUB_TOKEN or GH_TOKEN';
|
|
58
|
+
return { err: new Error(`real provenance for provider '${provider}' requires ${need} in the environment.`) };
|
|
59
|
+
}
|
|
60
|
+
return { provenance: factory(token) };
|
|
61
|
+
}
|
|
62
|
+
|
|
33
63
|
/**
|
|
34
64
|
* SEED-1 (d3-ingest-003) — posixify a path-shaped value at the operator/log
|
|
35
65
|
* SERIALIZATION boundary. `computeRecordPath`/`writeRecord` return OS-native
|
|
@@ -513,6 +543,25 @@ if (isMain) {
|
|
|
513
543
|
let submissionJson;
|
|
514
544
|
let provenanceMode = null;
|
|
515
545
|
let verifyOnlyFlag = false;
|
|
546
|
+
let verifyChainFlag = false;
|
|
547
|
+
// --verify-chain modifiers (only meaningful alongside it):
|
|
548
|
+
// --reconcile → collectOrphans (INGEST-PROACT-001): also flag on-disk records
|
|
549
|
+
// absent from the ledger (a torn persist). Makes the audit fail on orphans.
|
|
550
|
+
// --all → collectAllBreaks (INGEST-PROACT-004): continue past the first
|
|
551
|
+
// per-record-independent break and report every break in one pass.
|
|
552
|
+
let reconcileFlag = false;
|
|
553
|
+
let allBreaksFlag = false;
|
|
554
|
+
// Anchor verbs (optional, off-by-default, operator-run). --anchor-compute and
|
|
555
|
+
// --anchor-verify are fully offline (never import xrpl); --anchor-post lazily
|
|
556
|
+
// loads the optional xrpl package and needs XRPL_SEED.
|
|
557
|
+
let anchorComputeFlag = false;
|
|
558
|
+
let anchorPostFlag = false;
|
|
559
|
+
let anchorVerifyFlag = false;
|
|
560
|
+
let anchorMode = 'since-last';
|
|
561
|
+
let anchorAlgo = null;
|
|
562
|
+
let anchorNetwork = null;
|
|
563
|
+
let anchorTxFile = null;
|
|
564
|
+
let anchorTrustedAccounts = [];
|
|
516
565
|
const positionalArgs = [];
|
|
517
566
|
|
|
518
567
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -532,7 +581,13 @@ if (isMain) {
|
|
|
532
581
|
arg = arg.slice(0, eq);
|
|
533
582
|
}
|
|
534
583
|
}
|
|
535
|
-
|
|
584
|
+
// A space-form value is the NEXT token only when it is not itself a flag —
|
|
585
|
+
// otherwise `--flag --next` would swallow `--next` as `--flag`'s value and
|
|
586
|
+
// silently drop it. A `--`-prefixed next token means this flag has no value,
|
|
587
|
+
// so it falls through to its "requires a value" path (for `--provenance`,
|
|
588
|
+
// the downstream "--provenance flag is required" error).
|
|
589
|
+
const nextIsValue = args[i + 1] !== undefined && !args[i + 1].startsWith('--');
|
|
590
|
+
const hasValue = inlineValue !== null || nextIsValue;
|
|
536
591
|
const takeValue = () => (inlineValue !== null ? inlineValue : args[++i]);
|
|
537
592
|
|
|
538
593
|
if (arg === '--provenance' && hasValue) {
|
|
@@ -563,11 +618,131 @@ if (isMain) {
|
|
|
563
618
|
// F-252714-058: dry-run the pipeline without writing or rebuilding
|
|
564
619
|
// indexes. CI / operators preview what WOULD have been persisted.
|
|
565
620
|
verifyOnlyFlag = true;
|
|
621
|
+
} else if (arg === '--verify-chain') {
|
|
622
|
+
// Integrity chain v1: verify the append-only tamper-evident ledger at
|
|
623
|
+
// indexes/integrity/chain.jsonl, fully offline. No submission, no stdin,
|
|
624
|
+
// no provenance — a standalone audit command.
|
|
625
|
+
verifyChainFlag = true;
|
|
626
|
+
} else if (arg === '--reconcile') {
|
|
627
|
+
// Modifier for --verify-chain: also reconcile on-disk records against the
|
|
628
|
+
// ledger and fail on any orphan (INGEST-PROACT-001).
|
|
629
|
+
reconcileFlag = true;
|
|
630
|
+
} else if (arg === '--all') {
|
|
631
|
+
// Modifier for --verify-chain: report every per-record-independent break
|
|
632
|
+
// instead of stopping at the first (INGEST-PROACT-004).
|
|
633
|
+
allBreaksFlag = true;
|
|
634
|
+
} else if (arg === '--anchor-compute') {
|
|
635
|
+
// Optional XRPL anchor: compute + write the next anchor manifest. Offline.
|
|
636
|
+
anchorComputeFlag = true;
|
|
637
|
+
} else if (arg === '--anchor-post') {
|
|
638
|
+
// Optional XRPL anchor: compute if needed + post to XRPL. Needs the
|
|
639
|
+
// optional xrpl package (lazily loaded) and XRPL_SEED.
|
|
640
|
+
anchorPostFlag = true;
|
|
641
|
+
} else if (arg === '--anchor-verify') {
|
|
642
|
+
// Optional XRPL anchor: verify local manifests + run the truncation check.
|
|
643
|
+
// Offline reports honest NOT-verified for the on-chain leg.
|
|
644
|
+
anchorVerifyFlag = true;
|
|
645
|
+
} else if (arg === '--anchor-all') {
|
|
646
|
+
// Genesis snapshot mode for compute/post (covers the whole chain).
|
|
647
|
+
anchorMode = 'all';
|
|
648
|
+
} else if (arg === '--anchor-algo' && hasValue) {
|
|
649
|
+
anchorAlgo = takeValue();
|
|
650
|
+
} else if (arg === '--anchor-network' && hasValue) {
|
|
651
|
+
anchorNetwork = takeValue();
|
|
652
|
+
} else if (arg === '--anchor-tx' && hasValue) {
|
|
653
|
+
// Path to a JSON file containing a fetched XRPL tx (with Memos) for the
|
|
654
|
+
// on-chain leg of --anchor-verify. Offline-honest: omit it to run the
|
|
655
|
+
// truncation check only.
|
|
656
|
+
anchorTxFile = takeValue();
|
|
657
|
+
} else if (arg === '--anchor-trusted' && hasValue) {
|
|
658
|
+
// Comma-separated trusted anchor accounts (UNIONed with the bundled list).
|
|
659
|
+
anchorTrustedAccounts = takeValue().split(',').map((s) => s.trim()).filter(Boolean);
|
|
566
660
|
} else {
|
|
567
661
|
positionalArgs.push(args[i]);
|
|
568
662
|
}
|
|
569
663
|
}
|
|
570
664
|
|
|
665
|
+
// --verify-chain is a standalone, side-effect-free audit: it reads only the
|
|
666
|
+
// ledger + the record files it references, takes no submission, reads no
|
|
667
|
+
// stdin, and needs no provenance adapter. Handle it BEFORE the stdin read and
|
|
668
|
+
// provenance resolution so `node run.js --verify-chain` does not block on
|
|
669
|
+
// stdin or demand a --provenance flag. Exit 0 when the chain verifies, 1 on
|
|
670
|
+
// the first break (operator-legible output, no raw stack traces).
|
|
671
|
+
if (verifyChainFlag) {
|
|
672
|
+
const result = verifyChain(repoRoot, {
|
|
673
|
+
collectOrphans: reconcileFlag,
|
|
674
|
+
collectAllBreaks: allBreaksFlag,
|
|
675
|
+
});
|
|
676
|
+
logStage(result.ok ? 'verify_chain_complete' : 'error', {
|
|
677
|
+
correlation_id: synthCorrelationId(),
|
|
678
|
+
...(result.ok ? {} : { failed_stage: 'verify_chain' }),
|
|
679
|
+
verified: result.count,
|
|
680
|
+
head_digest: result.head_digest,
|
|
681
|
+
chain_ok: result.ok,
|
|
682
|
+
...(result.break ? { break_seq: result.break.seq, break_reason: result.break.reason } : {}),
|
|
683
|
+
// INGEST-PROACT-004 / -001: surface the full corruption scope when the
|
|
684
|
+
// operator asked for it, so a grep of the NDJSON shows how many breaks and
|
|
685
|
+
// orphans were found, not just the first break.
|
|
686
|
+
...(Array.isArray(result.breaks) ? { break_count: result.breaks.length } : {}),
|
|
687
|
+
...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {})
|
|
688
|
+
});
|
|
689
|
+
const lines = formatChainResult(result);
|
|
690
|
+
if (result.ok) {
|
|
691
|
+
for (const line of lines) console.log(line);
|
|
692
|
+
} else {
|
|
693
|
+
for (const line of lines) console.error(line);
|
|
694
|
+
}
|
|
695
|
+
process.exit(result.ok ? 0 : 1);
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
// Optional XRPL anchor verbs — operator-run, off by default, NOT in the normal
|
|
699
|
+
// ingest/CI path. Like --verify-chain these are standalone audit/operations:
|
|
700
|
+
// no submission, no stdin, no provenance adapter. --anchor-compute and
|
|
701
|
+
// --anchor-verify are fully offline (never import xrpl); --anchor-post lazily
|
|
702
|
+
// loads the optional xrpl package and needs XRPL_SEED. Each handler returns
|
|
703
|
+
// { ok, exitCode, lines, event } and run.js owns the console + logStage + exit.
|
|
704
|
+
if (anchorComputeFlag || anchorPostFlag || anchorVerifyFlag) {
|
|
705
|
+
const correlation_id = synthCorrelationId();
|
|
706
|
+
let result;
|
|
707
|
+
if (anchorComputeFlag) {
|
|
708
|
+
result = handleAnchorCompute(repoRoot, {
|
|
709
|
+
mode: anchorMode,
|
|
710
|
+
...(anchorAlgo ? { algo: anchorAlgo } : {}),
|
|
711
|
+
...(anchorNetwork ? { network: anchorNetwork } : {}),
|
|
712
|
+
});
|
|
713
|
+
} else if (anchorPostFlag) {
|
|
714
|
+
result = await handleAnchorPost(repoRoot, {
|
|
715
|
+
mode: anchorMode,
|
|
716
|
+
...(anchorNetwork ? { network: anchorNetwork } : {}),
|
|
717
|
+
});
|
|
718
|
+
} else {
|
|
719
|
+
// --anchor-verify: optionally load a fetched tx JSON for the on-chain leg.
|
|
720
|
+
let tx;
|
|
721
|
+
if (anchorTxFile) {
|
|
722
|
+
const { readFileSync } = await import('node:fs');
|
|
723
|
+
try {
|
|
724
|
+
tx = JSON.parse(readFileSync(resolve(anchorTxFile), 'utf-8'));
|
|
725
|
+
} catch (err) {
|
|
726
|
+
emitCliErrorEvent({
|
|
727
|
+
failedStage: 'anchor_verify_read_tx',
|
|
728
|
+
correlationId: correlation_id,
|
|
729
|
+
err,
|
|
730
|
+
humanPrefix: 'could not read --anchor-tx file'
|
|
731
|
+
});
|
|
732
|
+
process.exit(2);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
result = handleAnchorVerify(repoRoot, { tx, trustedAnchorAccounts: anchorTrustedAccounts });
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// logStage strips any inner `stage:` field (the positional name wins), so
|
|
739
|
+
// spreading result.event — which carries its own `stage` — is safe.
|
|
740
|
+
logStage(result.event.stage, { correlation_id, ...result.event });
|
|
741
|
+
const sink = result.exitCode === 0 ? console.log : console.error;
|
|
742
|
+
for (const line of result.lines) sink(line);
|
|
743
|
+
process.exit(result.exitCode);
|
|
744
|
+
}
|
|
745
|
+
|
|
571
746
|
if (!submissionJson) {
|
|
572
747
|
// Read from stdin
|
|
573
748
|
const chunks = [];
|
|
@@ -629,32 +804,36 @@ if (isMain) {
|
|
|
629
804
|
console.error('WARNING: Using stub provenance (test/dev only). Records will NOT have real provenance verification.');
|
|
630
805
|
provenance = stubProvenance;
|
|
631
806
|
} else if (provenanceMode === 'github') {
|
|
632
|
-
|
|
633
|
-
|
|
807
|
+
// --provenance=github selects REAL provenance; the actual provider is taken
|
|
808
|
+
// from submission.source.provider, so a GitLab submission is confirmed via
|
|
809
|
+
// gitlabProvenance end-to-end (the adapter registry keys on the provider).
|
|
810
|
+
const resolved = resolveProviderProvenance(submission);
|
|
811
|
+
if (resolved.err) {
|
|
634
812
|
emitCliErrorEvent({
|
|
635
813
|
failedStage: 'cli_provenance_resolve',
|
|
636
814
|
correlationId: cliCorrelationId,
|
|
637
815
|
submissionId: submission && submission.run_id ? submission.run_id : null,
|
|
638
|
-
err:
|
|
816
|
+
err: resolved.err,
|
|
639
817
|
humanPrefix: 'provenance precondition unmet'
|
|
640
818
|
});
|
|
641
819
|
process.exit(2);
|
|
642
820
|
}
|
|
643
|
-
provenance =
|
|
821
|
+
provenance = resolved.provenance;
|
|
644
822
|
} else if (process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true') {
|
|
645
|
-
// In CI without explicit flag: default to
|
|
646
|
-
|
|
647
|
-
|
|
823
|
+
// In CI without an explicit flag: default to real provenance, routed by the
|
|
824
|
+
// submission's source.provider (github | gitlab).
|
|
825
|
+
const resolved = resolveProviderProvenance(submission);
|
|
826
|
+
if (resolved.err) {
|
|
648
827
|
emitCliErrorEvent({
|
|
649
828
|
failedStage: 'cli_provenance_resolve',
|
|
650
829
|
correlationId: cliCorrelationId,
|
|
651
830
|
submissionId: submission && submission.run_id ? submission.run_id : null,
|
|
652
|
-
err:
|
|
831
|
+
err: resolved.err,
|
|
653
832
|
humanPrefix: 'provenance precondition unmet'
|
|
654
833
|
});
|
|
655
834
|
process.exit(2);
|
|
656
835
|
}
|
|
657
|
-
provenance =
|
|
836
|
+
provenance = resolved.provenance;
|
|
658
837
|
} else {
|
|
659
838
|
emitCliErrorEvent({
|
|
660
839
|
failedStage: 'cli_provenance_resolve',
|
package/verify-chain.js
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chain verifier — the offline tamper-evidence gate.
|
|
3
|
+
*
|
|
4
|
+
* `verifyChain(repoRoot)` reads the append-only ledger at
|
|
5
|
+
* `indexes/integrity/chain.jsonl` and, FULLY OFFLINE (no network, no GitHub
|
|
6
|
+
* API), proves the chain is internally consistent:
|
|
7
|
+
*
|
|
8
|
+
* 1. For each entry, load the record file at `entry.path` and recompute its
|
|
9
|
+
* digest with the SAME `submissionDigest` helper persist used. Assert it
|
|
10
|
+
* equals BOTH `entry.submission_digest` (the ledger's claim) AND
|
|
11
|
+
* `record.integrity.submission_digest` (the record's self-claim). All three
|
|
12
|
+
* must agree — a tamper that touches the record but not the ledger, or the
|
|
13
|
+
* ledger but not the record, breaks one of these equalities.
|
|
14
|
+
* 2. Assert `entry.prev_digest` equals the PREVIOUS entry's
|
|
15
|
+
* `submission_digest` (GENESIS_DIGEST for seq 0) — the chain link.
|
|
16
|
+
* 3. Assert `seq` is 0,1,2,… strictly monotonic from 0.
|
|
17
|
+
*
|
|
18
|
+
* On the FIRST break it stops and reports `{ seq, run_id, reason }`; the caller
|
|
19
|
+
* (`--verify-chain` in run.js, or any consumer) exits non-zero. On success it
|
|
20
|
+
* returns the record count and head digest.
|
|
21
|
+
*
|
|
22
|
+
* Honesty (threat model): this is tamper-EVIDENT, not tamper-PROOF. An actor
|
|
23
|
+
* with the ingest write credential can rewrite a record, recompute its digest,
|
|
24
|
+
* and rewrite the matching ledger line — that re-verifies clean. The verifier
|
|
25
|
+
* catches any mutation to a record or ledger entry that is NOT consistently
|
|
26
|
+
* reflected in BOTH (an out-of-band edit, disk corruption, a partial restore, a
|
|
27
|
+
* push that touched a record but not the chain), and it catches middle-deletion,
|
|
28
|
+
* reorder, and forged-insertion (they break the seq run or the prev-link).
|
|
29
|
+
*
|
|
30
|
+
* KNOWN LIMITATION — tail truncation. Removing the most-recent entries (and
|
|
31
|
+
* their record files) leaves a SHORTER but internally-consistent chain, which
|
|
32
|
+
* verifies OK: the offline verifier has no external record of the expected head
|
|
33
|
+
* or leaf-count, so it can prove what remains is consistent but NOT that the
|
|
34
|
+
* chain is COMPLETE. Detecting truncation requires an external anchor of the
|
|
35
|
+
* head/count outside the writer's control — exactly what the optional XRPL
|
|
36
|
+
* anchor provides (its on-chain Merkle root + leaf count make any truncation
|
|
37
|
+
* BELOW an anchored point detectable). The offline chain alone is a completeness
|
|
38
|
+
* floor for in-place tampering, not a defense against tail truncation.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
42
|
+
import { join, relative, sep } from 'node:path';
|
|
43
|
+
|
|
44
|
+
import { submissionDigest, GENESIS_DIGEST } from './lib/integrity.js';
|
|
45
|
+
import { readChainManifest } from './lib/chain-manifest.js';
|
|
46
|
+
import { findJsonFiles } from './rebuild-indexes.js';
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* @typedef {object} ChainBreak
|
|
50
|
+
* @property {number} seq - The seq at which verification failed.
|
|
51
|
+
* @property {string|null} run_id - The run_id of the failing entry, if known.
|
|
52
|
+
* @property {string} reason - Operator-legible description of what failed.
|
|
53
|
+
*/
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* @typedef {object} ChainOrphan
|
|
57
|
+
* @property {string|null} run_id - The orphan record's run_id (from its
|
|
58
|
+
* integrity block or its body), if known.
|
|
59
|
+
* @property {number|null} seq - The orphan record's self-claimed integrity.seq.
|
|
60
|
+
* @property {string} path - Repo-relative, forward-slashed path to the orphan file.
|
|
61
|
+
* @property {string} reason - Operator-legible description of why it is an orphan.
|
|
62
|
+
*/
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* @typedef {object} ChainVerifyResult
|
|
66
|
+
* @property {boolean} ok - True when the whole chain verifies (and, when
|
|
67
|
+
* collectOrphans is set, no orphan records exist).
|
|
68
|
+
* @property {number} count - Number of entries verified (0 for an empty chain).
|
|
69
|
+
* @property {string} head_digest - submission_digest of the last entry, or
|
|
70
|
+
* GENESIS_DIGEST for an empty chain.
|
|
71
|
+
* @property {ChainBreak|null} break - First break, or null when ok.
|
|
72
|
+
* @property {ChainBreak[]} [breaks] - All independent breaks (only present when
|
|
73
|
+
* `collectAllBreaks` is set). The first element equals `break`.
|
|
74
|
+
* @property {ChainOrphan[]} [orphans] - Records on disk but absent from the
|
|
75
|
+
* ledger (only present when `collectOrphans` is set).
|
|
76
|
+
*/
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Verify the integrity chain at `<repoRoot>/indexes/integrity/chain.jsonl`.
|
|
80
|
+
*
|
|
81
|
+
* @param {string} repoRoot - Absolute path to the testing-os repo root.
|
|
82
|
+
* @param {object} [opts]
|
|
83
|
+
* @param {boolean} [opts.collectAllBreaks=false] - INGEST-PROACT-004: continue
|
|
84
|
+
* past the FIRST break and return a `breaks[]` array of every
|
|
85
|
+
* per-record-independent break (digest-mismatch, missing-file). The default
|
|
86
|
+
* (false) stops at the first break — the fail-fast CI gate semantics. A
|
|
87
|
+
* structural break (non-monotonic seq, broken prev-link) still stops the
|
|
88
|
+
* walk even under this flag, because everything after it is unverifiable
|
|
89
|
+
* relative to a now-untrustworthy chain order.
|
|
90
|
+
* @param {boolean} [opts.collectOrphans=false] - INGEST-PROACT-001: after the
|
|
91
|
+
* chain walk, reconcile the on-disk records against the ledger and return an
|
|
92
|
+
* `orphans[]` array of any record file whose seq/run_id is absent from the
|
|
93
|
+
* ledger (a torn write between record-write and ledger-append). An orphan
|
|
94
|
+
* makes the result `ok: false` — the audit DETECTS the orphan instead of
|
|
95
|
+
* silently passing while a real record lives outside the tamper-evident chain.
|
|
96
|
+
* @returns {ChainVerifyResult}
|
|
97
|
+
*/
|
|
98
|
+
export function verifyChain(repoRoot, opts = {}) {
|
|
99
|
+
const collectAllBreaks = opts.collectAllBreaks === true;
|
|
100
|
+
const collectOrphans = opts.collectOrphans === true;
|
|
101
|
+
|
|
102
|
+
let entries;
|
|
103
|
+
try {
|
|
104
|
+
entries = readChainManifest(repoRoot);
|
|
105
|
+
} catch (err) {
|
|
106
|
+
// A corrupt (non-JSON) manifest line is itself a tamper signal. We cannot
|
|
107
|
+
// trust the ledger at all here, so orphan reconciliation is not meaningful.
|
|
108
|
+
const brk = { seq: -1, run_id: null, reason: err.message };
|
|
109
|
+
return {
|
|
110
|
+
ok: false,
|
|
111
|
+
count: 0,
|
|
112
|
+
head_digest: GENESIS_DIGEST,
|
|
113
|
+
break: brk,
|
|
114
|
+
...(collectAllBreaks ? { breaks: [brk] } : {}),
|
|
115
|
+
...(collectOrphans ? { orphans: [] } : {}),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const headDigest = entries.length === 0
|
|
120
|
+
? GENESIS_DIGEST
|
|
121
|
+
: (entries[entries.length - 1].submission_digest ?? GENESIS_DIGEST);
|
|
122
|
+
|
|
123
|
+
const breaks = [];
|
|
124
|
+
let prevDigest = GENESIS_DIGEST;
|
|
125
|
+
|
|
126
|
+
for (let i = 0; i < entries.length; i++) {
|
|
127
|
+
const entry = entries[i];
|
|
128
|
+
const seq = entry.seq;
|
|
129
|
+
const runId = entry.run_id ?? null;
|
|
130
|
+
const record = (reason) => breaks.push({ seq, run_id: runId, reason });
|
|
131
|
+
|
|
132
|
+
// (3) Monotonic seq and (2) chain-link are STRUCTURAL: a break here means
|
|
133
|
+
// the chain order itself is untrustworthy, so everything downstream is
|
|
134
|
+
// unverifiable relative to it. Record the break and stop — even under
|
|
135
|
+
// collectAllBreaks — rather than emitting a cascade of meaningless
|
|
136
|
+
// downstream prev-link errors.
|
|
137
|
+
if (seq !== i) {
|
|
138
|
+
record(`seq is not monotonic: expected ${i}, found ${seq}`);
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
if (entry.prev_digest !== prevDigest) {
|
|
142
|
+
record(`broken prev_digest link: expected ${prevDigest}, found ${entry.prev_digest}`);
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// (1) Per-record-INDEPENDENT checks: missing-file and digest-mismatch.
|
|
147
|
+
// Each is judged on this record alone, so under collectAllBreaks we record
|
|
148
|
+
// the break and CONTINUE to surface the full corruption scope. Under the
|
|
149
|
+
// default we record the first and stop.
|
|
150
|
+
const recordPath = join(repoRoot, entry.path);
|
|
151
|
+
if (!existsSync(recordPath)) {
|
|
152
|
+
record(`record file missing at ${entry.path} (could not read to recompute digest)`);
|
|
153
|
+
if (!collectAllBreaks) break;
|
|
154
|
+
prevDigest = entry.submission_digest;
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
let parsed;
|
|
159
|
+
try {
|
|
160
|
+
parsed = JSON.parse(readFileSync(recordPath, 'utf-8'));
|
|
161
|
+
} catch (err) {
|
|
162
|
+
record(`record file at ${entry.path} is not valid JSON: ${err.message}`);
|
|
163
|
+
if (!collectAllBreaks) break;
|
|
164
|
+
prevDigest = entry.submission_digest;
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const recomputed = submissionDigest(parsed);
|
|
169
|
+
if (recomputed !== entry.submission_digest) {
|
|
170
|
+
record(
|
|
171
|
+
`digest mismatch: recomputed ${recomputed} but manifest claims ${entry.submission_digest} ` +
|
|
172
|
+
`— record at ${entry.path} was modified after persist`
|
|
173
|
+
);
|
|
174
|
+
if (!collectAllBreaks) break;
|
|
175
|
+
prevDigest = entry.submission_digest;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const selfDigest = parsed.integrity?.submission_digest;
|
|
180
|
+
if (selfDigest !== recomputed) {
|
|
181
|
+
record(
|
|
182
|
+
`record self-digest mismatch: record.integrity.submission_digest is ` +
|
|
183
|
+
`${selfDigest ?? '(absent)'} but recomputed ${recomputed} — record at ${entry.path} ` +
|
|
184
|
+
`was modified after persist`
|
|
185
|
+
);
|
|
186
|
+
if (!collectAllBreaks) break;
|
|
187
|
+
prevDigest = entry.submission_digest;
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
prevDigest = entry.submission_digest;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const orphans = collectOrphans ? collectOrphanRecords(repoRoot, entries) : null;
|
|
195
|
+
|
|
196
|
+
const ok = breaks.length === 0 && (orphans === null || orphans.length === 0);
|
|
197
|
+
return {
|
|
198
|
+
ok,
|
|
199
|
+
count: entries.length,
|
|
200
|
+
head_digest: headDigest,
|
|
201
|
+
break: breaks.length > 0 ? breaks[0] : null,
|
|
202
|
+
...(collectAllBreaks ? { breaks } : {}),
|
|
203
|
+
...(collectOrphans ? { orphans } : {}),
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* INGEST-PROACT-001 — reconcile on-disk records against the ledger.
|
|
209
|
+
*
|
|
210
|
+
* Walks every `*.json` under `records/` (accepted AND `_rejected/`) and flags
|
|
211
|
+
* any record file whose `(run_id, seq)` identity is absent from the ledger.
|
|
212
|
+
* Such a file is an ORPHAN: persist wrote the record but the ledger append did
|
|
213
|
+
* not happen (a crash in the torn window between the two steps), so the record
|
|
214
|
+
* exists OUTSIDE the tamper-evident chain and the plain chain-walk is blind to
|
|
215
|
+
* it. The chain-manifest's own ledger lines are NOT under `records/`, so they
|
|
216
|
+
* are never mistaken for orphans.
|
|
217
|
+
*
|
|
218
|
+
* @param {string} repoRoot
|
|
219
|
+
* @param {Array<object>} entries - The ledger entries (already read).
|
|
220
|
+
* @returns {ChainOrphan[]}
|
|
221
|
+
*/
|
|
222
|
+
function collectOrphanRecords(repoRoot, entries) {
|
|
223
|
+
// Index the ledger by run_id and by repo-relative path so a record matches if
|
|
224
|
+
// EITHER identity is present — a record is ledgered as long as its line exists.
|
|
225
|
+
const ledgerRunIds = new Set();
|
|
226
|
+
const ledgerPaths = new Set();
|
|
227
|
+
for (const e of entries) {
|
|
228
|
+
if (e.run_id) ledgerRunIds.add(e.run_id);
|
|
229
|
+
if (e.path) ledgerPaths.add(e.path);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const orphans = [];
|
|
233
|
+
const recordsDir = join(repoRoot, 'records');
|
|
234
|
+
for (const absPath of findJsonFiles(recordsDir)) {
|
|
235
|
+
const relPath = relative(repoRoot, absPath).split(sep).join('/');
|
|
236
|
+
if (ledgerPaths.has(relPath)) continue;
|
|
237
|
+
|
|
238
|
+
let record;
|
|
239
|
+
try {
|
|
240
|
+
record = JSON.parse(readFileSync(absPath, 'utf-8'));
|
|
241
|
+
} catch {
|
|
242
|
+
// A record file we cannot even parse, that is also absent from the ledger,
|
|
243
|
+
// is still an orphan — report it with what little identity we have.
|
|
244
|
+
orphans.push({
|
|
245
|
+
run_id: null,
|
|
246
|
+
seq: null,
|
|
247
|
+
path: relPath,
|
|
248
|
+
reason: `record file present on disk but absent from the ledger, and not valid JSON`,
|
|
249
|
+
});
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const runId = record.run_id ?? null;
|
|
254
|
+
if (runId && ledgerRunIds.has(runId)) continue;
|
|
255
|
+
|
|
256
|
+
orphans.push({
|
|
257
|
+
run_id: runId,
|
|
258
|
+
seq: record.integrity?.seq ?? null,
|
|
259
|
+
path: relPath,
|
|
260
|
+
reason:
|
|
261
|
+
`record present on disk but absent from the ledger — a torn persist ` +
|
|
262
|
+
`(record written, ledger append missed). Re-run ingest for this record ` +
|
|
263
|
+
`or rebuild the chain to admit it into the tamper-evident ledger.`,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
return orphans;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Render a verifyChain result as operator-legible lines (no raw stack traces).
|
|
272
|
+
* Returned as an array so callers can choose the sink (console.log/console.error).
|
|
273
|
+
*
|
|
274
|
+
* @param {ChainVerifyResult} result
|
|
275
|
+
* @returns {string[]}
|
|
276
|
+
*/
|
|
277
|
+
export function formatChainResult(result) {
|
|
278
|
+
if (result.ok) {
|
|
279
|
+
const lines = [
|
|
280
|
+
`integrity chain OK: ${result.count} record(s) verified`,
|
|
281
|
+
`head digest: ${result.head_digest}`,
|
|
282
|
+
];
|
|
283
|
+
if (Array.isArray(result.orphans)) {
|
|
284
|
+
lines.push(`reconciliation: 0 orphan record(s) on disk`);
|
|
285
|
+
}
|
|
286
|
+
return lines;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
const lines = [];
|
|
290
|
+
|
|
291
|
+
// INGEST-PROACT-004: when collectAllBreaks populated breaks[], list every
|
|
292
|
+
// independent break so the operator sees the full corruption scope in one
|
|
293
|
+
// pass. Otherwise report the single first break (the CI-gate default).
|
|
294
|
+
const allBreaks = Array.isArray(result.breaks) ? result.breaks : (result.break ? [result.break] : []);
|
|
295
|
+
if (allBreaks.length > 1) {
|
|
296
|
+
lines.push(`integrity chain BROKEN: ${allBreaks.length} break(s) found`);
|
|
297
|
+
for (const b of allBreaks) {
|
|
298
|
+
lines.push(` - seq ${b.seq}${b.run_id ? ` (run_id: ${b.run_id})` : ''}: ${b.reason}`);
|
|
299
|
+
}
|
|
300
|
+
} else if (allBreaks.length === 1) {
|
|
301
|
+
const b = allBreaks[0];
|
|
302
|
+
lines.push(`integrity chain BROKEN at seq ${b.seq}${b.run_id ? ` (run_id: ${b.run_id})` : ''}`);
|
|
303
|
+
lines.push(`reason: ${b.reason}`);
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// INGEST-PROACT-001: an orphan makes the result not-ok even with zero chain
|
|
307
|
+
// breaks — name each orphan file + run_id and the recovery action.
|
|
308
|
+
if (Array.isArray(result.orphans) && result.orphans.length > 0) {
|
|
309
|
+
lines.push(`reconciliation: ${result.orphans.length} orphan record(s) on disk but absent from the ledger`);
|
|
310
|
+
for (const o of result.orphans) {
|
|
311
|
+
lines.push(` - ${o.path}${o.run_id ? ` (run_id: ${o.run_id})` : ''}: ${o.reason}`);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
return lines;
|
|
316
|
+
}
|