@dogfood-lab/ingest 1.3.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -22
- package/anchor/cli.js +185 -0
- package/anchor/compute-root.js +302 -0
- package/anchor/config.js +44 -0
- package/anchor/merkle.js +123 -0
- package/anchor/post-anchor.js +299 -0
- package/anchor/verify-anchor.js +358 -0
- package/lib/chain-manifest.js +106 -0
- package/lib/integrity.js +98 -0
- package/package.json +6 -2
- package/persist.js +41 -1
- package/rebuild-indexes.js +262 -20
- package/run.js +239 -25
- package/verify-chain.js +178 -0
package/run.js
CHANGED
|
@@ -17,19 +17,70 @@
|
|
|
17
17
|
* - regenerate indexes
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import { resolve, dirname } from 'node:path';
|
|
20
|
+
import { resolve, dirname, sep } from 'node:path';
|
|
21
21
|
import { fileURLToPath } from 'node:url';
|
|
22
22
|
import { randomBytes } from 'node:crypto';
|
|
23
23
|
|
|
24
24
|
import { verify } from '@dogfood-lab/verify';
|
|
25
|
-
import { stubProvenance,
|
|
25
|
+
import { stubProvenance, provenanceForProvider } from '@dogfood-lab/verify/validators/provenance.js';
|
|
26
26
|
import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
|
|
27
27
|
import { loadGlobalPolicy, loadRepoPolicy, loadScenarios } from './load-context.js';
|
|
28
28
|
import { isDuplicate, writeRecord, computeRecordPath } from './persist.js';
|
|
29
29
|
import { rebuildIndexes } from './rebuild-indexes.js';
|
|
30
|
+
import { verifyChain, formatChainResult } from './verify-chain.js';
|
|
31
|
+
import { handleAnchorCompute, handleAnchorPost, handleAnchorVerify } from './anchor/cli.js';
|
|
30
32
|
|
|
31
33
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
32
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Resolve the REAL provenance adapter for a submission, routed by
|
|
37
|
+
* `submission.source.provider` (github | gitlab), sourcing the provider's token
|
|
38
|
+
* from the environment. Returns `{ provenance }` on success or `{ err }` (a
|
|
39
|
+
* structured, operator-legible Error the caller surfaces via emitCliErrorEvent
|
|
40
|
+
* + exit 2). The adapter registry (`provenanceForProvider`) is the single
|
|
41
|
+
* provider-keyed seam; a provider in the schema enum without a registered
|
|
42
|
+
* adapter fails here loudly rather than silently skipping verification.
|
|
43
|
+
*
|
|
44
|
+
* @param {object} submission
|
|
45
|
+
* @returns {{ provenance: object } | { err: Error }}
|
|
46
|
+
*/
|
|
47
|
+
function resolveProviderProvenance(submission) {
|
|
48
|
+
const provider = (submission && submission.source && submission.source.provider) || 'github';
|
|
49
|
+
const factory = provenanceForProvider(provider);
|
|
50
|
+
if (!factory) {
|
|
51
|
+
return { err: new Error(`unknown provenance provider '${provider}' — no adapter registered (supported: github, gitlab).`) };
|
|
52
|
+
}
|
|
53
|
+
const token = provider === 'gitlab'
|
|
54
|
+
? (process.env.GITLAB_TOKEN || process.env.CI_JOB_TOKEN)
|
|
55
|
+
: (process.env.GITHUB_TOKEN || process.env.GH_TOKEN);
|
|
56
|
+
if (!token) {
|
|
57
|
+
const need = provider === 'gitlab' ? 'GITLAB_TOKEN or CI_JOB_TOKEN' : 'GITHUB_TOKEN or GH_TOKEN';
|
|
58
|
+
return { err: new Error(`real provenance for provider '${provider}' requires ${need} in the environment.`) };
|
|
59
|
+
}
|
|
60
|
+
return { provenance: factory(token) };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* SEED-1 (d3-ingest-003) — posixify a path-shaped value at the operator/log
|
|
65
|
+
* SERIALIZATION boundary. `computeRecordPath`/`writeRecord` return OS-native
|
|
66
|
+
* paths (backslash-separated, absolute, on win32) because those values are
|
|
67
|
+
* also used for real filesystem operations. But the moment a path crosses into
|
|
68
|
+
* CLI JSON output, NDJSON log lines, or a downstream report
|
|
69
|
+
* (dogfood-swarm persist.js records `report.dogfood.path`), it must be
|
|
70
|
+
* forward-slash so operators and log pivots see one canonical shape across
|
|
71
|
+
* OSes — and so a copy-paste into a raw.githubusercontent URL is not a broken
|
|
72
|
+
* link. We posixify ONLY here, at the emit sites, leaving the returned fs paths
|
|
73
|
+
* OS-native for the filesystem layer. Mirrors the boundary-normalize doctrine
|
|
74
|
+
* already used in rebuild-indexes.js and parse-regression-pins.js. NEVER a
|
|
75
|
+
* win32-skip.
|
|
76
|
+
*
|
|
77
|
+
* @param {string|null} p
|
|
78
|
+
* @returns {string|null}
|
|
79
|
+
*/
|
|
80
|
+
function posixifyPath(p) {
|
|
81
|
+
return typeof p === 'string' ? p.split(sep).join('/') : p;
|
|
82
|
+
}
|
|
83
|
+
|
|
33
84
|
/**
|
|
34
85
|
* Emit a single structured stage-transition log line via the shared helper.
|
|
35
86
|
*
|
|
@@ -238,7 +289,10 @@ export async function ingest(submission, options) {
|
|
|
238
289
|
logStage('persist_complete', {
|
|
239
290
|
submission_id: submissionId,
|
|
240
291
|
correlation_id,
|
|
241
|
-
path
|
|
292
|
+
// d3-ingest-003: posixify at the log boundary — `path` is OS-native from
|
|
293
|
+
// writeRecord (used for the fs write); the NDJSON log surface gets forward
|
|
294
|
+
// slashes so log pivots are byte-identical across OSes.
|
|
295
|
+
path: posixifyPath(path),
|
|
242
296
|
written,
|
|
243
297
|
duplicate: !written,
|
|
244
298
|
duration_ms: Date.now() - persistStart
|
|
@@ -281,11 +335,12 @@ export async function ingest(submission, options) {
|
|
|
281
335
|
failed_stage: 'rebuild_indexes',
|
|
282
336
|
message: err.message,
|
|
283
337
|
stack: truncatedStack,
|
|
284
|
-
|
|
338
|
+
// d3-ingest-003: operator-facing path → posixify at the log boundary.
|
|
339
|
+
record_persisted_at: posixifyPath(path),
|
|
285
340
|
recovery: 'next ingest will trigger a full rebuild of indexes/'
|
|
286
341
|
});
|
|
287
342
|
console.error(
|
|
288
|
-
`WARNING: record persisted at ${path}, but index rebuild failed: ${err.message}\n` +
|
|
343
|
+
`WARNING: record persisted at ${posixifyPath(path)}, but index rebuild failed: ${err.message}\n` +
|
|
289
344
|
` indexes/ may be stale until next ingest. To force rebuild now, re-run any test ingest.\n` +
|
|
290
345
|
` stack: ${stackPreview}`
|
|
291
346
|
);
|
|
@@ -429,7 +484,10 @@ export async function verifyOnly(submission, options) {
|
|
|
429
484
|
submission_id: submissionId,
|
|
430
485
|
correlation_id,
|
|
431
486
|
status: record.verification?.status ?? null,
|
|
432
|
-
|
|
487
|
+
// d3-ingest-003: posixify at the log boundary. The returned
|
|
488
|
+
// `would_persist_to` below stays OS-native so callers that resolve it
|
|
489
|
+
// against the filesystem keep a real fs path.
|
|
490
|
+
would_persist_to: posixifyPath(would_persist_to)
|
|
433
491
|
});
|
|
434
492
|
|
|
435
493
|
return { record, would_persist_to, verify_only: true };
|
|
@@ -467,18 +525,61 @@ const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(__dirname
|
|
|
467
525
|
|
|
468
526
|
if (isMain) {
|
|
469
527
|
const args = process.argv.slice(2);
|
|
470
|
-
|
|
528
|
+
// SEED-2 (d3-ingest-002) — make the CLI's repoRoot overridable so callers
|
|
529
|
+
// (notably dogfood-swarm's commands/persist.js execSync, and any test
|
|
530
|
+
// harness) can redirect every record write + index rebuild into a sandbox
|
|
531
|
+
// instead of the REAL working tree. Without this the only way to sandbox was
|
|
532
|
+
// a brittle source-copy of this file (the setupTempRunJs run.js-copy in
|
|
533
|
+
// d1b-001-cli-toplevel-error-event.test.js) that rewrote __dirname's `../..`
|
|
534
|
+
// walk. A production caller passes the real root explicitly; a test passes a
|
|
535
|
+
// temp dir; the default preserves the historical behavior when the env var
|
|
536
|
+
// is unset. resolve() makes a relative override absolute so the downstream
|
|
537
|
+
// join()s stay anchored.
|
|
538
|
+
const repoRoot = process.env.INGEST_REPO_ROOT
|
|
539
|
+
? resolve(process.env.INGEST_REPO_ROOT)
|
|
540
|
+
: resolve(__dirname, '../..');
|
|
471
541
|
|
|
472
542
|
// Parse CLI flags
|
|
473
543
|
let submissionJson;
|
|
474
544
|
let provenanceMode = null;
|
|
475
545
|
let verifyOnlyFlag = false;
|
|
546
|
+
let verifyChainFlag = false;
|
|
547
|
+
// Anchor verbs (optional, off-by-default, operator-run). --anchor-compute and
|
|
548
|
+
// --anchor-verify are fully offline (never import xrpl); --anchor-post lazily
|
|
549
|
+
// loads the optional xrpl package and needs XRPL_SEED.
|
|
550
|
+
let anchorComputeFlag = false;
|
|
551
|
+
let anchorPostFlag = false;
|
|
552
|
+
let anchorVerifyFlag = false;
|
|
553
|
+
let anchorMode = 'since-last';
|
|
554
|
+
let anchorAlgo = null;
|
|
555
|
+
let anchorNetwork = null;
|
|
556
|
+
let anchorTxFile = null;
|
|
557
|
+
let anchorTrustedAccounts = [];
|
|
476
558
|
const positionalArgs = [];
|
|
477
559
|
|
|
478
560
|
for (let i = 0; i < args.length; i++) {
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
561
|
+
// Accept BOTH the space form (`--flag value`) and the equals form
|
|
562
|
+
// (`--flag=value`). The prior parser matched only the space form, so a
|
|
563
|
+
// caller passing `--provenance=stub --file=...` (the shape dogfood-swarm's
|
|
564
|
+
// commands/persist.js builds for its execSync invocation) fell through to
|
|
565
|
+
// positionalArgs — `--provenance` then read as missing and the CLI exited 2
|
|
566
|
+
// ("--provenance flag is required"), silently breaking `swarm persist
|
|
567
|
+
// --ingest` on every platform. (dogfood-swarm self-audit follow-up.)
|
|
568
|
+
let arg = args[i];
|
|
569
|
+
let inlineValue = null;
|
|
570
|
+
if (arg.startsWith('--')) {
|
|
571
|
+
const eq = arg.indexOf('=');
|
|
572
|
+
if (eq !== -1) {
|
|
573
|
+
inlineValue = arg.slice(eq + 1);
|
|
574
|
+
arg = arg.slice(0, eq);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
const hasValue = inlineValue !== null || args[i + 1] !== undefined;
|
|
578
|
+
const takeValue = () => (inlineValue !== null ? inlineValue : args[++i]);
|
|
579
|
+
|
|
580
|
+
if (arg === '--provenance' && hasValue) {
|
|
581
|
+
provenanceMode = takeValue();
|
|
582
|
+
} else if (arg === '--file' && hasValue) {
|
|
482
583
|
const { readFileSync } = await import('node:fs');
|
|
483
584
|
// D1B-001 family (operator-legibility): a --file read failure
|
|
484
585
|
// (ENOENT/EACCES) routes through the structured error event and exits 2
|
|
@@ -488,7 +589,7 @@ if (isMain) {
|
|
|
488
589
|
// correlation id here (the same pivot the JSON.parse catch uses when
|
|
489
590
|
// there is no submission to derive a run_id from yet).
|
|
490
591
|
try {
|
|
491
|
-
submissionJson = readFileSync(resolve(
|
|
592
|
+
submissionJson = readFileSync(resolve(takeValue()), 'utf-8');
|
|
492
593
|
} catch (err) {
|
|
493
594
|
emitCliErrorEvent({
|
|
494
595
|
failedStage: 'cli_read_file',
|
|
@@ -498,17 +599,121 @@ if (isMain) {
|
|
|
498
599
|
});
|
|
499
600
|
process.exit(2);
|
|
500
601
|
}
|
|
501
|
-
} else if (
|
|
502
|
-
submissionJson =
|
|
503
|
-
} else if (
|
|
602
|
+
} else if (arg === '--payload' && hasValue) {
|
|
603
|
+
submissionJson = takeValue();
|
|
604
|
+
} else if (arg === '--verify-only') {
|
|
504
605
|
// F-252714-058: dry-run the pipeline without writing or rebuilding
|
|
505
606
|
// indexes. CI / operators preview what WOULD have been persisted.
|
|
506
607
|
verifyOnlyFlag = true;
|
|
608
|
+
} else if (arg === '--verify-chain') {
|
|
609
|
+
// Integrity chain v1: verify the append-only tamper-evident ledger at
|
|
610
|
+
// indexes/integrity/chain.jsonl, fully offline. No submission, no stdin,
|
|
611
|
+
// no provenance — a standalone audit command.
|
|
612
|
+
verifyChainFlag = true;
|
|
613
|
+
} else if (arg === '--anchor-compute') {
|
|
614
|
+
// Optional XRPL anchor: compute + write the next anchor manifest. Offline.
|
|
615
|
+
anchorComputeFlag = true;
|
|
616
|
+
} else if (arg === '--anchor-post') {
|
|
617
|
+
// Optional XRPL anchor: compute if needed + post to XRPL. Needs the
|
|
618
|
+
// optional xrpl package (lazily loaded) and XRPL_SEED.
|
|
619
|
+
anchorPostFlag = true;
|
|
620
|
+
} else if (arg === '--anchor-verify') {
|
|
621
|
+
// Optional XRPL anchor: verify local manifests + run the truncation check.
|
|
622
|
+
// Offline reports honest NOT-verified for the on-chain leg.
|
|
623
|
+
anchorVerifyFlag = true;
|
|
624
|
+
} else if (arg === '--anchor-all') {
|
|
625
|
+
// Genesis snapshot mode for compute/post (covers the whole chain).
|
|
626
|
+
anchorMode = 'all';
|
|
627
|
+
} else if (arg === '--anchor-algo' && hasValue) {
|
|
628
|
+
anchorAlgo = takeValue();
|
|
629
|
+
} else if (arg === '--anchor-network' && hasValue) {
|
|
630
|
+
anchorNetwork = takeValue();
|
|
631
|
+
} else if (arg === '--anchor-tx' && hasValue) {
|
|
632
|
+
// Path to a JSON file containing a fetched XRPL tx (with Memos) for the
|
|
633
|
+
// on-chain leg of --anchor-verify. Offline-honest: omit it to run the
|
|
634
|
+
// truncation check only.
|
|
635
|
+
anchorTxFile = takeValue();
|
|
636
|
+
} else if (arg === '--anchor-trusted' && hasValue) {
|
|
637
|
+
// Comma-separated trusted anchor accounts (UNIONed with the bundled list).
|
|
638
|
+
anchorTrustedAccounts = takeValue().split(',').map((s) => s.trim()).filter(Boolean);
|
|
507
639
|
} else {
|
|
508
640
|
positionalArgs.push(args[i]);
|
|
509
641
|
}
|
|
510
642
|
}
|
|
511
643
|
|
|
644
|
+
// --verify-chain is a standalone, side-effect-free audit: it reads only the
|
|
645
|
+
// ledger + the record files it references, takes no submission, reads no
|
|
646
|
+
// stdin, and needs no provenance adapter. Handle it BEFORE the stdin read and
|
|
647
|
+
// provenance resolution so `node run.js --verify-chain` does not block on
|
|
648
|
+
// stdin or demand a --provenance flag. Exit 0 when the chain verifies, 1 on
|
|
649
|
+
// the first break (operator-legible output, no raw stack traces).
|
|
650
|
+
if (verifyChainFlag) {
|
|
651
|
+
const result = verifyChain(repoRoot);
|
|
652
|
+
logStage(result.ok ? 'verify_chain_complete' : 'error', {
|
|
653
|
+
correlation_id: synthCorrelationId(),
|
|
654
|
+
...(result.ok ? {} : { failed_stage: 'verify_chain' }),
|
|
655
|
+
verified: result.count,
|
|
656
|
+
head_digest: result.head_digest,
|
|
657
|
+
chain_ok: result.ok,
|
|
658
|
+
...(result.break ? { break_seq: result.break.seq, break_reason: result.break.reason } : {})
|
|
659
|
+
});
|
|
660
|
+
const lines = formatChainResult(result);
|
|
661
|
+
if (result.ok) {
|
|
662
|
+
for (const line of lines) console.log(line);
|
|
663
|
+
} else {
|
|
664
|
+
for (const line of lines) console.error(line);
|
|
665
|
+
}
|
|
666
|
+
process.exit(result.ok ? 0 : 1);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// Optional XRPL anchor verbs — operator-run, off by default, NOT in the normal
|
|
670
|
+
// ingest/CI path. Like --verify-chain these are standalone audit/operations:
|
|
671
|
+
// no submission, no stdin, no provenance adapter. --anchor-compute and
|
|
672
|
+
// --anchor-verify are fully offline (never import xrpl); --anchor-post lazily
|
|
673
|
+
// loads the optional xrpl package and needs XRPL_SEED. Each handler returns
|
|
674
|
+
// { ok, exitCode, lines, event } and run.js owns the console + logStage + exit.
|
|
675
|
+
if (anchorComputeFlag || anchorPostFlag || anchorVerifyFlag) {
|
|
676
|
+
const correlation_id = synthCorrelationId();
|
|
677
|
+
let result;
|
|
678
|
+
if (anchorComputeFlag) {
|
|
679
|
+
result = handleAnchorCompute(repoRoot, {
|
|
680
|
+
mode: anchorMode,
|
|
681
|
+
...(anchorAlgo ? { algo: anchorAlgo } : {}),
|
|
682
|
+
...(anchorNetwork ? { network: anchorNetwork } : {}),
|
|
683
|
+
});
|
|
684
|
+
} else if (anchorPostFlag) {
|
|
685
|
+
result = await handleAnchorPost(repoRoot, {
|
|
686
|
+
mode: anchorMode,
|
|
687
|
+
...(anchorNetwork ? { network: anchorNetwork } : {}),
|
|
688
|
+
});
|
|
689
|
+
} else {
|
|
690
|
+
// --anchor-verify: optionally load a fetched tx JSON for the on-chain leg.
|
|
691
|
+
let tx;
|
|
692
|
+
if (anchorTxFile) {
|
|
693
|
+
const { readFileSync } = await import('node:fs');
|
|
694
|
+
try {
|
|
695
|
+
tx = JSON.parse(readFileSync(resolve(anchorTxFile), 'utf-8'));
|
|
696
|
+
} catch (err) {
|
|
697
|
+
emitCliErrorEvent({
|
|
698
|
+
failedStage: 'anchor_verify_read_tx',
|
|
699
|
+
correlationId: correlation_id,
|
|
700
|
+
err,
|
|
701
|
+
humanPrefix: 'could not read --anchor-tx file'
|
|
702
|
+
});
|
|
703
|
+
process.exit(2);
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
result = handleAnchorVerify(repoRoot, { tx, trustedAnchorAccounts: anchorTrustedAccounts });
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
// logStage strips any inner `stage:` field (the positional name wins), so
|
|
710
|
+
// spreading result.event — which carries its own `stage` — is safe.
|
|
711
|
+
logStage(result.event.stage, { correlation_id, ...result.event });
|
|
712
|
+
const sink = result.exitCode === 0 ? console.log : console.error;
|
|
713
|
+
for (const line of result.lines) sink(line);
|
|
714
|
+
process.exit(result.exitCode);
|
|
715
|
+
}
|
|
716
|
+
|
|
512
717
|
if (!submissionJson) {
|
|
513
718
|
// Read from stdin
|
|
514
719
|
const chunks = [];
|
|
@@ -570,32 +775,36 @@ if (isMain) {
|
|
|
570
775
|
console.error('WARNING: Using stub provenance (test/dev only). Records will NOT have real provenance verification.');
|
|
571
776
|
provenance = stubProvenance;
|
|
572
777
|
} else if (provenanceMode === 'github') {
|
|
573
|
-
|
|
574
|
-
|
|
778
|
+
// --provenance=github selects REAL provenance; the actual provider is taken
|
|
779
|
+
// from submission.source.provider, so a GitLab submission is confirmed via
|
|
780
|
+
// gitlabProvenance end-to-end (the adapter registry keys on the provider).
|
|
781
|
+
const resolved = resolveProviderProvenance(submission);
|
|
782
|
+
if (resolved.err) {
|
|
575
783
|
emitCliErrorEvent({
|
|
576
784
|
failedStage: 'cli_provenance_resolve',
|
|
577
785
|
correlationId: cliCorrelationId,
|
|
578
786
|
submissionId: submission && submission.run_id ? submission.run_id : null,
|
|
579
|
-
err:
|
|
787
|
+
err: resolved.err,
|
|
580
788
|
humanPrefix: 'provenance precondition unmet'
|
|
581
789
|
});
|
|
582
790
|
process.exit(2);
|
|
583
791
|
}
|
|
584
|
-
provenance =
|
|
792
|
+
provenance = resolved.provenance;
|
|
585
793
|
} else if (process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true') {
|
|
586
|
-
// In CI without explicit flag: default to
|
|
587
|
-
|
|
588
|
-
|
|
794
|
+
// In CI without an explicit flag: default to real provenance, routed by the
|
|
795
|
+
// submission's source.provider (github | gitlab).
|
|
796
|
+
const resolved = resolveProviderProvenance(submission);
|
|
797
|
+
if (resolved.err) {
|
|
589
798
|
emitCliErrorEvent({
|
|
590
799
|
failedStage: 'cli_provenance_resolve',
|
|
591
800
|
correlationId: cliCorrelationId,
|
|
592
801
|
submissionId: submission && submission.run_id ? submission.run_id : null,
|
|
593
|
-
err:
|
|
802
|
+
err: resolved.err,
|
|
594
803
|
humanPrefix: 'provenance precondition unmet'
|
|
595
804
|
});
|
|
596
805
|
process.exit(2);
|
|
597
806
|
}
|
|
598
|
-
provenance =
|
|
807
|
+
provenance = resolved.provenance;
|
|
599
808
|
} else {
|
|
600
809
|
emitCliErrorEvent({
|
|
601
810
|
failedStage: 'cli_provenance_resolve',
|
|
@@ -625,7 +834,10 @@ if (isMain) {
|
|
|
625
834
|
status: result.record.verification.status,
|
|
626
835
|
run_id: result.record.run_id ?? null,
|
|
627
836
|
verdict: result.record.overall_verdict?.verified ?? null,
|
|
628
|
-
|
|
837
|
+
// d3-ingest-003: posixify path-shaped CLI output so the operator
|
|
838
|
+
// contract is identical across OSes (a Windows backslash here breaks
|
|
839
|
+
// any downstream URL-build/string-match).
|
|
840
|
+
would_persist_to: posixifyPath(result.would_persist_to),
|
|
629
841
|
verify_only: true,
|
|
630
842
|
rejection_reasons: result.record.verification.rejection_reasons ?? []
|
|
631
843
|
}));
|
|
@@ -648,7 +860,9 @@ if (isMain) {
|
|
|
648
860
|
status: result.record.verification.status,
|
|
649
861
|
run_id: result.record.run_id ?? null,
|
|
650
862
|
verdict: result.record.overall_verdict?.verified ?? null,
|
|
651
|
-
|
|
863
|
+
// d3-ingest-003: posixify path-shaped CLI output (same family as
|
|
864
|
+
// would_persist_to above). dogfood-swarm's persist.js pivots on this.
|
|
865
|
+
path: posixifyPath(result.path),
|
|
652
866
|
written: result.written,
|
|
653
867
|
rejection_reasons: result.record.verification.rejection_reasons ?? []
|
|
654
868
|
}));
|
package/verify-chain.js
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chain verifier — the offline tamper-evidence gate.
|
|
3
|
+
*
|
|
4
|
+
* `verifyChain(repoRoot)` reads the append-only ledger at
|
|
5
|
+
* `indexes/integrity/chain.jsonl` and, FULLY OFFLINE (no network, no GitHub
|
|
6
|
+
* API), proves the chain is internally consistent:
|
|
7
|
+
*
|
|
8
|
+
* 1. For each entry, load the record file at `entry.path` and recompute its
|
|
9
|
+
* digest with the SAME `submissionDigest` helper persist used. Assert it
|
|
10
|
+
* equals BOTH `entry.submission_digest` (the ledger's claim) AND
|
|
11
|
+
* `record.integrity.submission_digest` (the record's self-claim). All three
|
|
12
|
+
* must agree — a tamper that touches the record but not the ledger, or the
|
|
13
|
+
* ledger but not the record, breaks one of these equalities.
|
|
14
|
+
* 2. Assert `entry.prev_digest` equals the PREVIOUS entry's
|
|
15
|
+
* `submission_digest` (GENESIS_DIGEST for seq 0) — the chain link.
|
|
16
|
+
* 3. Assert `seq` is 0,1,2,… strictly monotonic from 0.
|
|
17
|
+
*
|
|
18
|
+
* On the FIRST break it stops and reports `{ seq, run_id, reason }`; the caller
|
|
19
|
+
* (`--verify-chain` in run.js, or any consumer) exits non-zero. On success it
|
|
20
|
+
* returns the record count and head digest.
|
|
21
|
+
*
|
|
22
|
+
* Honesty (threat model): this is tamper-EVIDENT, not tamper-PROOF. An actor
|
|
23
|
+
* with the ingest write credential can rewrite a record, recompute its digest,
|
|
24
|
+
* and rewrite the matching ledger line — that re-verifies clean. The verifier
|
|
25
|
+
* catches any mutation to a record or ledger entry that is NOT consistently
|
|
26
|
+
* reflected in BOTH (an out-of-band edit, disk corruption, a partial restore, a
|
|
27
|
+
* push that touched a record but not the chain), and it catches middle-deletion,
|
|
28
|
+
* reorder, and forged-insertion (they break the seq run or the prev-link).
|
|
29
|
+
*
|
|
30
|
+
* KNOWN LIMITATION — tail truncation. Removing the most-recent entries (and
|
|
31
|
+
* their record files) leaves a SHORTER but internally-consistent chain, which
|
|
32
|
+
* verifies OK: the offline verifier has no external record of the expected head
|
|
33
|
+
* or leaf-count, so it can prove what remains is consistent but NOT that the
|
|
34
|
+
* chain is COMPLETE. Detecting truncation requires an external anchor of the
|
|
35
|
+
* head/count outside the writer's control — exactly what the optional XRPL
|
|
36
|
+
* anchor provides (its on-chain Merkle root + leaf count make any truncation
|
|
37
|
+
* BELOW an anchored point detectable). The offline chain alone is a completeness
|
|
38
|
+
* floor for in-place tampering, not a defense against tail truncation.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
42
|
+
import { join } from 'node:path';
|
|
43
|
+
|
|
44
|
+
import { submissionDigest, GENESIS_DIGEST } from './lib/integrity.js';
|
|
45
|
+
import { readChainManifest } from './lib/chain-manifest.js';
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @typedef {object} ChainBreak
|
|
49
|
+
* @property {number} seq - The seq at which verification failed.
|
|
50
|
+
* @property {string|null} run_id - The run_id of the failing entry, if known.
|
|
51
|
+
* @property {string} reason - Operator-legible description of what failed.
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* @typedef {object} ChainVerifyResult
|
|
56
|
+
* @property {boolean} ok - True when the whole chain verifies.
|
|
57
|
+
* @property {number} count - Number of entries verified (0 for an empty chain).
|
|
58
|
+
* @property {string} head_digest - submission_digest of the last entry, or
|
|
59
|
+
* GENESIS_DIGEST for an empty chain.
|
|
60
|
+
* @property {ChainBreak|null} break - First break, or null when ok.
|
|
61
|
+
*/
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Verify the integrity chain at `<repoRoot>/indexes/integrity/chain.jsonl`.
|
|
65
|
+
*
|
|
66
|
+
* @param {string} repoRoot - Absolute path to the testing-os repo root.
|
|
67
|
+
* @returns {ChainVerifyResult}
|
|
68
|
+
*/
|
|
69
|
+
export function verifyChain(repoRoot) {
|
|
70
|
+
let entries;
|
|
71
|
+
try {
|
|
72
|
+
entries = readChainManifest(repoRoot);
|
|
73
|
+
} catch (err) {
|
|
74
|
+
// A corrupt (non-JSON) manifest line is itself a tamper signal.
|
|
75
|
+
return {
|
|
76
|
+
ok: false,
|
|
77
|
+
count: 0,
|
|
78
|
+
head_digest: GENESIS_DIGEST,
|
|
79
|
+
break: { seq: -1, run_id: null, reason: err.message },
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (entries.length === 0) {
|
|
84
|
+
// An empty chain is trivially valid: genesis with no entries.
|
|
85
|
+
return { ok: true, count: 0, head_digest: GENESIS_DIGEST, break: null };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
let prevDigest = GENESIS_DIGEST;
|
|
89
|
+
|
|
90
|
+
for (let i = 0; i < entries.length; i++) {
|
|
91
|
+
const entry = entries[i];
|
|
92
|
+
const seq = entry.seq;
|
|
93
|
+
const runId = entry.run_id ?? null;
|
|
94
|
+
|
|
95
|
+
const fail = (reason) => ({
|
|
96
|
+
ok: false,
|
|
97
|
+
count: entries.length,
|
|
98
|
+
head_digest: entries[entries.length - 1].submission_digest ?? GENESIS_DIGEST,
|
|
99
|
+
break: { seq, run_id: runId, reason },
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
// (3) Monotonic seq: 0,1,2,…
|
|
103
|
+
if (seq !== i) {
|
|
104
|
+
return fail(`seq is not monotonic: expected ${i}, found ${seq}`);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// (2) Chain link: this entry's prev_digest must equal the previous entry's
|
|
108
|
+
// submission_digest (GENESIS for the first entry).
|
|
109
|
+
if (entry.prev_digest !== prevDigest) {
|
|
110
|
+
return fail(
|
|
111
|
+
`broken prev_digest link: expected ${prevDigest}, found ${entry.prev_digest}`
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// (1) Load the record and recompute its digest.
|
|
116
|
+
const recordPath = join(repoRoot, entry.path);
|
|
117
|
+
if (!existsSync(recordPath)) {
|
|
118
|
+
return fail(`record file missing at ${entry.path} (could not read to recompute digest)`);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
let record;
|
|
122
|
+
try {
|
|
123
|
+
record = JSON.parse(readFileSync(recordPath, 'utf-8'));
|
|
124
|
+
} catch (err) {
|
|
125
|
+
return fail(`record file at ${entry.path} is not valid JSON: ${err.message}`);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const recomputed = submissionDigest(record);
|
|
129
|
+
|
|
130
|
+
// The recomputed digest must equal the ledger's claim.
|
|
131
|
+
if (recomputed !== entry.submission_digest) {
|
|
132
|
+
return fail(
|
|
133
|
+
`digest mismatch: recomputed ${recomputed} but manifest claims ${entry.submission_digest} ` +
|
|
134
|
+
`— record at ${entry.path} was modified after persist`
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// …AND the record's own self-certified digest.
|
|
139
|
+
const selfDigest = record.integrity?.submission_digest;
|
|
140
|
+
if (selfDigest !== recomputed) {
|
|
141
|
+
return fail(
|
|
142
|
+
`record self-digest mismatch: record.integrity.submission_digest is ` +
|
|
143
|
+
`${selfDigest ?? '(absent)'} but recomputed ${recomputed} — record at ${entry.path} ` +
|
|
144
|
+
`was modified after persist`
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
prevDigest = entry.submission_digest;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return {
|
|
152
|
+
ok: true,
|
|
153
|
+
count: entries.length,
|
|
154
|
+
head_digest: entries[entries.length - 1].submission_digest,
|
|
155
|
+
break: null,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Render a verifyChain result as operator-legible lines (no raw stack traces).
|
|
161
|
+
* Returned as an array so callers can choose the sink (console.log/console.error).
|
|
162
|
+
*
|
|
163
|
+
* @param {ChainVerifyResult} result
|
|
164
|
+
* @returns {string[]}
|
|
165
|
+
*/
|
|
166
|
+
export function formatChainResult(result) {
|
|
167
|
+
if (result.ok) {
|
|
168
|
+
return [
|
|
169
|
+
`integrity chain OK: ${result.count} record(s) verified`,
|
|
170
|
+
`head digest: ${result.head_digest}`,
|
|
171
|
+
];
|
|
172
|
+
}
|
|
173
|
+
const b = result.break;
|
|
174
|
+
return [
|
|
175
|
+
`integrity chain BROKEN at seq ${b.seq}${b.run_id ? ` (run_id: ${b.run_id})` : ''}`,
|
|
176
|
+
`reason: ${b.reason}`,
|
|
177
|
+
];
|
|
178
|
+
}
|