@dogfood-lab/ingest 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/integrity.js +15 -1
- package/load-context.js +78 -20
- package/package.json +1 -1
- package/persist.js +241 -4
- package/rebuild-indexes.js +18 -2
- package/run.js +233 -32
- package/verify-chain.js +253 -17
package/lib/integrity.js
CHANGED
|
@@ -62,13 +62,27 @@ export function canonicalize(value) {
|
|
|
62
62
|
/**
|
|
63
63
|
* Deep clone with object keys sorted ascending at every depth. Arrays keep
|
|
64
64
|
* order (only their element objects are key-sorted). Primitives pass through.
|
|
65
|
+
*
|
|
66
|
+
* F-755d0f3f: the accumulator MUST be null-prototype. On a plain `{}` the
|
|
67
|
+
* assignment below would hit the inherited `Object.prototype.__proto__` setter
|
|
68
|
+
* for an own `__proto__` key and retarget the accumulator's prototype instead of
|
|
69
|
+
* creating an own key — dropping that field, and everything under it, from the
|
|
70
|
+
* canonical string and therefore from the record digest. `JSON.parse` creates a
|
|
71
|
+
* real own `__proto__` data property (CreateDataProperty, not Set), so any
|
|
72
|
+
* record file on disk can carry one, and verify-chain.js reaches this via a bare
|
|
73
|
+
* `JSON.parse` with no schema gate. That made two materially different records
|
|
74
|
+
* hash identically and let a tampered record re-verify clean.
|
|
75
|
+
*
|
|
76
|
+
* `JSON.stringify` serializes a null-prototype object identically to a plain
|
|
77
|
+
* one, so this is byte-for-byte compatible with every already-persisted record
|
|
78
|
+
* and the committed chain is unaffected.
|
|
65
79
|
*/
|
|
66
80
|
function sortDeep(value) {
|
|
67
81
|
if (Array.isArray(value)) {
|
|
68
82
|
return value.map(sortDeep);
|
|
69
83
|
}
|
|
70
84
|
if (value !== null && typeof value === 'object') {
|
|
71
|
-
const sorted =
|
|
85
|
+
const sorted = Object.create(null);
|
|
72
86
|
for (const key of Object.keys(value).sort()) {
|
|
73
87
|
sorted[key] = sortDeep(value[key]);
|
|
74
88
|
}
|
package/load-context.js
CHANGED
|
@@ -180,24 +180,6 @@ export function loadRepoPolicy(repoSlug, repoRoot) {
|
|
|
180
180
|
return parsed;
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
-
/**
|
|
184
|
-
* Default scenario fetcher that reads from the local filesystem.
|
|
185
|
-
* Used when dogfood-labs is dogfooding itself.
|
|
186
|
-
*
|
|
187
|
-
* @param {string} repoRoot - Root of the source repo
|
|
188
|
-
* @returns {object} Scenario fetch adapter
|
|
189
|
-
*/
|
|
190
|
-
export function localScenarioFetcher(repoRoot) {
|
|
191
|
-
return {
|
|
192
|
-
async fetch(scenarioId) {
|
|
193
|
-
if (!/^[\w-]+$/.test(scenarioId)) return null;
|
|
194
|
-
const path = join(repoRoot, 'dogfood', 'scenarios', `${scenarioId}.yaml`);
|
|
195
|
-
if (!existsSync(path)) return null;
|
|
196
|
-
return yaml.load(readFileSync(path, 'utf-8'));
|
|
197
|
-
}
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
|
|
201
183
|
/**
|
|
202
184
|
* Default per-request timeout for the GitHub scenario fetch. Mirrors the
|
|
203
185
|
* sibling `GITHUB_PROVENANCE_TIMEOUT_MS` constant at
|
|
@@ -271,6 +253,64 @@ function defaultSleep(ms) {
|
|
|
271
253
|
return new Promise((r) => setTimeout(r, ms));
|
|
272
254
|
}
|
|
273
255
|
|
|
256
|
+
/**
|
|
257
|
+
* COORD-001: parse a scenario body fetched from a consumer's source repo —
|
|
258
|
+
* the one `yaml.load` call site in this file that sees content a hostile or
|
|
259
|
+
* compromised submitter controls end to end. js-yaml 4.1.1 (the workspace
|
|
260
|
+
* floor; see package.json `"js-yaml": "^4.1.0"`) carries
|
|
261
|
+
* GHSA-h67p-54hq-rp68: chained `<<` merge keys cost O(depth) per level,
|
|
262
|
+
* because `mergeMappings()` (js-yaml lib/loader.js:304-321) re-copies
|
|
263
|
+
* `Object.keys(source)` for the FULL accumulated mapping at every level —
|
|
264
|
+
* so an n-level merge chain costs O(n^2) total. The attack is small BY
|
|
265
|
+
* CONSTRUCTION (each level adds only a couple of YAML lines), so
|
|
266
|
+
* GITHUB_SCENARIO_MAX_BYTES (a 1 MiB cap on the wire size) does not defend
|
|
267
|
+
* it: measured on the reference rig, a chain sized right at that 1 MiB cap
|
|
268
|
+
* costs ~61s of CPU, and scaling is ~quadratic (68 KB -> 127ms, 145 KB ->
|
|
269
|
+
* 656ms). Bumping js-yaml to v5 (which drops merge-key support outright)
|
|
270
|
+
* is blocked — Dependabot #50 is held open because v5 breaks two scripts/
|
|
271
|
+
* gates — so the mitigation lives here instead of in the dependency.
|
|
272
|
+
*
|
|
273
|
+
* CORE_SCHEMA is DEFAULT_SCHEMA minus exactly the YAML-1.1 extras
|
|
274
|
+
* (`timestamp`, `merge`, `binary`, `omap`, `pairs`, `set` — see js-yaml
|
|
275
|
+
* lib/schema/default.js). Without the registered `merge` type, the
|
|
276
|
+
* loader's `keyTag === 'tag:yaml.org,2002:merge'` branch in
|
|
277
|
+
* storeMappingPair() never matches a `<<` key, so `mergeMappings()` is
|
|
278
|
+
* never invoked and the O(depth) copy loop simply cannot run — `<<`
|
|
279
|
+
* resolves as an ordinary (and, against scenario.schema.json, rejected —
|
|
280
|
+
* `additionalProperties: false`) string key instead. This is verified
|
|
281
|
+
* behaviourally, not assumed from the option name, in
|
|
282
|
+
* coord-001-scenario-merge-bomb.test.js: a probe document proves the
|
|
283
|
+
* merge-key never fires, and a second probe proves the real fetch path
|
|
284
|
+
* (not just this helper in isolation) stays protected. Every field
|
|
285
|
+
* scenario.schema.json declares is a plain string/object/array/boolean —
|
|
286
|
+
* none relies on the dropped timestamp/binary/omap/pairs/set types — so
|
|
287
|
+
* this is a pure security hardening with no behavioural cost to a
|
|
288
|
+
* conforming scenario document.
|
|
289
|
+
*
|
|
290
|
+
* Scoped deliberately to THIS call site only. The other two `yaml.load`
|
|
291
|
+
* calls in this file (loadGlobalPolicy, loadRepoPolicy) both read
|
|
292
|
+
* maintainer-committed local files — the attacker does not control their
|
|
293
|
+
* content, only (in loadRepoPolicy's case) which existing file gets
|
|
294
|
+
* selected, and that selection is already segment-validated above.
|
|
295
|
+
* Widening this schema change to those sites is unnecessary risk for zero
|
|
296
|
+
* additional coverage.
|
|
297
|
+
*
|
|
298
|
+
* F-3be85850: this comment previously also named a THIRD "local files"
|
|
299
|
+
* call site, localScenarioFetcher — removed as dead code (zero callers
|
|
300
|
+
* anywhere in the repo, and not reachable even as an external package hook:
|
|
301
|
+
* load-context.js is not in @dogfood-lab/ingest's package.json `exports`
|
|
302
|
+
* map). Its own doc comment claimed it was "Used when dogfood-labs is
|
|
303
|
+
* dogfooding itself," but the real self-dogfood path (self-dogfood.yml)
|
|
304
|
+
* routes through the SAME public ingest.yml repository_dispatch pipeline
|
|
305
|
+
* every consumer uses, via githubScenarioFetcher below — never a local-disk
|
|
306
|
+
* reader. dogfood/scenarios/*.yaml remain the canonical scenario
|
|
307
|
+
* definitions; they are simply fetched over the GitHub Contents API rather
|
|
308
|
+
* than off local disk, even for this repo's own CI.
|
|
309
|
+
*/
|
|
310
|
+
export function parseUntrustedScenarioYaml(text) {
|
|
311
|
+
return yaml.load(text, { schema: yaml.CORE_SCHEMA });
|
|
312
|
+
}
|
|
313
|
+
|
|
274
314
|
/**
|
|
275
315
|
* GitHub scenario fetcher. Loads scenario definitions from a source repo
|
|
276
316
|
* via the GitHub API at a specific commit SHA.
|
|
@@ -458,7 +498,10 @@ export function githubScenarioFetcher(token, repoSlug, commitSha, opts = {}) {
|
|
|
458
498
|
|
|
459
499
|
let scenario;
|
|
460
500
|
try {
|
|
461
|
-
|
|
501
|
+
// COORD-001: merge-key DoS defense — see parseUntrustedScenarioYaml's
|
|
502
|
+
// doc comment. `text` is untrusted (fetched from the submitter's
|
|
503
|
+
// source repo), so it must never reach plain yaml.load().
|
|
504
|
+
scenario = parseUntrustedScenarioYaml(text);
|
|
462
505
|
} catch {
|
|
463
506
|
return { scenario: null, reason: 'parse_error', retryable: false };
|
|
464
507
|
}
|
|
@@ -488,7 +531,22 @@ export function githubScenarioFetcher(token, repoSlug, commitSha, opts = {}) {
|
|
|
488
531
|
for (let i = 0; i < attempts; i++) {
|
|
489
532
|
last = await attemptOnce(scenarioId);
|
|
490
533
|
if (last.scenario || !last.retryable || i === attempts - 1) break;
|
|
491
|
-
|
|
534
|
+
const waitMs = Math.min(RETRY_BASE_MS * (1 << i), RETRY_MAX_MS);
|
|
535
|
+
// F-2a5ddafa: this loop was completely silent on every retry but the
|
|
536
|
+
// last — an operator had zero early-warning signal that the source
|
|
537
|
+
// repo's API was degrading until the retry budget fully exhausted and
|
|
538
|
+
// threw (see the sibling fix in
|
|
539
|
+
// packages/verify/validators/provenance.js's confirm() loops, same
|
|
540
|
+
// finding). 'warn' (not 'error') since the fetch may still succeed on
|
|
541
|
+
// the next attempt.
|
|
542
|
+
logStage('warn', {
|
|
543
|
+
kind: 'scenario_fetch_retry',
|
|
544
|
+
scenario_id: scenarioId,
|
|
545
|
+
attempt: i + 1,
|
|
546
|
+
status_or_reason: last.reason || (last.fault ? last.fault.message : 'unknown'),
|
|
547
|
+
next_backoff_ms: waitMs,
|
|
548
|
+
});
|
|
549
|
+
await sleep(waitMs);
|
|
492
550
|
}
|
|
493
551
|
// V2-CROSS-BO-001: a transient fault that survived every retry is an
|
|
494
552
|
// outage — throw the classified operational error instead of returning a
|
package/package.json
CHANGED
package/persist.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* duplicate detection by run_id, directory creation.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { existsSync, mkdirSync, writeFileSync, renameSync, openSync, closeSync, unlinkSync } from 'node:fs';
|
|
9
|
+
import { existsSync, mkdirSync, writeFileSync, renameSync, openSync, closeSync, unlinkSync, readFileSync } from 'node:fs';
|
|
10
10
|
import { join, dirname, relative, sep } from 'node:path';
|
|
11
11
|
import { randomBytes } from 'node:crypto';
|
|
12
12
|
|
|
@@ -14,6 +14,8 @@ import { validateRecord } from './validate-record.js';
|
|
|
14
14
|
import { isUnsafeSegment } from './lib/unsafe-segment.js';
|
|
15
15
|
import { submissionDigest } from './lib/integrity.js';
|
|
16
16
|
import { readChainHead, appendChainEntry } from './lib/chain-manifest.js';
|
|
17
|
+
import { parseRejectionReason } from '@dogfood-lab/verify';
|
|
18
|
+
import { SUPPORTED_SCHEMA_VERSIONS } from '@dogfood-lab/schemas';
|
|
17
19
|
|
|
18
20
|
/**
|
|
19
21
|
* Error thrown when writeRecord loses a TOCTOU race for the same canonical path.
|
|
@@ -29,6 +31,40 @@ export class DuplicateRunIdError extends Error {
|
|
|
29
31
|
}
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Error thrown when writeRecord's SECOND computeRecordPath() call — the one
|
|
36
|
+
* that computes the real write target, reached only AFTER validateRecord()
|
|
37
|
+
* has already confirmed the record is schema-valid — still cannot produce a
|
|
38
|
+
* safe path.
|
|
39
|
+
*
|
|
40
|
+
* F-bbbe2e1f: computeRecordPath()'s isUnsafeSegment check (lib/unsafe-segment.js)
|
|
41
|
+
* is STRICTER than the record schema's own repo pattern
|
|
42
|
+
* (`^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$`), which permits an embedded `..` or a
|
|
43
|
+
* lone `.` segment (e.g. `../etc`). A record can therefore pass
|
|
44
|
+
* validateRecord() above and still fail here — this is never a schema
|
|
45
|
+
* problem, it is the traversal backstop firing on schema-valid input. Pre-fix
|
|
46
|
+
* that let a bare, unclassified computeRecordPath Error (no `.code`) escape
|
|
47
|
+
* uncaught — the exact shape F-a37d36f5 eliminated at the FIRST
|
|
48
|
+
* computeRecordPath call (the isDuplicate probe below) but not this second
|
|
49
|
+
* one, three lines after validateRecord(). Classifying it here matches
|
|
50
|
+
* RecordValidationError's discipline and stays fail-closed: nothing below
|
|
51
|
+
* this point has touched the filesystem yet (mkdirSync/openSync/writeFileSync
|
|
52
|
+
* all come after), so no partial write is possible either way.
|
|
53
|
+
*/
|
|
54
|
+
export class UnsafeRecordPathError extends Error {
|
|
55
|
+
constructor(record, cause) {
|
|
56
|
+
super(
|
|
57
|
+
`record passed schema validation but its path could not be safely computed ` +
|
|
58
|
+
`(repo: ${record.repo}, run_id: ${record.run_id}): ${cause.message}`,
|
|
59
|
+
{ cause }
|
|
60
|
+
);
|
|
61
|
+
this.name = 'UnsafeRecordPathError';
|
|
62
|
+
this.code = 'UNSAFE_RECORD_PATH';
|
|
63
|
+
this.repo = record.repo;
|
|
64
|
+
this.runId = record.run_id;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
32
68
|
/**
|
|
33
69
|
* Compute the canonical file path for a persisted record.
|
|
34
70
|
*
|
|
@@ -88,9 +124,180 @@ export function computeRecordPath(record, repoRoot) {
|
|
|
88
124
|
return join(repoRoot, base, org, repo, year, month, day, filename);
|
|
89
125
|
}
|
|
90
126
|
|
|
127
|
+
/**
|
|
128
|
+
* F-4036ae25: a prior `_rejected` record whose rejection_reasons are ALL
|
|
129
|
+
* retryable-class is "the submitter can fix this and resubmit", not an
|
|
130
|
+
* unappealable verdict about the run — the same doctrine that already exempts
|
|
131
|
+
* an unfilable (`_skipPersist`) rejection from consuming its run_id. Narrowing
|
|
132
|
+
* `_skipPersist` in verify/index.js means a filable-but-rejected submission
|
|
133
|
+
* now DOES persist (the fleet-wide audit trail F-4036ae25 restores), so this
|
|
134
|
+
* is the other half of that fix: persisting the evidence must not resurrect
|
|
135
|
+
* the exact run_id-poisoning pathology F-82429f90 eliminated for
|
|
136
|
+
* validator-crash faults.
|
|
137
|
+
*
|
|
138
|
+
* F-f8952a50 (wave 10): this used to check `parseRejectionReason(r).prefix
|
|
139
|
+
* === 'schema:'` — ONE of the ten prefixes parse-rejection.js's own "Prefix
|
|
140
|
+
* taxonomy" documents under `class: 'submission-bad'` ("the submitter fixes
|
|
141
|
+
* the payload and resubmits"). Proven live with `repo:mismatch` (a pure
|
|
142
|
+
* identity/addressing mistake — the run happened, only the repo/run_url
|
|
143
|
+
* pairing was mis-stated): a corrected resubmission's payload never even
|
|
144
|
+
* reached verify() a second time, because both isDuplicate() call sites
|
|
145
|
+
* treated the stale `repo:` rejection as an unconditional block — see
|
|
146
|
+
* f-0f9e4077-retry-collision-duplicate-rejection.test.js's 'F-f8952a50'
|
|
147
|
+
* describe block for the end-to-end proof.
|
|
148
|
+
*
|
|
149
|
+
* The fix reads `parseRejectionReason(r).retryable` — the classifier's OWN
|
|
150
|
+
* per-prefix routing decision (see parse-rejection.js's file header for the
|
|
151
|
+
* full split) — instead of re-deriving a prefix allowlist here. A future
|
|
152
|
+
* prefix parse-rejection.js adds is automatically retryable or not with no
|
|
153
|
+
* edit to this file. This is intentionally NARROWER than "every submission-bad
|
|
154
|
+
* class": `retryable` is false for `policy:` and `provenance:` even though
|
|
155
|
+
* both are class `submission-bad` — those two prefixes are a rendered VERDICT
|
|
156
|
+
* on the run's own reported content (see parse-rejection.js), and letting a
|
|
157
|
+
* resubmission retry past one would let a submitter launder a genuinely-bad
|
|
158
|
+
* run into an accepted one by resubmitting different self-reported content
|
|
159
|
+
* under the same run_id. That boundary is independently pinned end-to-end by
|
|
160
|
+
* schema-invalid-skip-persist.test.js's "REGRESSION GUARD" test and the
|
|
161
|
+
* "persist-a-verdict doctrine is preserved" describe block — this fix must
|
|
162
|
+
* not (and, via the per-prefix flag, does not) touch it.
|
|
163
|
+
*
|
|
164
|
+
* `'operational'` / `'ingest'` / `'unknown'`-class reasons are always
|
|
165
|
+
* `retryable: false` too (see parse-rejection.js). Most `'operational'`-class
|
|
166
|
+
* reasons are not actually reachable here in production (F-82429f90 and its
|
|
167
|
+
* provenance-fault:/scenario-fetch-fault: siblings THROW instead of
|
|
168
|
+
* persisting a `_rejected` record — see runValidator in verify/index.js).
|
|
169
|
+
*
|
|
170
|
+
* F-51780da9 (wave 22, confirming audit of F-be0deacd): `CONTRACT_SCHEMA_TOO_NEW:`
|
|
171
|
+
* is the one documented exception where the frozen `retryable: false` is NOT
|
|
172
|
+
* the whole story. `validators/schema-version.js` RETURNS it as an ordinary
|
|
173
|
+
* rejection string rather than throwing, so a too-new-major submission
|
|
174
|
+
* genuinely does persist to `_rejected/` and IS read by this function.
|
|
175
|
+
* `retryable` is a STATIC per-prefix classification stamped at parse time —
|
|
176
|
+
* it has no way to observe that testing-os has SINCE been upgraded past the
|
|
177
|
+
* declared major, so trusting it alone lets one stale TOO_NEW rejection
|
|
178
|
+
* permanently poison a run_id even after the exact condition it was
|
|
179
|
+
* rejecting no longer holds (proven live: seed a `_rejected` record with a
|
|
180
|
+
* TOO_NEW reason declaring a major the CURRENT build already understands —
|
|
181
|
+
* pre-fix this function still returns `false` forever, because it never asks
|
|
182
|
+
* whether the stored major is still actually too new). `reevaluateTooNewRetry`
|
|
183
|
+
* below is the correction: for this ONE prefix, re-derive whether the STORED
|
|
184
|
+
* rejection's declared major is still above the CURRENT build's
|
|
185
|
+
* `SUPPORTED_SCHEMA_VERSIONS.<contract>.maxMajor`, instead of trusting the
|
|
186
|
+
* frozen boolean. The other nine prefixes were swept for the same trap (any
|
|
187
|
+
* retryable semantics that depend on mutable environment rather than the
|
|
188
|
+
* submission itself) and none qualify: schema:/policy-config:/repo:/
|
|
189
|
+
* submission-contains-verifier-field:/unsafe-record-path:/steps[<id>]:/
|
|
190
|
+
* CONTRACT_SCHEMA_TOO_OLD: are already retryable:true (an already-permissive
|
|
191
|
+
* prefix has no "stuck forever" direction to fix); policy:/provenance: are a
|
|
192
|
+
* DELIBERATE, permanent anti-gaming verdict on the run's own reported
|
|
193
|
+
* content, not an environment-dependent snapshot that goes stale;
|
|
194
|
+
* provenance-fault:/scenario-fetch-fault:/submission-malformed:/
|
|
195
|
+
* VALIDATOR_FAULT_*: all throw rather than persist (never reach this
|
|
196
|
+
* function); scenario-load: is evaluated against an immutable git
|
|
197
|
+
* commit_sha, not a build-version boundary, so the same committed content at
|
|
198
|
+
* the same ref parses the same way forever — there is no analogous integer
|
|
199
|
+
* ceiling to re-derive the way TOO_NEW's `maxMajor` comparison allows.
|
|
200
|
+
*
|
|
201
|
+
* Any read/parse failure fails closed (blocking): a corrupted or unreadable
|
|
202
|
+
* evidence file must never silently unblock a path collision. The same
|
|
203
|
+
* failure mode covers a CONTRACT_SCHEMA_TOO_NEW: reason whose shape
|
|
204
|
+
* `reevaluateTooNewRetry` cannot parse, or whose contract key is no longer
|
|
205
|
+
* registered in `SUPPORTED_SCHEMA_VERSIONS` — both fail closed to "still
|
|
206
|
+
* blocking" rather than guessing.
|
|
207
|
+
*
|
|
208
|
+
* @param {string} rejectedPath - Absolute path already confirmed to exist.
|
|
209
|
+
* @returns {boolean} true when every rejection reason is individually
|
|
210
|
+
* retryable (see parseRejectionReason's `retryable` field), OR — for
|
|
211
|
+
* CONTRACT_SCHEMA_TOO_NEW: specifically — no longer applicable because
|
|
212
|
+
* testing-os has since been upgraded past the declared major.
|
|
213
|
+
*/
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* F-51780da9: matches a `CONTRACT_SCHEMA_TOO_NEW:` reason string emitted by
|
|
217
|
+
* `validators/schema-version.js` — `CONTRACT_SCHEMA_TOO_NEW: <contract>
|
|
218
|
+
* schema v<major>.<minor>.<patch> ...` — capturing the contract key and the
|
|
219
|
+
* declared MAJOR, the two pieces of information needed to re-ask, against
|
|
220
|
+
* the CURRENT build, the exact question the stored rejection answered
|
|
221
|
+
* against a possibly-older one. Anchored to the literal TOO_NEW prefix only
|
|
222
|
+
* (never TOO_OLD:, a distinct literal) and requires a trailing space after
|
|
223
|
+
* the three-part version so it matches both the full production message
|
|
224
|
+
* (`... but this build supports v...`) and a shortened test fixture
|
|
225
|
+
* (`... — upgrade testing-os`) without over-matching a malformed variant.
|
|
226
|
+
*/
|
|
227
|
+
const TOO_NEW_DECLARED_VERSION = /^CONTRACT_SCHEMA_TOO_NEW: (\S+) schema v(\d+)\.\d+\.\d+ /;
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* F-51780da9: re-evaluate a single STORED `CONTRACT_SCHEMA_TOO_NEW:` reason
|
|
231
|
+
* against the CURRENT build's `SUPPORTED_SCHEMA_VERSIONS`, instead of
|
|
232
|
+
* trusting the frozen `retryable: false` parse-rejection.js stamps on this
|
|
233
|
+
* prefix. `validators/schema-version.js` emits this prefix precisely when
|
|
234
|
+
* `declaredMajor > supported.maxMajor`; this re-runs that identical
|
|
235
|
+
* comparison against whatever `maxMajor` is TODAY, which a testing-os
|
|
236
|
+
* upgrade may have since raised past the stored declared major.
|
|
237
|
+
*
|
|
238
|
+
* @param {string} reason - a single rejection_reasons[] entry.
|
|
239
|
+
* @returns {boolean} `true` when the declared major is no longer above the
|
|
240
|
+
* current build's supported ceiling (the rejection has been resolved by a
|
|
241
|
+
* since-applied upgrade); `false` when it is still genuinely too new, the
|
|
242
|
+
* contract key is unrecognized, or `reason` is not a
|
|
243
|
+
* CONTRACT_SCHEMA_TOO_NEW: reason at all — every non-match fails closed to
|
|
244
|
+
* "still blocking" rather than guessing.
|
|
245
|
+
*/
|
|
246
|
+
function reevaluateTooNewRetry(reason) {
|
|
247
|
+
const match = typeof reason === 'string' ? reason.match(TOO_NEW_DECLARED_VERSION) : null;
|
|
248
|
+
if (!match) return false;
|
|
249
|
+
const [, contract, declaredMajorStr] = match;
|
|
250
|
+
const supported = SUPPORTED_SCHEMA_VERSIONS[contract];
|
|
251
|
+
if (!supported) return false;
|
|
252
|
+
return Number(declaredMajorStr) <= supported.maxMajor;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function isRetryableRejection(rejectedPath) {
|
|
256
|
+
let parsed;
|
|
257
|
+
try {
|
|
258
|
+
parsed = JSON.parse(readFileSync(rejectedPath, 'utf-8'));
|
|
259
|
+
} catch {
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
const reasons = parsed?.verification?.rejection_reasons;
|
|
263
|
+
if (parsed?.verification?.status !== 'rejected' || !Array.isArray(reasons) || reasons.length === 0) {
|
|
264
|
+
return false;
|
|
265
|
+
}
|
|
266
|
+
// F-51780da9: a reason counts as retryable either via the classifier's own
|
|
267
|
+
// static per-prefix flag, or — for CONTRACT_SCHEMA_TOO_NEW: specifically —
|
|
268
|
+
// via the re-derived, current-environment check above. Every other prefix
|
|
269
|
+
// is unaffected: reevaluateTooNewRetry returns false for any reason it
|
|
270
|
+
// does not recognize, so `.every()`'s existing behavior is unchanged for
|
|
271
|
+
// schema:/policy:/repo:/provenance:/etc.
|
|
272
|
+
return reasons.every(r => parseRejectionReason(r).retryable === true || reevaluateTooNewRetry(r));
|
|
273
|
+
}
|
|
274
|
+
|
|
91
275
|
/**
|
|
92
276
|
* Check if a record with this run_id already exists (accepted or rejected).
|
|
93
277
|
*
|
|
278
|
+
* F-0f9e4077 (wave-6 regression, fixed wave-8): computeRecordPath() is keyed
|
|
279
|
+
* on (run_id, date, accepted|rejected) — never on rejection_reasons content.
|
|
280
|
+
* So once a `_rejected` record occupies a path, ANY new record whose OWN
|
|
281
|
+
* status is STILL 'rejected' is headed to that exact same path no matter
|
|
282
|
+
* which violation it reports (an unexpected field with a different name, a
|
|
283
|
+
* different value, even a byte-identical resubmission all collide
|
|
284
|
+
* identically — computeRecordPath never looks at content). The
|
|
285
|
+
* isRetryableRejection carve-out below exists so a stale retryable-class
|
|
286
|
+
* rejection (F-f8952a50: any prefix parse-rejection.js's taxonomy flags
|
|
287
|
+
* `retryable: true` — shape/addressing mistakes, not just schema:) does not
|
|
288
|
+
* block a DIFFERENT-status (now accepted) record from reaching ITS OWN,
|
|
289
|
+
* different path — it was never a promise that the
|
|
290
|
+
* REJECTED path specifically is free. Pre-fix, skipping the status check let
|
|
291
|
+
* an uncorrected retry fall through as "not a duplicate," reach
|
|
292
|
+
* writeRecord()'s exclusive-create, and throw DuplicateRunIdError — a
|
|
293
|
+
* purely sequential, single-writer collision masquerading as the two-writer
|
|
294
|
+
* TOCTOU race that error class exists for. Gating on
|
|
295
|
+
* `record.verification.status === 'accepted'` restores the carve-out to
|
|
296
|
+
* exactly the case it can actually help (a record now headed elsewhere)
|
|
297
|
+
* while making a still-rejected collision an unconditional, ordinary
|
|
298
|
+
* duplicate — so writeRecord() short-circuits before ever attempting the
|
|
299
|
+
* write, and no throw is reachable for this class anymore.
|
|
300
|
+
*
|
|
94
301
|
* @param {string} runId
|
|
95
302
|
* @param {object} record - The record (used for repo/timing to compute path)
|
|
96
303
|
* @param {string} repoRoot
|
|
@@ -105,7 +312,14 @@ export function isDuplicate(runId, record, repoRoot) {
|
|
|
105
312
|
// Check rejected path
|
|
106
313
|
const rejectedRecord = { ...record, verification: { ...record.verification, status: 'rejected' } };
|
|
107
314
|
const rejectedPath = computeRecordPath(rejectedRecord, repoRoot);
|
|
108
|
-
if (existsSync(rejectedPath))
|
|
315
|
+
if (existsSync(rejectedPath)) {
|
|
316
|
+
// A record that is itself still rejected can only ever land at THIS
|
|
317
|
+
// path — asking whether the OLD occupant is retryable is moot when the
|
|
318
|
+
// NEW attempt has nowhere else to go. Only a record now bound for the
|
|
319
|
+
// accepted path (real forward progress) gets to ask that question.
|
|
320
|
+
if (record.verification?.status !== 'accepted') return true;
|
|
321
|
+
return !isRetryableRejection(rejectedPath);
|
|
322
|
+
}
|
|
109
323
|
|
|
110
324
|
return false;
|
|
111
325
|
}
|
|
@@ -130,7 +344,21 @@ export function isDuplicate(runId, record, repoRoot) {
|
|
|
130
344
|
* @throws {DuplicateRunIdError} when a concurrent writer won the race
|
|
131
345
|
*/
|
|
132
346
|
export function writeRecord(record, repoRoot) {
|
|
133
|
-
|
|
347
|
+
// F-a37d36f5: family sibling of the ingest.js pre-check fix — writeRecord
|
|
348
|
+
// carries its OWN premature isDuplicate call, reached even when the
|
|
349
|
+
// caller's own duplicate check was skipped or already cleared. A record
|
|
350
|
+
// whose repo/run_id computeRecordPath rejects (invalid format, unsafe
|
|
351
|
+
// segment) cannot possibly collide with an existing file, so treating the
|
|
352
|
+
// throw as "not a duplicate" here is the same semantically-free
|
|
353
|
+
// short-circuit as the run.js fix — it lets validateRecord() below be the
|
|
354
|
+
// authoritative judge instead of a raw path-computation throw.
|
|
355
|
+
let duplicate;
|
|
356
|
+
try {
|
|
357
|
+
duplicate = isDuplicate(record.run_id, record, repoRoot);
|
|
358
|
+
} catch {
|
|
359
|
+
duplicate = false;
|
|
360
|
+
}
|
|
361
|
+
if (duplicate) {
|
|
134
362
|
const path = computeRecordPath(record, repoRoot);
|
|
135
363
|
return { path, written: false };
|
|
136
364
|
}
|
|
@@ -160,7 +388,16 @@ export function writeRecord(record, repoRoot) {
|
|
|
160
388
|
// schema is the contract every downstream consumer relies on.
|
|
161
389
|
validateRecord(record);
|
|
162
390
|
|
|
163
|
-
|
|
391
|
+
// F-bbbe2e1f: see UnsafeRecordPathError's doc comment. Wrap this SECOND
|
|
392
|
+
// computeRecordPath() call — the first is inside isDuplicate() above,
|
|
393
|
+
// already guarded by F-a37d36f5 — so a schema-valid-but-unsafe repo (e.g.
|
|
394
|
+
// `../etc`) throws a classified error instead of a bare one.
|
|
395
|
+
let path;
|
|
396
|
+
try {
|
|
397
|
+
path = computeRecordPath(record, repoRoot);
|
|
398
|
+
} catch (err) {
|
|
399
|
+
throw new UnsafeRecordPathError(record, err);
|
|
400
|
+
}
|
|
164
401
|
const dir = dirname(path);
|
|
165
402
|
|
|
166
403
|
mkdirSync(dir, { recursive: true });
|
package/rebuild-indexes.js
CHANGED
|
@@ -281,13 +281,29 @@ export function rebuildIndexes(repoRoot, options = {}) {
|
|
|
281
281
|
|
|
282
282
|
// --- latest-by-repo.json ---
|
|
283
283
|
// Keyed by repo, then product_surface. Only accepted records count.
|
|
284
|
-
|
|
284
|
+
//
|
|
285
|
+
// F-89b7dcd5: `record.repo` and `sr.product_surface` come from loadRecord()
|
|
286
|
+
// (JSON.parse only, no schema gate — see its JSDoc) rather than validated
|
|
287
|
+
// submission input, so a hand-committed record with repo: '__proto__' would
|
|
288
|
+
// otherwise resolve `latestByRepo['__proto__']` to Object.prototype itself
|
|
289
|
+
// (truthy, on a plain `{}`), skip the init branch below, and land the
|
|
290
|
+
// surface write ON Object.prototype — real global prototype pollution for
|
|
291
|
+
// the rest of this process. Object.create(null) removes the prototype
|
|
292
|
+
// chain entirely: `latestByRepo['__proto__']` is a plain (absent) data
|
|
293
|
+
// property, not the special accessor. JSON.stringify (below, at the
|
|
294
|
+
// commitGroupRename call) serializes a null-prototype object identically
|
|
295
|
+
// to a plain one, so this costs nothing on the write side. Not reachable
|
|
296
|
+
// from the ingest write path today (writeRecord's validateRecord() gate
|
|
297
|
+
// constrains repo's shape before persist.js ever writes a file), so this
|
|
298
|
+
// is defense-in-depth against a hand-committed or otherwise out-of-band
|
|
299
|
+
// record file, not a live vulnerability.
|
|
300
|
+
const latestByRepo = Object.create(null);
|
|
285
301
|
|
|
286
302
|
for (const record of allRecords) {
|
|
287
303
|
if (record.verification?.status !== 'accepted') continue;
|
|
288
304
|
|
|
289
305
|
const repo = record.repo;
|
|
290
|
-
if (!latestByRepo[repo]) latestByRepo[repo] =
|
|
306
|
+
if (!latestByRepo[repo]) latestByRepo[repo] = Object.create(null);
|
|
291
307
|
|
|
292
308
|
for (const sr of record.scenario_results || []) {
|
|
293
309
|
const surface = sr.product_surface;
|
package/run.js
CHANGED
|
@@ -25,7 +25,8 @@ import { verify } from '@dogfood-lab/verify';
|
|
|
25
25
|
import { stubProvenance, provenanceForProvider } from '@dogfood-lab/verify/validators/provenance.js';
|
|
26
26
|
import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
|
|
27
27
|
import { loadGlobalPolicy, loadRepoPolicy, loadScenarios, githubScenarioFetcher } from './load-context.js';
|
|
28
|
-
import { isDuplicate, writeRecord, computeRecordPath } from './persist.js';
|
|
28
|
+
import { isDuplicate, writeRecord, computeRecordPath, UnsafeRecordPathError } from './persist.js';
|
|
29
|
+
import { RecordValidationError } from './validate-record.js';
|
|
29
30
|
import { rebuildIndexes } from './rebuild-indexes.js';
|
|
30
31
|
import { verifyChain, formatChainResult } from './verify-chain.js';
|
|
31
32
|
import { handleAnchorCompute, handleAnchorPost, handleAnchorVerify } from './anchor/cli.js';
|
|
@@ -350,7 +351,40 @@ export async function ingest(submission, options) {
|
|
|
350
351
|
timing: submission.timing,
|
|
351
352
|
verification: { status: 'accepted' }
|
|
352
353
|
};
|
|
353
|
-
|
|
354
|
+
// F-f8952a50 (wave 10): this probe's hardcoded `status: 'accepted'` is
|
|
355
|
+
// ALSO the site the finding proved a corrected repo:mismatch resubmission
|
|
356
|
+
// was swallowed at — isDuplicate() only asks isRetryableRejection() when
|
|
357
|
+
// the record it's handed claims 'accepted' (see isDuplicate's own
|
|
358
|
+
// comment), so this probe is the FIRST place a stale non-schema rejection
|
|
359
|
+
// could mask a genuine correction, before verify() ever runs. No separate
|
|
360
|
+
// fix belongs HERE, though: this probe and writeRecord()'s own internal
|
|
361
|
+
// isDuplicate() call (persist.js) share the exact same isDuplicate() /
|
|
362
|
+
// isRetryableRejection() code path, so persist.js's per-prefix
|
|
363
|
+
// `retryable` fix (parse-rejection.js) already reaches both call sites —
|
|
364
|
+
// widening retryability there is what unblocks this probe too, with
|
|
365
|
+
// nothing probe-specific to change.
|
|
366
|
+
// F-a37d36f5: isDuplicate -> computeRecordPath runs against RAW untrusted
|
|
367
|
+
// submission.repo/run_id, three steps BEFORE verify()'s schema gate. A
|
|
368
|
+
// malformed repo ('a/b/c', a path-traversal attempt, etc.) makes
|
|
369
|
+
// computeRecordPath THROW ('invalid repo format' / 'unsafe repo segment'
|
|
370
|
+
// / 'unsafe run_id') — good, the traversal guards hold — but letting that
|
|
371
|
+
// throw escape here inverts this repo's submission-bad vs operational
|
|
372
|
+
// doctrine: it propagates as an uncaught fault (exit 2, "operator error"
|
|
373
|
+
// per the CLI's own USAGE block) for input that is squarely the
|
|
374
|
+
// SUBMITTER's to fix, and no `_rejected` evidence record is ever
|
|
375
|
+
// written. A malformed repo/run_id can never collide with an existing
|
|
376
|
+
// record anyway, so treating an unpathable submission as "not a
|
|
377
|
+
// duplicate" is semantically free — verify()'s schema gate is the
|
|
378
|
+
// authoritative judge of bad input, and writeRecord's own
|
|
379
|
+
// computeRecordPath (which runs AFTER validateRecord, on the SCHEMA-
|
|
380
|
+
// VALIDATED persisted record) remains the real enforcement point.
|
|
381
|
+
let duplicate;
|
|
382
|
+
try {
|
|
383
|
+
duplicate = isDuplicate(submission.run_id, probeRecord, repoRoot);
|
|
384
|
+
} catch {
|
|
385
|
+
duplicate = false;
|
|
386
|
+
}
|
|
387
|
+
if (duplicate) {
|
|
354
388
|
logStage('rejected_pre_persist', {
|
|
355
389
|
submission_id: submissionId,
|
|
356
390
|
correlation_id,
|
|
@@ -463,7 +497,72 @@ export async function ingest(submission, options) {
|
|
|
463
497
|
return { record, path: null, written: false, duplicate: false };
|
|
464
498
|
}
|
|
465
499
|
const persistStart = Date.now();
|
|
466
|
-
|
|
500
|
+
let path, written;
|
|
501
|
+
try {
|
|
502
|
+
({ path, written } = writeRecord(record, repoRoot));
|
|
503
|
+
} catch (err) {
|
|
504
|
+
// F-4acd28d8: computeRecordPath()'s traversal guard (isUnsafeSegment) is
|
|
505
|
+
// STRICTER than the submission schema's repo pattern (F-bbbe2e1f — e.g.
|
|
506
|
+
// `../etc` is schema-valid but traversal-unsafe), so a record can reach
|
|
507
|
+
// here without `_skipPersist` ever having been set. The record's own
|
|
508
|
+
// identifier is what's unfilable — submission-bad, not an operator
|
|
509
|
+
// incident — so route it like `_skipPersist` above instead of letting the
|
|
510
|
+
// throw reach the outer CLI catch, which would misreport it as
|
|
511
|
+
// failed_stage:'cli_parse_payload' and exit 2 ("operator error") for
|
|
512
|
+
// content that is squarely the submitter's to fix.
|
|
513
|
+
//
|
|
514
|
+
// RecordValidationError is the SIBLING gap F-4036ae25's _skipPersist
|
|
515
|
+
// narrowing opens: dogfood-record.schema.json mirrors the submission
|
|
516
|
+
// schema's constraints on every source-authored field verify() copies
|
|
517
|
+
// verbatim (ref, source, timing, scenario_results, overall_verdict.proposed),
|
|
518
|
+
// so a submission that is schema-invalid on one of THOSE fields (not just
|
|
519
|
+
// repo/run_id/timing.finished_at) reaches writeRecord() filable-by-identity
|
|
520
|
+
// and still fails validateRecord() here. Only catch it when the submission
|
|
521
|
+
// was ALREADY schema-invalid (record.verification.schema_valid === false)
|
|
522
|
+
// — that is the authoritative signal this is submission-bad fallout, not a
|
|
523
|
+
// genuine internal defect in what verify() assembled for an
|
|
524
|
+
// otherwise-valid submission, which must keep crashing loudly.
|
|
525
|
+
const isUnfilableRecordPath = err instanceof UnsafeRecordPathError;
|
|
526
|
+
const isSubmissionBadRecordShape =
|
|
527
|
+
err instanceof RecordValidationError && record.verification.schema_valid === false;
|
|
528
|
+
if (!isUnfilableRecordPath && !isSubmissionBadRecordShape) {
|
|
529
|
+
throw err;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// verify() already rejected this submission for a real reason whenever
|
|
533
|
+
// one exists (e.g. repo:mismatch, or the schema violation itself) — those
|
|
534
|
+
// reasons survive untouched. When verify() had no reason to reject (an
|
|
535
|
+
// otherwise-accepted record whose ONLY problem is storage-unsafety),
|
|
536
|
+
// downgrade the verdict here — the same shape used below for a late
|
|
537
|
+
// scenario-load rejection.
|
|
538
|
+
if (isUnfilableRecordPath) {
|
|
539
|
+
record.verification.rejection_reasons.push(`unsafe-record-path: ${err.message}`);
|
|
540
|
+
if (record.verification.status === 'accepted') {
|
|
541
|
+
record.verification.status = 'rejected';
|
|
542
|
+
record.verification.policy_valid = false;
|
|
543
|
+
if (record.overall_verdict.verified === 'pass') {
|
|
544
|
+
record.overall_verdict.verified = 'fail';
|
|
545
|
+
record.overall_verdict.downgraded = true;
|
|
546
|
+
if (!record.overall_verdict.downgrade_reasons) {
|
|
547
|
+
record.overall_verdict.downgrade_reasons = [];
|
|
548
|
+
}
|
|
549
|
+
record.overall_verdict.downgrade_reasons.push('record path could not be safely computed');
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
logStage('rejected_pre_persist', {
|
|
555
|
+
submission_id: submissionId,
|
|
556
|
+
correlation_id,
|
|
557
|
+
reason: isUnfilableRecordPath ? 'unsafe_record_path' : 'record_schema_invalid_from_submission',
|
|
558
|
+
rejection_reasons: record.verification.rejection_reasons ?? [],
|
|
559
|
+
// Operator-debug only: validateRecord()'s OWN structured errors, distinct
|
|
560
|
+
// from (and not merged into) the submitter-facing rejection_reasons above,
|
|
561
|
+
// which stay exactly as verify() computed them against the submission schema.
|
|
562
|
+
...(isSubmissionBadRecordShape ? { record_validation_errors: err.errors } : {})
|
|
563
|
+
});
|
|
564
|
+
return { record, path: null, written: false, duplicate: false };
|
|
565
|
+
}
|
|
467
566
|
logStage('persist_complete', {
|
|
468
567
|
submission_id: submissionId,
|
|
469
568
|
correlation_id,
|
|
@@ -525,7 +624,20 @@ export async function ingest(submission, options) {
|
|
|
525
624
|
}
|
|
526
625
|
}
|
|
527
626
|
|
|
528
|
-
|
|
627
|
+
// F-7b97fbd4 (wave 10): mirror persist_complete's OWN `duplicate: !written`
|
|
628
|
+
// (logged a few lines above, off the same `written`) instead of a bare
|
|
629
|
+
// `false` that silently disagreed with it. By this point `written` can be
|
|
630
|
+
// `false` for exactly one reason — writeRecord()'s internal isDuplicate()
|
|
631
|
+
// blocked the write as a collision (persist.js) — every OTHER
|
|
632
|
+
// non-persisting outcome in this function (record._skipPersist above; the
|
|
633
|
+
// UnsafeRecordPathError / RecordValidationError catch above that) already
|
|
634
|
+
// returns its own honest `duplicate: false` earlier and never reaches this
|
|
635
|
+
// line. The CLI wrapper's `if (result.duplicate)` branch depends on this
|
|
636
|
+
// being accurate: a blocked resubmission must take the terse exit-0
|
|
637
|
+
// `{status:'duplicate'}` path, the same one an early-detected duplicate
|
|
638
|
+
// (the pre-verify probe above) already takes, not fall through to the
|
|
639
|
+
// full rejected-record exit-1 shape.
|
|
640
|
+
return { record, path, written, duplicate: !written };
|
|
529
641
|
}
|
|
530
642
|
|
|
531
643
|
/**
|
|
@@ -664,10 +776,12 @@ export async function verifyOnly(submission, options) {
|
|
|
664
776
|
would_persist_to = computeRecordPath(record, repoRoot);
|
|
665
777
|
} catch {
|
|
666
778
|
// Defensive: if a record passes verify() but still trips path
|
|
667
|
-
// computation (e.g
|
|
668
|
-
// verify-only side-effect-free. Real ingest
|
|
669
|
-
//
|
|
670
|
-
//
|
|
779
|
+
// computation (e.g. F-bbbe2e1f's `../etc` — schema-valid but
|
|
780
|
+
// traversal-unsafe), keep verify-only side-effect-free. Real ingest
|
|
781
|
+
// hits the SAME underlying computeRecordPath failure inside
|
|
782
|
+
// writeRecord() but now catches it too (F-4acd28d8, UnsafeRecordPathError)
|
|
783
|
+
// rather than letting it escape; verify-only just returns null here and
|
|
784
|
+
// lets the operator see the rejection in record.verification.rejection_reasons.
|
|
671
785
|
would_persist_to = null;
|
|
672
786
|
}
|
|
673
787
|
}
|
|
@@ -719,27 +833,30 @@ function emitCliErrorEvent({ failedStage, correlationId, submissionId = null, er
|
|
|
719
833
|
// to live IN the tool, not scattered across docs.
|
|
720
834
|
const USAGE = `ingest — persist a dogfood submission (verify → policy → provenance → write)
|
|
721
835
|
|
|
722
|
-
|
|
836
|
+
Usage:
|
|
723
837
|
node packages/ingest/run.js --file <path> --provenance=github|stub
|
|
724
838
|
node packages/ingest/run.js --payload '<json>' --provenance=github|stub
|
|
725
839
|
echo '<json>' | node packages/ingest/run.js --provenance=github|stub
|
|
726
840
|
|
|
727
|
-
|
|
841
|
+
Input (exactly one; stdin used when neither flag is given):
|
|
728
842
|
--file <path> Read the submission JSON from a file.
|
|
729
843
|
--payload <json> Pass the submission JSON inline.
|
|
730
844
|
(stdin) Pipe the submission JSON on stdin.
|
|
731
845
|
|
|
732
|
-
|
|
846
|
+
Provenance (required for an ingest):
|
|
733
847
|
--provenance=github Confirm the source run via the GitHub API.
|
|
734
848
|
--provenance=stub No-network local confirm (dry-run / dev only).
|
|
735
849
|
|
|
736
|
-
|
|
850
|
+
Modes:
|
|
737
851
|
--verify-only Run the full pipeline WITHOUT writing or rebuilding
|
|
738
852
|
indexes; report where a real ingest WOULD have landed.
|
|
739
853
|
|
|
740
|
-
|
|
854
|
+
Standalone audit verbs (no submission, no stdin, no --provenance):
|
|
741
855
|
--verify-chain Verify the append-only integrity ledger (offline).
|
|
742
|
-
--reconcile Also fail on on-disk records
|
|
856
|
+
--reconcile Also fail on genuine torn-write orphans (on-disk records
|
|
857
|
+
missing from the ledger). Records that predate the
|
|
858
|
+
integrity chain itself are reported separately and do
|
|
859
|
+
not fail the audit.
|
|
743
860
|
--all Report every independent break instead of the first.
|
|
744
861
|
--anchor-compute Compute + write the next XRPL anchor manifest (offline).
|
|
745
862
|
--anchor-post Compute if needed + post the anchor to XRPL (needs XRPL_SEED).
|
|
@@ -747,7 +864,7 @@ STANDALONE AUDIT VERBS (no submission, no stdin, no --provenance):
|
|
|
747
864
|
|
|
748
865
|
-h, --help Show this help.
|
|
749
866
|
|
|
750
|
-
|
|
867
|
+
Exit codes:
|
|
751
868
|
0 success 1 integrity/audit break 2 operator error (flags / IO / JSON)`;
|
|
752
869
|
|
|
753
870
|
const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(__dirname, 'run.js');
|
|
@@ -820,9 +937,38 @@ if (isMain) {
|
|
|
820
937
|
const hasValue = inlineValue !== null || nextIsValue;
|
|
821
938
|
const takeValue = () => (inlineValue !== null ? inlineValue : args[++i]);
|
|
822
939
|
|
|
823
|
-
|
|
940
|
+
// F-e0bcbc47 (amended wave 29, after the first attempt regressed
|
|
941
|
+
// F-INGEST-003): `flagIs(name)` both TESTS the token and RECORDS that
|
|
942
|
+
// `name` is a flag this chain knows about, so the catch-all below can tell
|
|
943
|
+
// "unknown flag" (no branch knows this name) apart from "known flag, no
|
|
944
|
+
// value" (a branch knows the name but its `&& hasValue` guard failed —
|
|
945
|
+
// exactly what F-INGEST-003 causes for `--anchor-network --anchor-compute`).
|
|
946
|
+
// The first attempt at this fix had a bare `else if (arg.startsWith('--'))`
|
|
947
|
+
// catch-all that could not distinguish the two and so reported a KNOWN flag
|
|
948
|
+
// as "unknown argument: --anchor-network" — a claim wider than what it
|
|
949
|
+
// actually checked, which is the exact class this repo keeps paying for.
|
|
950
|
+
//
|
|
951
|
+
// `knownFlagName` is DERIVED from the chain itself rather than duplicated
|
|
952
|
+
// into a second list of flag names: the only way a name enters the known
|
|
953
|
+
// set is for the chain to literally test for it one line below. A parallel
|
|
954
|
+
// `KNOWN_FLAGS` enumeration would be a second population to keep in sync
|
|
955
|
+
// (the same enumeration-vs-property class as CONTROL_CLASS / ZALGO_RUN /
|
|
956
|
+
// DASH_CONFUSABLES), and it would drift the first time someone adds a flag
|
|
957
|
+
// and forgets the list. Adding a branch here cannot desync it — the branch
|
|
958
|
+
// IS the registration. The one residual shape is a future branch written as
|
|
959
|
+
// a bare `arg === '--new'` instead of `flagIs('--new')`; that would make the
|
|
960
|
+
// new flag report as unknown when dangling, and is guarded by
|
|
961
|
+
// f-e0bcbc47-unknown-flag-rejection.test.js's chain-shape assertion.
|
|
962
|
+
let knownFlagName = false;
|
|
963
|
+
const flagIs = (name) => {
|
|
964
|
+
if (arg !== name) return false;
|
|
965
|
+
knownFlagName = true;
|
|
966
|
+
return true;
|
|
967
|
+
};
|
|
968
|
+
|
|
969
|
+
if (flagIs('--provenance') && hasValue) {
|
|
824
970
|
provenanceMode = takeValue();
|
|
825
|
-
} else if (
|
|
971
|
+
} else if (flagIs('--file') && hasValue) {
|
|
826
972
|
const { readFileSync } = await import('node:fs');
|
|
827
973
|
// D1B-001 family (operator-legibility): a --file read failure
|
|
828
974
|
// (ENOENT/EACCES) routes through the structured error event and exits 2
|
|
@@ -842,53 +988,103 @@ if (isMain) {
|
|
|
842
988
|
});
|
|
843
989
|
process.exit(2);
|
|
844
990
|
}
|
|
845
|
-
} else if (
|
|
991
|
+
} else if (flagIs('--payload') && hasValue) {
|
|
846
992
|
submissionJson = takeValue();
|
|
847
|
-
} else if (
|
|
993
|
+
} else if (flagIs('--verify-only')) {
|
|
848
994
|
// F-252714-058: dry-run the pipeline without writing or rebuilding
|
|
849
995
|
// indexes. CI / operators preview what WOULD have been persisted.
|
|
850
996
|
verifyOnlyFlag = true;
|
|
851
|
-
} else if (
|
|
997
|
+
} else if (flagIs('--verify-chain')) {
|
|
852
998
|
// Integrity chain v1: verify the append-only tamper-evident ledger at
|
|
853
999
|
// indexes/integrity/chain.jsonl, fully offline. No submission, no stdin,
|
|
854
1000
|
// no provenance — a standalone audit command.
|
|
855
1001
|
verifyChainFlag = true;
|
|
856
|
-
} else if (
|
|
1002
|
+
} else if (flagIs('--reconcile')) {
|
|
857
1003
|
// Modifier for --verify-chain: also reconcile on-disk records against the
|
|
858
1004
|
// ledger and fail on any orphan (INGEST-PROACT-001).
|
|
859
1005
|
reconcileFlag = true;
|
|
860
|
-
} else if (
|
|
1006
|
+
} else if (flagIs('--all')) {
|
|
861
1007
|
// Modifier for --verify-chain: report every per-record-independent break
|
|
862
1008
|
// instead of stopping at the first (INGEST-PROACT-004).
|
|
863
1009
|
allBreaksFlag = true;
|
|
864
|
-
} else if (
|
|
1010
|
+
} else if (flagIs('--anchor-compute')) {
|
|
865
1011
|
// Optional XRPL anchor: compute + write the next anchor manifest. Offline.
|
|
866
1012
|
anchorComputeFlag = true;
|
|
867
|
-
} else if (
|
|
1013
|
+
} else if (flagIs('--anchor-post')) {
|
|
868
1014
|
// Optional XRPL anchor: compute if needed + post to XRPL. Needs the
|
|
869
1015
|
// optional xrpl package (lazily loaded) and XRPL_SEED.
|
|
870
1016
|
anchorPostFlag = true;
|
|
871
|
-
} else if (
|
|
1017
|
+
} else if (flagIs('--anchor-verify')) {
|
|
872
1018
|
// Optional XRPL anchor: verify local manifests + run the truncation check.
|
|
873
1019
|
// Offline reports honest NOT-verified for the on-chain leg.
|
|
874
1020
|
anchorVerifyFlag = true;
|
|
875
|
-
} else if (
|
|
1021
|
+
} else if (flagIs('--anchor-all')) {
|
|
876
1022
|
// Genesis snapshot mode for compute/post (covers the whole chain).
|
|
877
1023
|
anchorMode = 'all';
|
|
878
|
-
} else if (
|
|
1024
|
+
} else if (flagIs('--anchor-algo') && hasValue) {
|
|
879
1025
|
anchorAlgo = takeValue();
|
|
880
|
-
} else if (
|
|
1026
|
+
} else if (flagIs('--anchor-network') && hasValue) {
|
|
881
1027
|
anchorNetwork = takeValue();
|
|
882
|
-
} else if (
|
|
1028
|
+
} else if (flagIs('--anchor-tx') && hasValue) {
|
|
883
1029
|
// Path to a JSON file containing a fetched XRPL tx (with Memos) for the
|
|
884
1030
|
// on-chain leg of --anchor-verify. Offline-honest: omit it to run the
|
|
885
1031
|
// truncation check only.
|
|
886
1032
|
anchorTxFile = takeValue();
|
|
887
|
-
} else if (
|
|
1033
|
+
} else if (flagIs('--anchor-trusted') && hasValue) {
|
|
888
1034
|
// Comma-separated trusted anchor accounts (UNIONed with the bundled list).
|
|
889
1035
|
anchorTrustedAccounts = takeValue().split(',').map((s) => s.trim()).filter(Boolean);
|
|
890
|
-
} else if (
|
|
1036
|
+
} else if (flagIs('-h') || flagIs('--help') || flagIs('--usage')) {
|
|
891
1037
|
helpFlag = true;
|
|
1038
|
+
} else if (arg.startsWith('--') && knownFlagName) {
|
|
1039
|
+
// F-e0bcbc47 / F-INGEST-003 boundary: a KNOWN flag whose `&& hasValue`
|
|
1040
|
+
// guard failed — i.e. a value flag given no value, because the next
|
|
1041
|
+
// token was itself a flag (`--anchor-network --anchor-compute`) or it
|
|
1042
|
+
// ended argv. F-INGEST-003 pins that the FOLLOWING flag is still parsed
|
|
1043
|
+
// on its own rather than swallowed as this one's value, so this token
|
|
1044
|
+
// keeps its historical path (fall through to positionalArgs, continue)
|
|
1045
|
+
// and `--anchor-compute` runs. Rejecting here instead would break that
|
|
1046
|
+
// pinned contract.
|
|
1047
|
+
//
|
|
1048
|
+
// RESIDUAL, STATED (not fixed here): the dangling flag is still
|
|
1049
|
+
// IGNORED — `--anchor-network` with no value does not set the network
|
|
1050
|
+
// and does not fail the run. That is pre-existing behavior and this
|
|
1051
|
+
// amend does not change it. What changes is that it is no longer
|
|
1052
|
+
// SILENT: an operator gets a structured warn naming the flag instead of
|
|
1053
|
+
// the value vanishing with no trace. Making it a hard error is a real
|
|
1054
|
+
// contract change (it would stop `--anchor-compute` from running and
|
|
1055
|
+
// rewrite F-INGEST-003's pin), which is a bigger claim than a Stage C
|
|
1056
|
+
// humanization amend should make unilaterally — filed as a follow-up
|
|
1057
|
+
// rather than landed here.
|
|
1058
|
+
logStage('warn', {
|
|
1059
|
+
kind: 'cli_flag_missing_value',
|
|
1060
|
+
flag: arg,
|
|
1061
|
+
correlation_id: synthCorrelationId(),
|
|
1062
|
+
message: `${arg} was given no value and is being ignored (the next token is a flag, not a value)`
|
|
1063
|
+
});
|
|
1064
|
+
positionalArgs.push(args[i]);
|
|
1065
|
+
} else if (arg.startsWith('--')) {
|
|
1066
|
+
// F-e0bcbc47 (Stage C humanization): a genuinely unrecognized `--flag`
|
|
1067
|
+
// — no branch above knows this name at all. It used to fall through to
|
|
1068
|
+
// the dead `positionalArgs` sink (declared, pushed to, and never read
|
|
1069
|
+
// anywhere else in this file — confirmed by grep) and the CLI proceeded
|
|
1070
|
+
// to read stdin, hit EOF, and crashed with a raw 'Unexpected end of
|
|
1071
|
+
// JSON input' stack that misattributed the failure to the submission
|
|
1072
|
+
// payload rather than the misspelled flag (live-proven trigger:
|
|
1073
|
+
// `--provenance=stub --fiel <path>`, a one-character typo of --file).
|
|
1074
|
+
// Reject at the point of the typo instead — matches the sibling pattern
|
|
1075
|
+
// already correct in this same domain (packages/verify/cli.js:
|
|
1076
|
+
// `unknown argument: --version` -> exit 2, no stack, no fallthrough).
|
|
1077
|
+
// Scoped to `--`-prefixed tokens only (a bare positional stays in the
|
|
1078
|
+
// historical positionalArgs sink) because that is exactly the shape a
|
|
1079
|
+
// mistyped flag takes, and it is the shape the reachable repro above
|
|
1080
|
+
// hits BEFORE any value token is even read.
|
|
1081
|
+
emitCliErrorEvent({
|
|
1082
|
+
failedStage: 'cli_parse_args',
|
|
1083
|
+
correlationId: synthCorrelationId(),
|
|
1084
|
+
err: new Error(`unknown argument: ${arg}`),
|
|
1085
|
+
humanPrefix: 'invalid CLI invocation'
|
|
1086
|
+
});
|
|
1087
|
+
process.exit(2);
|
|
892
1088
|
} else {
|
|
893
1089
|
positionalArgs.push(args[i]);
|
|
894
1090
|
}
|
|
@@ -923,7 +1119,12 @@ if (isMain) {
|
|
|
923
1119
|
// operator asked for it, so a grep of the NDJSON shows how many breaks and
|
|
924
1120
|
// orphans were found, not just the first break.
|
|
925
1121
|
...(Array.isArray(result.breaks) ? { break_count: result.breaks.length } : {}),
|
|
926
|
-
...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {})
|
|
1122
|
+
...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {}),
|
|
1123
|
+
// F-29134790: pre-adoption records are excluded from chain_ok but still
|
|
1124
|
+
// worth a grep-able count — an operator diffing orphan_count over time
|
|
1125
|
+
// should see the pre-chain figure hold steady while orphan_count reflects
|
|
1126
|
+
// only genuine torn writes.
|
|
1127
|
+
...(Array.isArray(result.pre_adoption) ? { pre_adoption_count: result.pre_adoption.length } : {})
|
|
927
1128
|
});
|
|
928
1129
|
const lines = formatChainResult(result);
|
|
929
1130
|
if (result.ok) {
|
package/verify-chain.js
CHANGED
|
@@ -27,6 +27,17 @@
|
|
|
27
27
|
* push that touched a record but not the chain), and it catches middle-deletion,
|
|
28
28
|
* reorder, and forged-insertion (they break the seq run or the prev-link).
|
|
29
29
|
*
|
|
30
|
+
* The `pre_adoption` exemption inside collectOrphanRecords (below) narrows the
|
|
31
|
+
* "push that touched a record but not the chain" promise one step further: it
|
|
32
|
+
* excuses an un-ledgered, no-integrity record from `ok` only when the record's
|
|
33
|
+
* OWN claimed timing.finished_at/verification.verified_at predates
|
|
34
|
+
* CHAIN_ADOPTION_DATE (F-1907b2bc, wave 14), so dropping a record with a
|
|
35
|
+
* fabricated RECENT date is still caught as a genuine orphan. Dropping one with
|
|
36
|
+
* a fabricated OLD date is not caught — that record is indistinguishable from
|
|
37
|
+
* genuine history, because both are self-reported fields the same privileged
|
|
38
|
+
* writer controls. That residual is the SAME already-disclosed ingest-write-
|
|
39
|
+
* credential trust boundary named two paragraphs up, not a new one.
|
|
40
|
+
*
|
|
30
41
|
* KNOWN LIMITATION — tail truncation. Removing the most-recent entries (and
|
|
31
42
|
* their record files) leaves a SHORTER but internally-consistent chain, which
|
|
32
43
|
* verifies OK: the offline verifier has no external record of the expected head
|
|
@@ -72,7 +83,16 @@ import { findJsonFiles } from './rebuild-indexes.js';
|
|
|
72
83
|
* @property {ChainBreak[]} [breaks] - All independent breaks (only present when
|
|
73
84
|
* `collectAllBreaks` is set). The first element equals `break`.
|
|
74
85
|
* @property {ChainOrphan[]} [orphans] - Records on disk but absent from the
|
|
75
|
-
* ledger (only present when `collectOrphans`
|
|
86
|
+
* ledger AND written by chain-aware code (only present when `collectOrphans`
|
|
87
|
+
* is set). These are genuine torn writes — persist-succeeded, ledger-append-
|
|
88
|
+
* missed — and they make `ok` false.
|
|
89
|
+
* @property {ChainOrphan[]} [pre_adoption] - Records on disk but absent from
|
|
90
|
+
* the ledger that predate the chain's adoption: no `integrity` block AND a
|
|
91
|
+
* self-reported timing.finished_at/verification.verified_at before
|
|
92
|
+
* CHAIN_ADOPTION_DATE (F-1907b2bc — a no-integrity record whose own claimed
|
|
93
|
+
* timestamp is on/after that date is a genuine `orphans` entry instead;
|
|
94
|
+
* only present when `collectOrphans` is set). Visible for operator
|
|
95
|
+
* awareness; does NOT make `ok` false.
|
|
76
96
|
*/
|
|
77
97
|
|
|
78
98
|
/**
|
|
@@ -93,6 +113,10 @@ import { findJsonFiles } from './rebuild-indexes.js';
|
|
|
93
113
|
* ledger (a torn write between record-write and ledger-append). An orphan
|
|
94
114
|
* makes the result `ok: false` — the audit DETECTS the orphan instead of
|
|
95
115
|
* silently passing while a real record lives outside the tamper-evident chain.
|
|
116
|
+
* F-29134790 (wave 12): also returns `pre_adoption[]` — records absent from
|
|
117
|
+
* the ledger that predate the chain's existence entirely (see
|
|
118
|
+
* collectOrphanRecords's adoption-boundary doc below). Pre-adoption records
|
|
119
|
+
* are visible for operator awareness but do NOT affect `ok`.
|
|
96
120
|
* @returns {ChainVerifyResult}
|
|
97
121
|
*/
|
|
98
122
|
export function verifyChain(repoRoot, opts = {}) {
|
|
@@ -112,7 +136,10 @@ export function verifyChain(repoRoot, opts = {}) {
|
|
|
112
136
|
head_digest: GENESIS_DIGEST,
|
|
113
137
|
break: brk,
|
|
114
138
|
...(collectAllBreaks ? { breaks: [brk] } : {}),
|
|
115
|
-
|
|
139
|
+
// pre_adoption included alongside orphans (both empty here) so the
|
|
140
|
+
// result shape is consistent whenever collectOrphans is set, regardless
|
|
141
|
+
// of which early-return path produced it.
|
|
142
|
+
...(collectOrphans ? { orphans: [], pre_adoption: [] } : {}),
|
|
116
143
|
};
|
|
117
144
|
}
|
|
118
145
|
|
|
@@ -191,8 +218,12 @@ export function verifyChain(repoRoot, opts = {}) {
|
|
|
191
218
|
prevDigest = entry.submission_digest;
|
|
192
219
|
}
|
|
193
220
|
|
|
194
|
-
const
|
|
221
|
+
const reconciled = collectOrphans ? collectOrphanRecords(repoRoot, entries) : null;
|
|
222
|
+
const orphans = reconciled ? reconciled.orphans : null;
|
|
223
|
+
const preAdoption = reconciled ? reconciled.preAdoption : null;
|
|
195
224
|
|
|
225
|
+
// Only genuine orphans (post-adoption, ledger-eligible records) affect ok.
|
|
226
|
+
// pre_adoption is informational — see collectOrphanRecords's doc comment.
|
|
196
227
|
const ok = breaks.length === 0 && (orphans === null || orphans.length === 0);
|
|
197
228
|
return {
|
|
198
229
|
ok,
|
|
@@ -200,36 +231,168 @@ export function verifyChain(repoRoot, opts = {}) {
|
|
|
200
231
|
head_digest: headDigest,
|
|
201
232
|
break: breaks.length > 0 ? breaks[0] : null,
|
|
202
233
|
...(collectAllBreaks ? { breaks } : {}),
|
|
203
|
-
...(collectOrphans ? { orphans } : {}),
|
|
234
|
+
...(collectOrphans ? { orphans, pre_adoption: preAdoption } : {}),
|
|
204
235
|
};
|
|
205
236
|
}
|
|
206
237
|
|
|
238
|
+
/**
|
|
239
|
+
* The commit that gave persist.js's writeRecord its UNCONDITIONAL
|
|
240
|
+
* integrity-stamping logic — f4ca987 "feat(integrity): tamper-evident hash
|
|
241
|
+
* chain over records/", 2026-06-21T18:35:36-04:00 (2026-06-21T22:35:36Z).
|
|
242
|
+
* Every record genuinely written by writeRecord() from this date onward
|
|
243
|
+
* carries a fully-formed `integrity` block; a record with none at all can
|
|
244
|
+
* only predate it — see collectOrphanRecords' F-1907b2bc doc paragraph below.
|
|
245
|
+
*
|
|
246
|
+
* Set to the START of the commit's UTC calendar day, not its exact
|
|
247
|
+
* timestamp, so the boundary errs toward NOT excusing a borderline record
|
|
248
|
+
* (the fail-loud direction) rather than toward hair-splitting a commit
|
|
249
|
+
* second no real record is anywhere near — the actual gap in this repo's own
|
|
250
|
+
* history is five weeks wide (last genuine pre-chain record 2026-05-26,
|
|
251
|
+
* first genuine ledgered record 2026-07-03; see F-29134790's doc paragraph
|
|
252
|
+
* below), so this constant has a wide margin on both sides.
|
|
253
|
+
*/
|
|
254
|
+
export const CHAIN_ADOPTION_DATE = new Date('2026-06-21T00:00:00Z');
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* The later of a record's own self-reported `timing.finished_at` /
|
|
258
|
+
* `verification.verified_at`, or `null` when neither is present and
|
|
259
|
+
* parseable. F-1907b2bc: used only to sanity-check a would-be `preAdoption`
|
|
260
|
+
* bucketing — this is NOT a trust boundary (the same privileged writer that
|
|
261
|
+
* can drop a record with no `integrity` block can also forge these fields;
|
|
262
|
+
* see the top-of-file threat-model paragraph). It only stops the tool from
|
|
263
|
+
* asserting "predates the chain" when the record's own data already
|
|
264
|
+
* contradicts that claim, or supports no claim at all.
|
|
265
|
+
*
|
|
266
|
+
* F-a85b7e84 (wave 16, LOW): a candidate must be a `string` before it is
|
|
267
|
+
* handed to `new Date()`. JSON has no Date type — every field a real
|
|
268
|
+
* writeRecord() call ever produces here is a string (Date.prototype.toJSON
|
|
269
|
+
* always serializes to one) or absent — so `typeof value === 'string'` costs
|
|
270
|
+
* nothing against genuine records. Without it, `new Date()`'s permissive
|
|
271
|
+
* ToPrimitive coercion accepts non-date JSON values that a bare falsy check
|
|
272
|
+
* (`if (!value) continue`) does not filter out: `new Date(true)` coerces the
|
|
273
|
+
* boolean to the number 1 (1970-01-01T00:00:00.001Z), a bare integer is read
|
|
274
|
+
* as an epoch-millisecond offset, and an array's comma-joined
|
|
275
|
+
* `toString()` (e.g. `[2026, 3, 1]` -> `'2026,3,1'`) fallback-parses as a
|
|
276
|
+
* real date — all landing well before CHAIN_ADOPTION_DATE with none of the
|
|
277
|
+
* "type a plausible fake ISO string" effort the surviving forgery residual
|
|
278
|
+
* (this function's own doc paragraph above) assumes. A non-string value now
|
|
279
|
+
* falls through to the same "no readable timestamp" orphan branch as an
|
|
280
|
+
* absent field, exactly like an unparseable string already did.
|
|
281
|
+
*
|
|
282
|
+
* @param {object} record
|
|
283
|
+
* @returns {Date|null}
|
|
284
|
+
*/
|
|
285
|
+
function latestClaimedTimestamp(record) {
|
|
286
|
+
const candidates = [record.timing?.finished_at, record.verification?.verified_at];
|
|
287
|
+
let latest = null;
|
|
288
|
+
for (const value of candidates) {
|
|
289
|
+
if (typeof value !== 'string') continue;
|
|
290
|
+
const parsed = new Date(value);
|
|
291
|
+
if (isNaN(parsed.getTime())) continue;
|
|
292
|
+
if (latest === null || parsed.getTime() > latest.getTime()) latest = parsed;
|
|
293
|
+
}
|
|
294
|
+
return latest;
|
|
295
|
+
}
|
|
296
|
+
|
|
207
297
|
/**
|
|
208
298
|
* INGEST-PROACT-001 — reconcile on-disk records against the ledger.
|
|
209
299
|
*
|
|
210
300
|
* Walks every `*.json` under `records/` (accepted AND `_rejected/`) and flags
|
|
211
|
-
* any record file whose
|
|
212
|
-
*
|
|
213
|
-
*
|
|
301
|
+
* any record file whose repo-relative PATH is absent from the ledger. Such a
|
|
302
|
+
* file is an ORPHAN: persist wrote the record but the ledger append did not
|
|
303
|
+
* happen (a crash in the torn window between the two steps), so the record
|
|
214
304
|
* exists OUTSIDE the tamper-evident chain and the plain chain-walk is blind to
|
|
215
305
|
* it. The chain-manifest's own ledger lines are NOT under `records/`, so they
|
|
216
306
|
* are never mistaken for orphans.
|
|
217
307
|
*
|
|
308
|
+
* F-d4bcf5d0 (wave 10, HIGH): this used to ALSO fall back to a run_id-only
|
|
309
|
+
* match — "a record is accounted for if EITHER its path OR its run_id is
|
|
310
|
+
* ledgered anywhere". That fallback was unsound: a single run_id can
|
|
311
|
+
* legitimately own TWO on-disk records at TWO different paths (the
|
|
312
|
+
* accepted-after-a-prior-rejection resubmission flow persist.js's
|
|
313
|
+
* isRetryableRejection carve-out exists for — see
|
|
314
|
+
* F-0f9e4077/F-4036ae25/F-f8952a50), each needing its OWN ledger line. Once
|
|
315
|
+
* ANY entry for a run_id was ledgered, the fallback treated EVERY later
|
|
316
|
+
* record sharing that run_id as accounted for — including one at a
|
|
317
|
+
* genuinely un-ledgered path, exactly the torn-write shape this
|
|
318
|
+
* reconciliation pass exists to catch (proven live: a ledgered `_rejected`
|
|
319
|
+
* record followed by an accepted record for the SAME run_id whose OWN
|
|
320
|
+
* ledger line was then lost — the plain run_id match hid the orphan).
|
|
321
|
+
* Path identity is both the necessary AND the sufficient check: every ledger
|
|
322
|
+
* line carries an exact `path` field (lib/chain-manifest.js's documented
|
|
323
|
+
* line shape, always populated by persist.js's appendChainEntry call), and
|
|
324
|
+
* this function's own `relPath` computation (`relative(repoRoot,
|
|
325
|
+
* absPath).split(sep).join('/')`) is the identical normalization persist.js
|
|
326
|
+
* uses to produce that field. Dropping the run_id fallback removes the
|
|
327
|
+
* unsound generalization without weakening real coverage — see
|
|
328
|
+
* ingest-proact-001-004-verify-chain-reconcile-allbreaks.test.js's
|
|
329
|
+
* 'shared run_id, different paths' regression test.
|
|
330
|
+
*
|
|
331
|
+
* F-29134790 (wave 12, HIGH): path-identity membership is correct for records
|
|
332
|
+
* the chain COULD have ledgered, but on its own it has no ADOPTION BOUNDARY —
|
|
333
|
+
* every un-ledgered path reads identically, so a record that predates the
|
|
334
|
+
* chain feature's existence is indistinguishable from a genuine crash-window
|
|
335
|
+
* torn write. Proven live against this repo's own `records/` tree: 53 files
|
|
336
|
+
* with no ledger line, every one dated 2026-03-19 through 2026-05-26 — all
|
|
337
|
+
* BEFORE persist.js gained its integrity-stamping logic (f4ca987,
|
|
338
|
+
* 2026-06-21) — versus the 7 real ledgered records, all dated 2026-07-03
|
|
339
|
+
* onward. Pre-fix, `verifyChain({ collectOrphans: true })` reported all 53 as
|
|
340
|
+
* indistinguishable "torn persist" orphans (a 53:0 signal-to-noise ratio; a
|
|
341
|
+
* genuine orphan today would be the 54th unlabeled line).
|
|
342
|
+
*
|
|
343
|
+
* The fix needs no new marker file or backfill tool: persist.js's writeRecord
|
|
344
|
+
* already stamps `record.integrity = { submission_digest, prev_digest, seq }`
|
|
345
|
+
* UNCONDITIONALLY, in memory, BEFORE it writes the record file and BEFORE it
|
|
346
|
+
* appends the ledger line (see persist.js — the stamp happens ahead of
|
|
347
|
+
* validateRecord() and the atomic file write). That ordering guarantees a
|
|
348
|
+
* genuine post-adoption torn write (file landed, ledger append lost) still
|
|
349
|
+
* carries a fully-formed `integrity` block on disk; only a record written
|
|
350
|
+
* before persist.js gained this stamping logic can lack one entirely. So
|
|
351
|
+
* "does this record carry an `integrity` block" is not a proxy for the
|
|
352
|
+
* adoption boundary — it IS the adoption boundary, already recorded per-record
|
|
353
|
+
* by the exact code change that introduced the chain, for free. A record
|
|
354
|
+
* missing `integrity` is bucketed into the returned `preAdoption` array
|
|
355
|
+
* (visible, does not fail `ok`); a record WITH an `integrity` block that is
|
|
356
|
+
* still un-ledgered remains a genuine `orphans` entry exactly as before.
|
|
357
|
+
*
|
|
358
|
+
* F-1907b2bc (wave 14, MEDIUM): the paragraph above treats "does this record
|
|
359
|
+
* carry an `integrity` block" as the WHOLE adoption boundary, but that
|
|
360
|
+
* equivalence only holds for records that actually passed through the real
|
|
361
|
+
* writeRecord(). A record dropped directly onto disk — bypassing writeRecord
|
|
362
|
+
* entirely — also has no `integrity` block, and pre-fix was silently
|
|
363
|
+
* absorbed into `preAdoption` with a "predates the chain" reason string that
|
|
364
|
+
* is provably FALSE whenever the dropped record's own claimed timestamp is on
|
|
365
|
+
* or after CHAIN_ADOPTION_DATE (persist.js has stamped `integrity`
|
|
366
|
+
* UNCONDITIONALLY on every record it writes since that date — see the
|
|
367
|
+
* constant's doc comment above — so a record genuinely written from then on
|
|
368
|
+
* could not lack one). The fix cross-checks a would-be `preAdoption` record's
|
|
369
|
+
* own `timing.finished_at`/`verification.verified_at` (the later of the two
|
|
370
|
+
* — see `latestClaimedTimestamp`) against CHAIN_ADOPTION_DATE before
|
|
371
|
+
* accepting the "predates the chain" narrative; a record with no verifiable
|
|
372
|
+
* timestamp of its own, or one on/after the cutoff, is bucketed into
|
|
373
|
+
* `orphans` instead, with a reason string honest about why. This closes the
|
|
374
|
+
* LOW-effort version of dropping a forged record (not bothering to fake a
|
|
375
|
+
* plausible pre-adoption date) but not a DILIGENT one that also forges an old
|
|
376
|
+
* timing/verification block — no purely offline, record-content-based check
|
|
377
|
+
* can tell a forged old date from a genuine one; see the threat-model
|
|
378
|
+
* paragraph at the top of this file.
|
|
379
|
+
*
|
|
218
380
|
* @param {string} repoRoot
|
|
219
381
|
* @param {Array<object>} entries - The ledger entries (already read).
|
|
220
|
-
* @returns {ChainOrphan[]}
|
|
382
|
+
* @returns {{ orphans: ChainOrphan[], preAdoption: ChainOrphan[] }} `orphans`
|
|
383
|
+
* are un-ledgered records stamped by chain-aware code — genuine torn
|
|
384
|
+
* writes. `preAdoption` are un-ledgered records with no `integrity` block —
|
|
385
|
+
* they predate the chain and were never ledgerable. Only `orphans` affects
|
|
386
|
+
* verifyChain's `ok`.
|
|
221
387
|
*/
|
|
222
388
|
function collectOrphanRecords(repoRoot, entries) {
|
|
223
|
-
// Index the ledger by run_id and by repo-relative path so a record matches if
|
|
224
|
-
// EITHER identity is present — a record is ledgered as long as its line exists.
|
|
225
|
-
const ledgerRunIds = new Set();
|
|
226
389
|
const ledgerPaths = new Set();
|
|
227
390
|
for (const e of entries) {
|
|
228
|
-
if (e.run_id) ledgerRunIds.add(e.run_id);
|
|
229
391
|
if (e.path) ledgerPaths.add(e.path);
|
|
230
392
|
}
|
|
231
393
|
|
|
232
394
|
const orphans = [];
|
|
395
|
+
const preAdoption = [];
|
|
233
396
|
const recordsDir = join(repoRoot, 'records');
|
|
234
397
|
for (const absPath of findJsonFiles(recordsDir)) {
|
|
235
398
|
const relPath = relative(repoRoot, absPath).split(sep).join('/');
|
|
@@ -240,7 +403,9 @@ function collectOrphanRecords(repoRoot, entries) {
|
|
|
240
403
|
record = JSON.parse(readFileSync(absPath, 'utf-8'));
|
|
241
404
|
} catch {
|
|
242
405
|
// A record file we cannot even parse, that is also absent from the ledger,
|
|
243
|
-
// is still an orphan — report it with what little identity we have.
|
|
406
|
+
// is still an orphan — report it with what little identity we have. We
|
|
407
|
+
// cannot read `.integrity` to classify it as pre-adoption either, so this
|
|
408
|
+
// fails loud rather than being silently excused.
|
|
244
409
|
orphans.push({
|
|
245
410
|
run_id: null,
|
|
246
411
|
seq: null,
|
|
@@ -250,11 +415,67 @@ function collectOrphanRecords(repoRoot, entries) {
|
|
|
250
415
|
continue;
|
|
251
416
|
}
|
|
252
417
|
|
|
253
|
-
|
|
254
|
-
|
|
418
|
+
// A value that parses but is not a plain record object (null, an array, a
|
|
419
|
+
// bare primitive) is not something we can read `.integrity` off to
|
|
420
|
+
// classify as pre-adoption. Mirroring the not-valid-JSON branch above:
|
|
421
|
+
// when we cannot positively identify what the record actually IS, fail
|
|
422
|
+
// loud as a genuine orphan rather than silently excusing it into
|
|
423
|
+
// preAdoption — the adoption boundary is a claim we can only make about
|
|
424
|
+
// something shaped like a record.
|
|
425
|
+
if (record === null || typeof record !== 'object' || Array.isArray(record)) {
|
|
426
|
+
orphans.push({
|
|
427
|
+
run_id: null,
|
|
428
|
+
seq: null,
|
|
429
|
+
path: relPath,
|
|
430
|
+
reason: `record file present on disk but absent from the ledger, and its JSON content is not a record object`,
|
|
431
|
+
});
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// Adoption boundary — see the F-29134790 doc paragraph above. "No
|
|
436
|
+
// integrity block" is NECESSARY for pre-adoption but, on its own, not
|
|
437
|
+
// SUFFICIENT — see the F-1907b2bc doc paragraph above. Cross-check the
|
|
438
|
+
// record's own claimed timestamp against CHAIN_ADOPTION_DATE before
|
|
439
|
+
// accepting the "predates the chain" narrative.
|
|
440
|
+
if (record.integrity == null) {
|
|
441
|
+
const claimedAt = latestClaimedTimestamp(record);
|
|
442
|
+
const isPreAdoption = claimedAt !== null && claimedAt.getTime() < CHAIN_ADOPTION_DATE.getTime();
|
|
443
|
+
|
|
444
|
+
if (isPreAdoption) {
|
|
445
|
+
preAdoption.push({
|
|
446
|
+
run_id: record.run_id ?? null,
|
|
447
|
+
seq: null,
|
|
448
|
+
path: relPath,
|
|
449
|
+
reason:
|
|
450
|
+
`record predates the integrity chain (no integrity block — written ` +
|
|
451
|
+
`before tamper-evident chaining was introduced); expected for ` +
|
|
452
|
+
`historical records, not counted as an audit failure.`,
|
|
453
|
+
});
|
|
454
|
+
continue;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
orphans.push({
|
|
458
|
+
run_id: record.run_id ?? null,
|
|
459
|
+
seq: null,
|
|
460
|
+
path: relPath,
|
|
461
|
+
reason: claimedAt === null
|
|
462
|
+
? `record has no integrity block, and no readable timing.finished_at ` +
|
|
463
|
+
`or verification.verified_at either — its "predates the chain" claim ` +
|
|
464
|
+
`cannot be supported by anything on the record itself, so it is ` +
|
|
465
|
+
`treated as a genuine orphan rather than silently excused as historical.`
|
|
466
|
+
: `record has no integrity block, but its own timing.finished_at / ` +
|
|
467
|
+
`verification.verified_at (latest: ${claimedAt.toISOString()}) is on or ` +
|
|
468
|
+
`after the chain-adoption date (${CHAIN_ADOPTION_DATE.toISOString()}) — ` +
|
|
469
|
+
`a record genuinely written by writeRecord() from that date onward ` +
|
|
470
|
+
`always carries an integrity block, so this one cannot genuinely ` +
|
|
471
|
+
`predate the chain. Re-run ingest for this record or investigate how ` +
|
|
472
|
+
`it reached disk without one.`,
|
|
473
|
+
});
|
|
474
|
+
continue;
|
|
475
|
+
}
|
|
255
476
|
|
|
256
477
|
orphans.push({
|
|
257
|
-
run_id:
|
|
478
|
+
run_id: record.run_id ?? null,
|
|
258
479
|
seq: record.integrity?.seq ?? null,
|
|
259
480
|
path: relPath,
|
|
260
481
|
reason:
|
|
@@ -264,7 +485,7 @@ function collectOrphanRecords(repoRoot, entries) {
|
|
|
264
485
|
});
|
|
265
486
|
}
|
|
266
487
|
|
|
267
|
-
return orphans;
|
|
488
|
+
return { orphans, preAdoption };
|
|
268
489
|
}
|
|
269
490
|
|
|
270
491
|
/**
|
|
@@ -282,6 +503,13 @@ export function formatChainResult(result) {
|
|
|
282
503
|
];
|
|
283
504
|
if (Array.isArray(result.orphans)) {
|
|
284
505
|
lines.push(`reconciliation: 0 orphan record(s) on disk`);
|
|
506
|
+
// F-29134790: ok:true only guarantees zero GENUINE orphans — pre-adoption
|
|
507
|
+
// records (no integrity block) are excused from ok but still worth
|
|
508
|
+
// surfacing so an operator isn't left wondering why the disk record
|
|
509
|
+
// count exceeds the ledgered count.
|
|
510
|
+
if (Array.isArray(result.pre_adoption) && result.pre_adoption.length > 0) {
|
|
511
|
+
lines.push(` (${result.pre_adoption.length} pre-chain record(s) predate the integrity ledger and are excluded from reconciliation)`);
|
|
512
|
+
}
|
|
285
513
|
}
|
|
286
514
|
return lines;
|
|
287
515
|
}
|
|
@@ -312,5 +540,13 @@ export function formatChainResult(result) {
|
|
|
312
540
|
}
|
|
313
541
|
}
|
|
314
542
|
|
|
543
|
+
// F-29134790: pre-adoption records never make ok false, but when the result
|
|
544
|
+
// IS not-ok for some other reason (a break, or a genuine orphan), still name
|
|
545
|
+
// the excluded count so an operator triaging the failure isn't misled into
|
|
546
|
+
// thinking every un-ledgered record on disk is part of the break.
|
|
547
|
+
if (Array.isArray(result.pre_adoption) && result.pre_adoption.length > 0) {
|
|
548
|
+
lines.push(`reconciliation: ${result.pre_adoption.length} pre-chain record(s) predate the integrity ledger (excluded from reconciliation, not an audit failure)`);
|
|
549
|
+
}
|
|
550
|
+
|
|
315
551
|
return lines;
|
|
316
552
|
}
|