@dogfood-lab/ingest 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/integrity.js CHANGED
@@ -62,13 +62,27 @@ export function canonicalize(value) {
62
62
  /**
63
63
  * Deep clone with object keys sorted ascending at every depth. Arrays keep
64
64
  * order (only their element objects are key-sorted). Primitives pass through.
65
+ *
66
+ * F-755d0f3f: the accumulator MUST be null-prototype. On a plain `{}` the
67
+ * assignment below would hit the inherited `Object.prototype.__proto__` setter
68
+ * for an own `__proto__` key and retarget the accumulator's prototype instead of
69
+ * creating an own key — dropping that field, and everything under it, from the
70
+ * canonical string and therefore from the record digest. `JSON.parse` creates a
71
+ * real own `__proto__` data property (CreateDataProperty, not Set), so any
72
+ * record file on disk can carry one, and verify-chain.js reaches this via a bare
73
+ * `JSON.parse` with no schema gate. That made two materially different records
74
+ * hash identically and let a tampered record re-verify clean.
75
+ *
76
+ * `JSON.stringify` serializes a null-prototype object identically to a plain
77
+ * one, so this is byte-for-byte compatible with every already-persisted record
78
+ * and the committed chain is unaffected.
65
79
  */
66
80
  function sortDeep(value) {
67
81
  if (Array.isArray(value)) {
68
82
  return value.map(sortDeep);
69
83
  }
70
84
  if (value !== null && typeof value === 'object') {
71
- const sorted = {};
85
+ const sorted = Object.create(null);
72
86
  for (const key of Object.keys(value).sort()) {
73
87
  sorted[key] = sortDeep(value[key]);
74
88
  }
package/load-context.js CHANGED
@@ -180,24 +180,6 @@ export function loadRepoPolicy(repoSlug, repoRoot) {
180
180
  return parsed;
181
181
  }
182
182
 
183
- /**
184
- * Default scenario fetcher that reads from the local filesystem.
185
- * Used when dogfood-labs is dogfooding itself.
186
- *
187
- * @param {string} repoRoot - Root of the source repo
188
- * @returns {object} Scenario fetch adapter
189
- */
190
- export function localScenarioFetcher(repoRoot) {
191
- return {
192
- async fetch(scenarioId) {
193
- if (!/^[\w-]+$/.test(scenarioId)) return null;
194
- const path = join(repoRoot, 'dogfood', 'scenarios', `${scenarioId}.yaml`);
195
- if (!existsSync(path)) return null;
196
- return yaml.load(readFileSync(path, 'utf-8'));
197
- }
198
- };
199
- }
200
-
201
183
  /**
202
184
  * Default per-request timeout for the GitHub scenario fetch. Mirrors the
203
185
  * sibling `GITHUB_PROVENANCE_TIMEOUT_MS` constant at
@@ -271,6 +253,64 @@ function defaultSleep(ms) {
271
253
  return new Promise((r) => setTimeout(r, ms));
272
254
  }
273
255
 
256
+ /**
257
+ * COORD-001: parse a scenario body fetched from a consumer's source repo —
258
+ * the one `yaml.load` call site in this file that sees content a hostile or
259
+ * compromised submitter controls end to end. js-yaml 4.1.1 (the workspace
260
+ * floor; see package.json `"js-yaml": "^4.1.0"`) carries
261
+ * GHSA-h67p-54hq-rp68: chained `<<` merge keys cost O(depth) per level,
262
+ * because `mergeMappings()` (js-yaml lib/loader.js:304-321) re-copies
263
+ * `Object.keys(source)` for the FULL accumulated mapping at every level —
264
+ * so an n-level merge chain costs O(n^2) total. The attack is small BY
265
+ * CONSTRUCTION (each level adds only a couple of YAML lines), so
266
+ * GITHUB_SCENARIO_MAX_BYTES (a 1 MiB cap on the wire size) does not defend
267
+ * it: measured on the reference rig, a chain sized right at that 1 MiB cap
268
+ * costs ~61s of CPU, and scaling is ~quadratic (68 KB -> 127ms, 145 KB ->
269
+ * 656ms). Bumping js-yaml to v5 (which drops merge-key support outright)
270
+ * is blocked — Dependabot #50 is held open because v5 breaks two scripts/
271
+ * gates — so the mitigation lives here instead of in the dependency.
272
+ *
273
+ * CORE_SCHEMA is DEFAULT_SCHEMA minus exactly the YAML-1.1 extras
274
+ * (`timestamp`, `merge`, `binary`, `omap`, `pairs`, `set` — see js-yaml
275
+ * lib/schema/default.js). Without the registered `merge` type, the
276
+ * loader's `keyTag === 'tag:yaml.org,2002:merge'` branch in
277
+ * storeMappingPair() never matches a `<<` key, so `mergeMappings()` is
278
+ * never invoked and the O(depth) copy loop simply cannot run — `<<`
279
+ * resolves as an ordinary (and, against scenario.schema.json, rejected —
280
+ * `additionalProperties: false`) string key instead. This is verified
281
+ * behaviourally, not assumed from the option name, in
282
+ * coord-001-scenario-merge-bomb.test.js: a probe document proves the
283
+ * merge-key never fires, and a second probe proves the real fetch path
284
+ * (not just this helper in isolation) stays protected. Every field
285
+ * scenario.schema.json declares is a plain string/object/array/boolean —
286
+ * none relies on the dropped timestamp/binary/omap/pairs/set types — so
287
+ * this is a pure security hardening with no behavioural cost to a
288
+ * conforming scenario document.
289
+ *
290
+ * Scoped deliberately to THIS call site only. The other two `yaml.load`
291
+ * calls in this file (loadGlobalPolicy, loadRepoPolicy) both read
292
+ * maintainer-committed local files — the attacker does not control their
293
+ * content, only (in loadRepoPolicy's case) which existing file gets
294
+ * selected, and that selection is already segment-validated above.
295
+ * Widening this schema change to those sites is unnecessary risk for zero
296
+ * additional coverage.
297
+ *
298
+ * F-3be85850: this comment previously also named a THIRD "local files"
299
+ * call site, localScenarioFetcher — removed as dead code (zero callers
300
+ * anywhere in the repo, and not reachable even as an external package hook:
301
+ * load-context.js is not in @dogfood-lab/ingest's package.json `exports`
302
+ * map). Its own doc comment claimed it was "Used when dogfood-labs is
303
+ * dogfooding itself," but the real self-dogfood path (self-dogfood.yml)
304
+ * routes through the SAME public ingest.yml repository_dispatch pipeline
305
+ * every consumer uses, via githubScenarioFetcher below — never a local-disk
306
+ * reader. dogfood/scenarios/*.yaml remain the canonical scenario
307
+ * definitions; they are simply fetched over the GitHub Contents API rather
308
+ * than off local disk, even for this repo's own CI.
309
+ */
310
+ export function parseUntrustedScenarioYaml(text) {
311
+ return yaml.load(text, { schema: yaml.CORE_SCHEMA });
312
+ }
313
+
274
314
  /**
275
315
  * GitHub scenario fetcher. Loads scenario definitions from a source repo
276
316
  * via the GitHub API at a specific commit SHA.
@@ -458,7 +498,10 @@ export function githubScenarioFetcher(token, repoSlug, commitSha, opts = {}) {
458
498
 
459
499
  let scenario;
460
500
  try {
461
- scenario = yaml.load(text);
501
+ // COORD-001: merge-key DoS defense — see parseUntrustedScenarioYaml's
502
+ // doc comment. `text` is untrusted (fetched from the submitter's
503
+ // source repo), so it must never reach plain yaml.load().
504
+ scenario = parseUntrustedScenarioYaml(text);
462
505
  } catch {
463
506
  return { scenario: null, reason: 'parse_error', retryable: false };
464
507
  }
@@ -488,7 +531,22 @@ export function githubScenarioFetcher(token, repoSlug, commitSha, opts = {}) {
488
531
  for (let i = 0; i < attempts; i++) {
489
532
  last = await attemptOnce(scenarioId);
490
533
  if (last.scenario || !last.retryable || i === attempts - 1) break;
491
- await sleep(Math.min(RETRY_BASE_MS * (1 << i), RETRY_MAX_MS));
534
+ const waitMs = Math.min(RETRY_BASE_MS * (1 << i), RETRY_MAX_MS);
535
+ // F-2a5ddafa: this loop was completely silent on every retry but the
536
+ // last — an operator had zero early-warning signal that the source
537
+ // repo's API was degrading until the retry budget fully exhausted and
538
+ // threw (see the sibling fix in
539
+ // packages/verify/validators/provenance.js's confirm() loops, same
540
+ // finding). 'warn' (not 'error') since the fetch may still succeed on
541
+ // the next attempt.
542
+ logStage('warn', {
543
+ kind: 'scenario_fetch_retry',
544
+ scenario_id: scenarioId,
545
+ attempt: i + 1,
546
+ status_or_reason: last.reason || (last.fault ? last.fault.message : 'unknown'),
547
+ next_backoff_ms: waitMs,
548
+ });
549
+ await sleep(waitMs);
492
550
  }
493
551
  // V2-CROSS-BO-001: a transient fault that survived every retry is an
494
552
  // outage — throw the classified operational error instead of returning a
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dogfood-lab/ingest",
3
- "version": "1.9.0",
3
+ "version": "1.11.0",
4
4
  "type": "module",
5
5
  "description": "Ingestion pipeline for testing-os. Thin glue: dispatch → verifier → persist → indexes.",
6
6
  "main": "run.js",
package/persist.js CHANGED
@@ -6,7 +6,7 @@
6
6
  * duplicate detection by run_id, directory creation.
7
7
  */
8
8
 
9
- import { existsSync, mkdirSync, writeFileSync, renameSync, openSync, closeSync, unlinkSync } from 'node:fs';
9
+ import { existsSync, mkdirSync, writeFileSync, renameSync, openSync, closeSync, unlinkSync, readFileSync } from 'node:fs';
10
10
  import { join, dirname, relative, sep } from 'node:path';
11
11
  import { randomBytes } from 'node:crypto';
12
12
 
@@ -14,6 +14,8 @@ import { validateRecord } from './validate-record.js';
14
14
  import { isUnsafeSegment } from './lib/unsafe-segment.js';
15
15
  import { submissionDigest } from './lib/integrity.js';
16
16
  import { readChainHead, appendChainEntry } from './lib/chain-manifest.js';
17
+ import { parseRejectionReason } from '@dogfood-lab/verify';
18
+ import { SUPPORTED_SCHEMA_VERSIONS } from '@dogfood-lab/schemas';
17
19
 
18
20
  /**
19
21
  * Error thrown when writeRecord loses a TOCTOU race for the same canonical path.
@@ -29,6 +31,40 @@ export class DuplicateRunIdError extends Error {
29
31
  }
30
32
  }
31
33
 
34
+ /**
35
+ * Error thrown when writeRecord's SECOND computeRecordPath() call — the one
36
+ * that computes the real write target, reached only AFTER validateRecord()
37
+ * has already confirmed the record is schema-valid — still cannot produce a
38
+ * safe path.
39
+ *
40
+ * F-bbbe2e1f: computeRecordPath()'s isUnsafeSegment check (lib/unsafe-segment.js)
41
+ * is STRICTER than the record schema's own repo pattern
42
+ * (`^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$`), which permits an embedded `..` or a
43
+ * lone `.` segment (e.g. `../etc`). A record can therefore pass
44
+ * validateRecord() above and still fail here — this is never a schema
45
+ * problem, it is the traversal backstop firing on schema-valid input. Pre-fix
46
+ * that let a bare, unclassified computeRecordPath Error (no `.code`) escape
47
+ * uncaught — the exact shape F-a37d36f5 eliminated at the FIRST
48
+ * computeRecordPath call (the isDuplicate probe below) but not this second
49
+ * one, three lines after validateRecord(). Classifying it here matches
50
+ * RecordValidationError's discipline and stays fail-closed: nothing below
51
+ * this point has touched the filesystem yet (mkdirSync/openSync/writeFileSync
52
+ * all come after), so no partial write is possible either way.
53
+ */
54
+ export class UnsafeRecordPathError extends Error {
55
+ constructor(record, cause) {
56
+ super(
57
+ `record passed schema validation but its path could not be safely computed ` +
58
+ `(repo: ${record.repo}, run_id: ${record.run_id}): ${cause.message}`,
59
+ { cause }
60
+ );
61
+ this.name = 'UnsafeRecordPathError';
62
+ this.code = 'UNSAFE_RECORD_PATH';
63
+ this.repo = record.repo;
64
+ this.runId = record.run_id;
65
+ }
66
+ }
67
+
32
68
  /**
33
69
  * Compute the canonical file path for a persisted record.
34
70
  *
@@ -88,9 +124,180 @@ export function computeRecordPath(record, repoRoot) {
88
124
  return join(repoRoot, base, org, repo, year, month, day, filename);
89
125
  }
90
126
 
127
+ /**
128
+ * F-4036ae25: a prior `_rejected` record whose rejection_reasons are ALL
129
+ * retryable-class is "the submitter can fix this and resubmit", not an
130
+ * unappealable verdict about the run — the same doctrine that already exempts
131
+ * an unfilable (`_skipPersist`) rejection from consuming its run_id. Narrowing
132
+ * `_skipPersist` in verify/index.js means a filable-but-rejected submission
133
+ * now DOES persist (the fleet-wide audit trail F-4036ae25 restores), so this
134
+ * is the other half of that fix: persisting the evidence must not resurrect
135
+ * the exact run_id-poisoning pathology F-82429f90 eliminated for
136
+ * validator-crash faults.
137
+ *
138
+ * F-f8952a50 (wave 10): this used to check `parseRejectionReason(r).prefix
139
+ * === 'schema:'` — ONE of the ten prefixes parse-rejection.js's own "Prefix
140
+ * taxonomy" documents under `class: 'submission-bad'` ("the submitter fixes
141
+ * the payload and resubmits"). Proven live with `repo:mismatch` (a pure
142
+ * identity/addressing mistake — the run happened, only the repo/run_url
143
+ * pairing was mis-stated): a corrected resubmission's payload never even
144
+ * reached verify() a second time, because both isDuplicate() call sites
145
+ * treated the stale `repo:` rejection as an unconditional block — see
146
+ * f-0f9e4077-retry-collision-duplicate-rejection.test.js's 'F-f8952a50'
147
+ * describe block for the end-to-end proof.
148
+ *
149
+ * The fix reads `parseRejectionReason(r).retryable` — the classifier's OWN
150
+ * per-prefix routing decision (see parse-rejection.js's file header for the
151
+ * full split) — instead of re-deriving a prefix allowlist here. A future
152
+ * prefix parse-rejection.js adds is automatically retryable or not with no
153
+ * edit to this file. This is intentionally NARROWER than "every submission-bad
154
+ * class": `retryable` is false for `policy:` and `provenance:` even though
155
+ * both are class `submission-bad` — those two prefixes are a rendered VERDICT
156
+ * on the run's own reported content (see parse-rejection.js), and letting a
157
+ * resubmission retry past one would let a submitter launder a genuinely-bad
158
+ * run into an accepted one by resubmitting different self-reported content
159
+ * under the same run_id. That boundary is independently pinned end-to-end by
160
+ * schema-invalid-skip-persist.test.js's "REGRESSION GUARD" test and the
161
+ * "persist-a-verdict doctrine is preserved" describe block — this fix must
162
+ * not (and, via the per-prefix flag, does not) touch it.
163
+ *
164
+ * `'operational'` / `'ingest'` / `'unknown'`-class reasons are always
165
+ * `retryable: false` too (see parse-rejection.js). Most `'operational'`-class
166
+ * reasons are not actually reachable here in production (F-82429f90 and its
167
+ * provenance-fault:/scenario-fetch-fault: siblings THROW instead of
168
+ * persisting a `_rejected` record — see runValidator in verify/index.js).
169
+ *
170
+ * F-51780da9 (wave 22, confirming audit of F-be0deacd): `CONTRACT_SCHEMA_TOO_NEW:`
171
+ * is the one documented exception where the frozen `retryable: false` is NOT
172
+ * the whole story. `validators/schema-version.js` RETURNS it as an ordinary
173
+ * rejection string rather than throwing, so a too-new-major submission
174
+ * genuinely does persist to `_rejected/` and IS read by this function.
175
+ * `retryable` is a STATIC per-prefix classification stamped at parse time —
176
+ * it has no way to observe that testing-os has SINCE been upgraded past the
177
+ * declared major, so trusting it alone lets one stale TOO_NEW rejection
178
+ * permanently poison a run_id even after the exact condition it was
179
+ * rejecting no longer holds (proven live: seed a `_rejected` record with a
180
+ * TOO_NEW reason declaring a major the CURRENT build already understands —
181
+ * pre-fix this function still returns `false` forever, because it never asks
182
+ * whether the stored major is still actually too new). `reevaluateTooNewRetry`
183
+ * below is the correction: for this ONE prefix, re-derive whether the STORED
184
+ * rejection's declared major is still above the CURRENT build's
185
+ * `SUPPORTED_SCHEMA_VERSIONS.<contract>.maxMajor`, instead of trusting the
186
+ * frozen boolean. The other nine prefixes were swept for the same trap (any
187
+ * retryable semantics that depend on mutable environment rather than the
188
+ * submission itself) and none qualify: schema:/policy-config:/repo:/
189
+ * submission-contains-verifier-field:/unsafe-record-path:/steps[<id>]:/
190
+ * CONTRACT_SCHEMA_TOO_OLD: are already retryable:true (an already-permissive
191
+ * prefix has no "stuck forever" direction to fix); policy:/provenance: are a
192
+ * DELIBERATE, permanent anti-gaming verdict on the run's own reported
193
+ * content, not an environment-dependent snapshot that goes stale;
194
+ * provenance-fault:/scenario-fetch-fault:/submission-malformed:/
195
+ * VALIDATOR_FAULT_*: all throw rather than persist (never reach this
196
+ * function); scenario-load: is evaluated against an immutable git
197
+ * commit_sha, not a build-version boundary, so the same committed content at
198
+ * the same ref parses the same way forever — there is no analogous integer
199
+ * ceiling to re-derive the way TOO_NEW's `maxMajor` comparison allows.
200
+ *
201
+ * Any read/parse failure fails closed (blocking): a corrupted or unreadable
202
+ * evidence file must never silently unblock a path collision. The same
203
+ * failure mode covers a CONTRACT_SCHEMA_TOO_NEW: reason whose shape
204
+ * `reevaluateTooNewRetry` cannot parse, or whose contract key is no longer
205
+ * registered in `SUPPORTED_SCHEMA_VERSIONS` — both fail closed to "still
206
+ * blocking" rather than guessing.
207
+ *
208
+ * @param {string} rejectedPath - Absolute path already confirmed to exist.
209
+ * @returns {boolean} true when every rejection reason is individually
210
+ * retryable (see parseRejectionReason's `retryable` field), OR — for
211
+ * CONTRACT_SCHEMA_TOO_NEW: specifically — no longer applicable because
212
+ * testing-os has since been upgraded past the declared major.
213
+ */
214
+
215
+ /**
216
+ * F-51780da9: matches a `CONTRACT_SCHEMA_TOO_NEW:` reason string emitted by
217
+ * `validators/schema-version.js` — `CONTRACT_SCHEMA_TOO_NEW: <contract>
218
+ * schema v<major>.<minor>.<patch> ...` — capturing the contract key and the
219
+ * declared MAJOR, the two pieces of information needed to re-ask, against
220
+ * the CURRENT build, the exact question the stored rejection answered
221
+ * against a possibly-older one. Anchored to the literal TOO_NEW prefix only
222
+ * (never TOO_OLD:, a distinct literal) and requires a trailing space after
223
+ * the three-part version so it matches both the full production message
224
+ * (`... but this build supports v...`) and a shortened test fixture
225
+ * (`... — upgrade testing-os`) without over-matching a malformed variant.
226
+ */
227
+ const TOO_NEW_DECLARED_VERSION = /^CONTRACT_SCHEMA_TOO_NEW: (\S+) schema v(\d+)\.\d+\.\d+ /;
228
+
229
+ /**
230
+ * F-51780da9: re-evaluate a single STORED `CONTRACT_SCHEMA_TOO_NEW:` reason
231
+ * against the CURRENT build's `SUPPORTED_SCHEMA_VERSIONS`, instead of
232
+ * trusting the frozen `retryable: false` parse-rejection.js stamps on this
233
+ * prefix. `validators/schema-version.js` emits this prefix precisely when
234
+ * `declaredMajor > supported.maxMajor`; this re-runs that identical
235
+ * comparison against whatever `maxMajor` is TODAY, which a testing-os
236
+ * upgrade may have since raised past the stored declared major.
237
+ *
238
+ * @param {string} reason - a single rejection_reasons[] entry.
239
+ * @returns {boolean} `true` when the declared major is no longer above the
240
+ * current build's supported ceiling (the rejection has been resolved by a
241
+ * since-applied upgrade); `false` when it is still genuinely too new, the
242
+ * contract key is unrecognized, or `reason` is not a
243
+ * CONTRACT_SCHEMA_TOO_NEW: reason at all — every non-match fails closed to
244
+ * "still blocking" rather than guessing.
245
+ */
246
+ function reevaluateTooNewRetry(reason) {
247
+ const match = typeof reason === 'string' ? reason.match(TOO_NEW_DECLARED_VERSION) : null;
248
+ if (!match) return false;
249
+ const [, contract, declaredMajorStr] = match;
250
+ const supported = SUPPORTED_SCHEMA_VERSIONS[contract];
251
+ if (!supported) return false;
252
+ return Number(declaredMajorStr) <= supported.maxMajor;
253
+ }
254
+
255
+ function isRetryableRejection(rejectedPath) {
256
+ let parsed;
257
+ try {
258
+ parsed = JSON.parse(readFileSync(rejectedPath, 'utf-8'));
259
+ } catch {
260
+ return false;
261
+ }
262
+ const reasons = parsed?.verification?.rejection_reasons;
263
+ if (parsed?.verification?.status !== 'rejected' || !Array.isArray(reasons) || reasons.length === 0) {
264
+ return false;
265
+ }
266
+ // F-51780da9: a reason counts as retryable either via the classifier's own
267
+ // static per-prefix flag, or — for CONTRACT_SCHEMA_TOO_NEW: specifically —
268
+ // via the re-derived, current-environment check above. Every other prefix
269
+ // is unaffected: reevaluateTooNewRetry returns false for any reason it
270
+ // does not recognize, so `.every()`'s existing behavior is unchanged for
271
+ // schema:/policy:/repo:/provenance:/etc.
272
+ return reasons.every(r => parseRejectionReason(r).retryable === true || reevaluateTooNewRetry(r));
273
+ }
274
+
91
275
  /**
92
276
  * Check if a record with this run_id already exists (accepted or rejected).
93
277
  *
278
+ * F-0f9e4077 (wave-6 regression, fixed wave-8): computeRecordPath() is keyed
279
+ * on (run_id, date, accepted|rejected) — never on rejection_reasons content.
280
+ * So once a `_rejected` record occupies a path, ANY new record whose OWN
281
+ * status is STILL 'rejected' is headed to that exact same path no matter
282
+ * which violation it reports (an unexpected field with a different name, a
283
+ * different value, even a byte-identical resubmission all collide
284
+ * identically — computeRecordPath never looks at content). The
285
+ * isRetryableRejection carve-out below exists so a stale retryable-class
286
+ * rejection (F-f8952a50: any prefix parse-rejection.js's taxonomy flags
287
+ * `retryable: true` — shape/addressing mistakes, not just schema:) does not
288
+ * block a DIFFERENT-status (now accepted) record from reaching ITS OWN,
289
+ * different path — it was never a promise that the
290
+ * REJECTED path specifically is free. Pre-fix, skipping the status check let
291
+ * an uncorrected retry fall through as "not a duplicate," reach
292
+ * writeRecord()'s exclusive-create, and throw DuplicateRunIdError — a
293
+ * purely sequential, single-writer collision masquerading as the two-writer
294
+ * TOCTOU race that error class exists for. Gating on
295
+ * `record.verification.status === 'accepted'` restores the carve-out to
296
+ * exactly the case it can actually help (a record now headed elsewhere)
297
+ * while making a still-rejected collision an unconditional, ordinary
298
+ * duplicate — so writeRecord() short-circuits before ever attempting the
299
+ * write, and no throw is reachable for this class anymore.
300
+ *
94
301
  * @param {string} runId
95
302
  * @param {object} record - The record (used for repo/timing to compute path)
96
303
  * @param {string} repoRoot
@@ -105,7 +312,14 @@ export function isDuplicate(runId, record, repoRoot) {
105
312
  // Check rejected path
106
313
  const rejectedRecord = { ...record, verification: { ...record.verification, status: 'rejected' } };
107
314
  const rejectedPath = computeRecordPath(rejectedRecord, repoRoot);
108
- if (existsSync(rejectedPath)) return true;
315
+ if (existsSync(rejectedPath)) {
316
+ // A record that is itself still rejected can only ever land at THIS
317
+ // path — asking whether the OLD occupant is retryable is moot when the
318
+ // NEW attempt has nowhere else to go. Only a record now bound for the
319
+ // accepted path (real forward progress) gets to ask that question.
320
+ if (record.verification?.status !== 'accepted') return true;
321
+ return !isRetryableRejection(rejectedPath);
322
+ }
109
323
 
110
324
  return false;
111
325
  }
@@ -130,7 +344,21 @@ export function isDuplicate(runId, record, repoRoot) {
130
344
  * @throws {DuplicateRunIdError} when a concurrent writer won the race
131
345
  */
132
346
  export function writeRecord(record, repoRoot) {
133
- if (isDuplicate(record.run_id, record, repoRoot)) {
347
+ // F-a37d36f5: family sibling of the ingest.js pre-check fix — writeRecord
348
+ // carries its OWN premature isDuplicate call, reached even when the
349
+ // caller's own duplicate check was skipped or already cleared. A record
350
+ // whose repo/run_id computeRecordPath rejects (invalid format, unsafe
351
+ // segment) cannot possibly collide with an existing file, so treating the
352
+ // throw as "not a duplicate" here is the same semantically-free
353
+ // short-circuit as the run.js fix — it lets validateRecord() below be the
354
+ // authoritative judge instead of a raw path-computation throw.
355
+ let duplicate;
356
+ try {
357
+ duplicate = isDuplicate(record.run_id, record, repoRoot);
358
+ } catch {
359
+ duplicate = false;
360
+ }
361
+ if (duplicate) {
134
362
  const path = computeRecordPath(record, repoRoot);
135
363
  return { path, written: false };
136
364
  }
@@ -160,7 +388,16 @@ export function writeRecord(record, repoRoot) {
160
388
  // schema is the contract every downstream consumer relies on.
161
389
  validateRecord(record);
162
390
 
163
- const path = computeRecordPath(record, repoRoot);
391
+ // F-bbbe2e1f: see UnsafeRecordPathError's doc comment. Wrap this SECOND
392
+ // computeRecordPath() call — the first is inside isDuplicate() above,
393
+ // already guarded by F-a37d36f5 — so a schema-valid-but-unsafe repo (e.g.
394
+ // `../etc`) throws a classified error instead of a bare one.
395
+ let path;
396
+ try {
397
+ path = computeRecordPath(record, repoRoot);
398
+ } catch (err) {
399
+ throw new UnsafeRecordPathError(record, err);
400
+ }
164
401
  const dir = dirname(path);
165
402
 
166
403
  mkdirSync(dir, { recursive: true });
@@ -281,13 +281,29 @@ export function rebuildIndexes(repoRoot, options = {}) {
281
281
 
282
282
  // --- latest-by-repo.json ---
283
283
  // Keyed by repo, then product_surface. Only accepted records count.
284
- const latestByRepo = {};
284
+ //
285
+ // F-89b7dcd5: `record.repo` and `sr.product_surface` come from loadRecord()
286
+ // (JSON.parse only, no schema gate — see its JSDoc) rather than validated
287
+ // submission input, so a hand-committed record with repo: '__proto__' would
288
+ // otherwise resolve `latestByRepo['__proto__']` to Object.prototype itself
289
+ // (truthy, on a plain `{}`), skip the init branch below, and land the
290
+ // surface write ON Object.prototype — real global prototype pollution for
291
+ // the rest of this process. Object.create(null) removes the prototype
292
+ // chain entirely: `latestByRepo['__proto__']` is a plain (absent) data
293
+ // property, not the special accessor. JSON.stringify (below, at the
294
+ // commitGroupRename call) serializes a null-prototype object identically
295
+ // to a plain one, so this costs nothing on the write side. Not reachable
296
+ // from the ingest write path today (writeRecord's validateRecord() gate
297
+ // constrains repo's shape before persist.js ever writes a file), so this
298
+ // is defense-in-depth against a hand-committed or otherwise out-of-band
299
+ // record file, not a live vulnerability.
300
+ const latestByRepo = Object.create(null);
285
301
 
286
302
  for (const record of allRecords) {
287
303
  if (record.verification?.status !== 'accepted') continue;
288
304
 
289
305
  const repo = record.repo;
290
- if (!latestByRepo[repo]) latestByRepo[repo] = {};
306
+ if (!latestByRepo[repo]) latestByRepo[repo] = Object.create(null);
291
307
 
292
308
  for (const sr of record.scenario_results || []) {
293
309
  const surface = sr.product_surface;
package/run.js CHANGED
@@ -25,7 +25,8 @@ import { verify } from '@dogfood-lab/verify';
25
25
  import { stubProvenance, provenanceForProvider } from '@dogfood-lab/verify/validators/provenance.js';
26
26
  import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
27
27
  import { loadGlobalPolicy, loadRepoPolicy, loadScenarios, githubScenarioFetcher } from './load-context.js';
28
- import { isDuplicate, writeRecord, computeRecordPath } from './persist.js';
28
+ import { isDuplicate, writeRecord, computeRecordPath, UnsafeRecordPathError } from './persist.js';
29
+ import { RecordValidationError } from './validate-record.js';
29
30
  import { rebuildIndexes } from './rebuild-indexes.js';
30
31
  import { verifyChain, formatChainResult } from './verify-chain.js';
31
32
  import { handleAnchorCompute, handleAnchorPost, handleAnchorVerify } from './anchor/cli.js';
@@ -350,7 +351,40 @@ export async function ingest(submission, options) {
350
351
  timing: submission.timing,
351
352
  verification: { status: 'accepted' }
352
353
  };
353
- if (isDuplicate(submission.run_id, probeRecord, repoRoot)) {
354
+ // F-f8952a50 (wave 10): this probe's hardcoded `status: 'accepted'` is
355
+ // ALSO the site the finding proved a corrected repo:mismatch resubmission
356
+ // was swallowed at — isDuplicate() only asks isRetryableRejection() when
357
+ // the record it's handed claims 'accepted' (see isDuplicate's own
358
+ // comment), so this probe is the FIRST place a stale non-schema rejection
359
+ // could mask a genuine correction, before verify() ever runs. No separate
360
+ // fix belongs HERE, though: this probe and writeRecord()'s own internal
361
+ // isDuplicate() call (persist.js) share the exact same isDuplicate() /
362
+ // isRetryableRejection() code path, so persist.js's per-prefix
363
+ // `retryable` fix (parse-rejection.js) already reaches both call sites —
364
+ // widening retryability there is what unblocks this probe too, with
365
+ // nothing probe-specific to change.
366
+ // F-a37d36f5: isDuplicate -> computeRecordPath runs against RAW untrusted
367
+ // submission.repo/run_id, three steps BEFORE verify()'s schema gate. A
368
+ // malformed repo ('a/b/c', a path-traversal attempt, etc.) makes
369
+ // computeRecordPath THROW ('invalid repo format' / 'unsafe repo segment'
370
+ // / 'unsafe run_id') — good, the traversal guards hold — but letting that
371
+ // throw escape here inverts this repo's submission-bad vs operational
372
+ // doctrine: it propagates as an uncaught fault (exit 2, "operator error"
373
+ // per the CLI's own USAGE block) for input that is squarely the
374
+ // SUBMITTER's to fix, and no `_rejected` evidence record is ever
375
+ // written. A malformed repo/run_id can never collide with an existing
376
+ // record anyway, so treating an unpathable submission as "not a
377
+ // duplicate" is semantically free — verify()'s schema gate is the
378
+ // authoritative judge of bad input, and writeRecord's own
379
+ // computeRecordPath (which runs AFTER validateRecord, on the SCHEMA-
380
+ // VALIDATED persisted record) remains the real enforcement point.
381
+ let duplicate;
382
+ try {
383
+ duplicate = isDuplicate(submission.run_id, probeRecord, repoRoot);
384
+ } catch {
385
+ duplicate = false;
386
+ }
387
+ if (duplicate) {
354
388
  logStage('rejected_pre_persist', {
355
389
  submission_id: submissionId,
356
390
  correlation_id,
@@ -463,7 +497,72 @@ export async function ingest(submission, options) {
463
497
  return { record, path: null, written: false, duplicate: false };
464
498
  }
465
499
  const persistStart = Date.now();
466
- const { path, written } = writeRecord(record, repoRoot);
500
+ let path, written;
501
+ try {
502
+ ({ path, written } = writeRecord(record, repoRoot));
503
+ } catch (err) {
504
+ // F-4acd28d8: computeRecordPath()'s traversal guard (isUnsafeSegment) is
505
+ // STRICTER than the submission schema's repo pattern (F-bbbe2e1f — e.g.
506
+ // `../etc` is schema-valid but traversal-unsafe), so a record can reach
507
+ // here without `_skipPersist` ever having been set. The record's own
508
+ // identifier is what's unfilable — submission-bad, not an operator
509
+ // incident — so route it like `_skipPersist` above instead of letting the
510
+ // throw reach the outer CLI catch, which would misreport it as
511
+ // failed_stage:'cli_parse_payload' and exit 2 ("operator error") for
512
+ // content that is squarely the submitter's to fix.
513
+ //
514
+ // RecordValidationError is the SIBLING gap F-4036ae25's _skipPersist
515
+ // narrowing opens: dogfood-record.schema.json mirrors the submission
516
+ // schema's constraints on every source-authored field verify() copies
517
+ // verbatim (ref, source, timing, scenario_results, overall_verdict.proposed),
518
+ // so a submission that is schema-invalid on one of THOSE fields (not just
519
+ // repo/run_id/timing.finished_at) reaches writeRecord() filable-by-identity
520
+ // and still fails validateRecord() here. Only catch it when the submission
521
+ // was ALREADY schema-invalid (record.verification.schema_valid === false)
522
+ // — that is the authoritative signal this is submission-bad fallout, not a
523
+ // genuine internal defect in what verify() assembled for an
524
+ // otherwise-valid submission, which must keep crashing loudly.
525
+ const isUnfilableRecordPath = err instanceof UnsafeRecordPathError;
526
+ const isSubmissionBadRecordShape =
527
+ err instanceof RecordValidationError && record.verification.schema_valid === false;
528
+ if (!isUnfilableRecordPath && !isSubmissionBadRecordShape) {
529
+ throw err;
530
+ }
531
+
532
+ // verify() already rejected this submission for a real reason whenever
533
+ // one exists (e.g. repo:mismatch, or the schema violation itself) — those
534
+ // reasons survive untouched. When verify() had no reason to reject (an
535
+ // otherwise-accepted record whose ONLY problem is storage-unsafety),
536
+ // downgrade the verdict here — the same shape used below for a late
537
+ // scenario-load rejection.
538
+ if (isUnfilableRecordPath) {
539
+ record.verification.rejection_reasons.push(`unsafe-record-path: ${err.message}`);
540
+ if (record.verification.status === 'accepted') {
541
+ record.verification.status = 'rejected';
542
+ record.verification.policy_valid = false;
543
+ if (record.overall_verdict.verified === 'pass') {
544
+ record.overall_verdict.verified = 'fail';
545
+ record.overall_verdict.downgraded = true;
546
+ if (!record.overall_verdict.downgrade_reasons) {
547
+ record.overall_verdict.downgrade_reasons = [];
548
+ }
549
+ record.overall_verdict.downgrade_reasons.push('record path could not be safely computed');
550
+ }
551
+ }
552
+ }
553
+
554
+ logStage('rejected_pre_persist', {
555
+ submission_id: submissionId,
556
+ correlation_id,
557
+ reason: isUnfilableRecordPath ? 'unsafe_record_path' : 'record_schema_invalid_from_submission',
558
+ rejection_reasons: record.verification.rejection_reasons ?? [],
559
+ // Operator-debug only: validateRecord()'s OWN structured errors, distinct
560
+ // from (and not merged into) the submitter-facing rejection_reasons above,
561
+ // which stay exactly as verify() computed them against the submission schema.
562
+ ...(isSubmissionBadRecordShape ? { record_validation_errors: err.errors } : {})
563
+ });
564
+ return { record, path: null, written: false, duplicate: false };
565
+ }
467
566
  logStage('persist_complete', {
468
567
  submission_id: submissionId,
469
568
  correlation_id,
@@ -525,7 +624,20 @@ export async function ingest(submission, options) {
525
624
  }
526
625
  }
527
626
 
528
- return { record, path, written, duplicate: false };
627
+ // F-7b97fbd4 (wave 10): mirror persist_complete's OWN `duplicate: !written`
628
+ // (logged a few lines above, off the same `written`) instead of a bare
629
+ // `false` that silently disagreed with it. By this point `written` can be
630
+ // `false` for exactly one reason — writeRecord()'s internal isDuplicate()
631
+ // blocked the write as a collision (persist.js) — every OTHER
632
+ // non-persisting outcome in this function (record._skipPersist above; the
633
+ // UnsafeRecordPathError / RecordValidationError catch above that) already
634
+ // returns its own honest `duplicate: false` earlier and never reaches this
635
+ // line. The CLI wrapper's `if (result.duplicate)` branch depends on this
636
+ // being accurate: a blocked resubmission must take the terse exit-0
637
+ // `{status:'duplicate'}` path, the same one an early-detected duplicate
638
+ // (the pre-verify probe above) already takes, not fall through to the
639
+ // full rejected-record exit-1 shape.
640
+ return { record, path, written, duplicate: !written };
529
641
  }
530
642
 
531
643
  /**
@@ -664,10 +776,12 @@ export async function verifyOnly(submission, options) {
664
776
  would_persist_to = computeRecordPath(record, repoRoot);
665
777
  } catch {
666
778
  // Defensive: if a record passes verify() but still trips path
667
- // computation (e.g., a future schema with looser constraints), keep
668
- // verify-only side-effect-free. Real ingest would surface the throw
669
- // via writeRecord; verify-only just returns null and lets the operator
670
- // see the rejection in record.verification.rejection_reasons.
779
+ // computation (e.g. F-bbbe2e1f's `../etc` schema-valid but
780
+ // traversal-unsafe), keep verify-only side-effect-free. Real ingest
781
+ // hits the SAME underlying computeRecordPath failure inside
782
+ // writeRecord() but now catches it too (F-4acd28d8, UnsafeRecordPathError)
783
+ // rather than letting it escape; verify-only just returns null here and
784
+ // lets the operator see the rejection in record.verification.rejection_reasons.
671
785
  would_persist_to = null;
672
786
  }
673
787
  }
@@ -719,27 +833,30 @@ function emitCliErrorEvent({ failedStage, correlationId, submissionId = null, er
719
833
  // to live IN the tool, not scattered across docs.
720
834
  const USAGE = `ingest — persist a dogfood submission (verify → policy → provenance → write)
721
835
 
722
- USAGE:
836
+ Usage:
723
837
  node packages/ingest/run.js --file <path> --provenance=github|stub
724
838
  node packages/ingest/run.js --payload '<json>' --provenance=github|stub
725
839
  echo '<json>' | node packages/ingest/run.js --provenance=github|stub
726
840
 
727
- INPUT (exactly one; stdin used when neither flag is given):
841
+ Input (exactly one; stdin used when neither flag is given):
728
842
  --file <path> Read the submission JSON from a file.
729
843
  --payload <json> Pass the submission JSON inline.
730
844
  (stdin) Pipe the submission JSON on stdin.
731
845
 
732
- PROVENANCE (required for an ingest):
846
+ Provenance (required for an ingest):
733
847
  --provenance=github Confirm the source run via the GitHub API.
734
848
  --provenance=stub No-network local confirm (dry-run / dev only).
735
849
 
736
- MODES:
850
+ Modes:
737
851
  --verify-only Run the full pipeline WITHOUT writing or rebuilding
738
852
  indexes; report where a real ingest WOULD have landed.
739
853
 
740
- STANDALONE AUDIT VERBS (no submission, no stdin, no --provenance):
854
+ Standalone audit verbs (no submission, no stdin, no --provenance):
741
855
  --verify-chain Verify the append-only integrity ledger (offline).
742
- --reconcile Also fail on on-disk records missing from the ledger.
856
+ --reconcile Also fail on genuine torn-write orphans (on-disk records
857
+ missing from the ledger). Records that predate the
858
+ integrity chain itself are reported separately and do
859
+ not fail the audit.
743
860
  --all Report every independent break instead of the first.
744
861
  --anchor-compute Compute + write the next XRPL anchor manifest (offline).
745
862
  --anchor-post Compute if needed + post the anchor to XRPL (needs XRPL_SEED).
@@ -747,7 +864,7 @@ STANDALONE AUDIT VERBS (no submission, no stdin, no --provenance):
747
864
 
748
865
  -h, --help Show this help.
749
866
 
750
- EXIT CODES:
867
+ Exit codes:
751
868
  0 success 1 integrity/audit break 2 operator error (flags / IO / JSON)`;
752
869
 
753
870
  const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(__dirname, 'run.js');
@@ -820,9 +937,38 @@ if (isMain) {
820
937
  const hasValue = inlineValue !== null || nextIsValue;
821
938
  const takeValue = () => (inlineValue !== null ? inlineValue : args[++i]);
822
939
 
823
- if (arg === '--provenance' && hasValue) {
940
+ // F-e0bcbc47 (amended wave 29, after the first attempt regressed
941
+ // F-INGEST-003): `flagIs(name)` both TESTS the token and RECORDS that
942
+ // `name` is a flag this chain knows about, so the catch-all below can tell
943
+ // "unknown flag" (no branch knows this name) apart from "known flag, no
944
+ // value" (a branch knows the name but its `&& hasValue` guard failed —
945
+ // exactly what F-INGEST-003 causes for `--anchor-network --anchor-compute`).
946
+ // The first attempt at this fix had a bare `else if (arg.startsWith('--'))`
947
+ // catch-all that could not distinguish the two and so reported a KNOWN flag
948
+ // as "unknown argument: --anchor-network" — a claim wider than what it
949
+ // actually checked, which is the exact class this repo keeps paying for.
950
+ //
951
+ // `knownFlagName` is DERIVED from the chain itself rather than duplicated
952
+ // into a second list of flag names: the only way a name enters the known
953
+ // set is for the chain to literally test for it one line below. A parallel
954
+ // `KNOWN_FLAGS` enumeration would be a second population to keep in sync
955
+ // (the same enumeration-vs-property class as CONTROL_CLASS / ZALGO_RUN /
956
+ // DASH_CONFUSABLES), and it would drift the first time someone adds a flag
957
+ // and forgets the list. Adding a branch here cannot desync it — the branch
958
+ // IS the registration. The one residual shape is a future branch written as
959
+ // a bare `arg === '--new'` instead of `flagIs('--new')`; that would make the
960
+ // new flag report as unknown when dangling, and is guarded by
961
+ // f-e0bcbc47-unknown-flag-rejection.test.js's chain-shape assertion.
962
+ let knownFlagName = false;
963
+ const flagIs = (name) => {
964
+ if (arg !== name) return false;
965
+ knownFlagName = true;
966
+ return true;
967
+ };
968
+
969
+ if (flagIs('--provenance') && hasValue) {
824
970
  provenanceMode = takeValue();
825
- } else if (arg === '--file' && hasValue) {
971
+ } else if (flagIs('--file') && hasValue) {
826
972
  const { readFileSync } = await import('node:fs');
827
973
  // D1B-001 family (operator-legibility): a --file read failure
828
974
  // (ENOENT/EACCES) routes through the structured error event and exits 2
@@ -842,53 +988,103 @@ if (isMain) {
842
988
  });
843
989
  process.exit(2);
844
990
  }
845
- } else if (arg === '--payload' && hasValue) {
991
+ } else if (flagIs('--payload') && hasValue) {
846
992
  submissionJson = takeValue();
847
- } else if (arg === '--verify-only') {
993
+ } else if (flagIs('--verify-only')) {
848
994
  // F-252714-058: dry-run the pipeline without writing or rebuilding
849
995
  // indexes. CI / operators preview what WOULD have been persisted.
850
996
  verifyOnlyFlag = true;
851
- } else if (arg === '--verify-chain') {
997
+ } else if (flagIs('--verify-chain')) {
852
998
  // Integrity chain v1: verify the append-only tamper-evident ledger at
853
999
  // indexes/integrity/chain.jsonl, fully offline. No submission, no stdin,
854
1000
  // no provenance — a standalone audit command.
855
1001
  verifyChainFlag = true;
856
- } else if (arg === '--reconcile') {
1002
+ } else if (flagIs('--reconcile')) {
857
1003
  // Modifier for --verify-chain: also reconcile on-disk records against the
858
1004
  // ledger and fail on any orphan (INGEST-PROACT-001).
859
1005
  reconcileFlag = true;
860
- } else if (arg === '--all') {
1006
+ } else if (flagIs('--all')) {
861
1007
  // Modifier for --verify-chain: report every per-record-independent break
862
1008
  // instead of stopping at the first (INGEST-PROACT-004).
863
1009
  allBreaksFlag = true;
864
- } else if (arg === '--anchor-compute') {
1010
+ } else if (flagIs('--anchor-compute')) {
865
1011
  // Optional XRPL anchor: compute + write the next anchor manifest. Offline.
866
1012
  anchorComputeFlag = true;
867
- } else if (arg === '--anchor-post') {
1013
+ } else if (flagIs('--anchor-post')) {
868
1014
  // Optional XRPL anchor: compute if needed + post to XRPL. Needs the
869
1015
  // optional xrpl package (lazily loaded) and XRPL_SEED.
870
1016
  anchorPostFlag = true;
871
- } else if (arg === '--anchor-verify') {
1017
+ } else if (flagIs('--anchor-verify')) {
872
1018
  // Optional XRPL anchor: verify local manifests + run the truncation check.
873
1019
  // Offline reports honest NOT-verified for the on-chain leg.
874
1020
  anchorVerifyFlag = true;
875
- } else if (arg === '--anchor-all') {
1021
+ } else if (flagIs('--anchor-all')) {
876
1022
  // Genesis snapshot mode for compute/post (covers the whole chain).
877
1023
  anchorMode = 'all';
878
- } else if (arg === '--anchor-algo' && hasValue) {
1024
+ } else if (flagIs('--anchor-algo') && hasValue) {
879
1025
  anchorAlgo = takeValue();
880
- } else if (arg === '--anchor-network' && hasValue) {
1026
+ } else if (flagIs('--anchor-network') && hasValue) {
881
1027
  anchorNetwork = takeValue();
882
- } else if (arg === '--anchor-tx' && hasValue) {
1028
+ } else if (flagIs('--anchor-tx') && hasValue) {
883
1029
  // Path to a JSON file containing a fetched XRPL tx (with Memos) for the
884
1030
  // on-chain leg of --anchor-verify. Offline-honest: omit it to run the
885
1031
  // truncation check only.
886
1032
  anchorTxFile = takeValue();
887
- } else if (arg === '--anchor-trusted' && hasValue) {
1033
+ } else if (flagIs('--anchor-trusted') && hasValue) {
888
1034
  // Comma-separated trusted anchor accounts (UNIONed with the bundled list).
889
1035
  anchorTrustedAccounts = takeValue().split(',').map((s) => s.trim()).filter(Boolean);
890
- } else if (arg === '-h' || arg === '--help' || arg === '--usage') {
1036
+ } else if (flagIs('-h') || flagIs('--help') || flagIs('--usage')) {
891
1037
  helpFlag = true;
1038
+ } else if (arg.startsWith('--') && knownFlagName) {
1039
+ // F-e0bcbc47 / F-INGEST-003 boundary: a KNOWN flag whose `&& hasValue`
1040
+ // guard failed — i.e. a value flag given no value, because the next
1041
+ // token was itself a flag (`--anchor-network --anchor-compute`) or it
1042
+ // ended argv. F-INGEST-003 pins that the FOLLOWING flag is still parsed
1043
+ // on its own rather than swallowed as this one's value, so this token
1044
+ // keeps its historical path (fall through to positionalArgs, continue)
1045
+ // and `--anchor-compute` runs. Rejecting here instead would break that
1046
+ // pinned contract.
1047
+ //
1048
+ // RESIDUAL, STATED (not fixed here): the dangling flag is still
1049
+ // IGNORED — `--anchor-network` with no value does not set the network
1050
+ // and does not fail the run. That is pre-existing behavior and this
1051
+ // amend does not change it. What changes is that it is no longer
1052
+ // SILENT: an operator gets a structured warn naming the flag instead of
1053
+ // the value vanishing with no trace. Making it a hard error is a real
1054
+ // contract change (it would stop `--anchor-compute` from running and
1055
+ // rewrite F-INGEST-003's pin), which is a bigger claim than a Stage C
1056
+ // humanization amend should make unilaterally — filed as a follow-up
1057
+ // rather than landed here.
1058
+ logStage('warn', {
1059
+ kind: 'cli_flag_missing_value',
1060
+ flag: arg,
1061
+ correlation_id: synthCorrelationId(),
1062
+ message: `${arg} was given no value and is being ignored (the next token is a flag, not a value)`
1063
+ });
1064
+ positionalArgs.push(args[i]);
1065
+ } else if (arg.startsWith('--')) {
1066
+ // F-e0bcbc47 (Stage C humanization): a genuinely unrecognized `--flag`
1067
+ // — no branch above knows this name at all. It used to fall through to
1068
+ // the dead `positionalArgs` sink (declared, pushed to, and never read
1069
+ // anywhere else in this file — confirmed by grep) and the CLI proceeded
1070
+ // to read stdin, hit EOF, and crashed with a raw 'Unexpected end of
1071
+ // JSON input' stack that misattributed the failure to the submission
1072
+ // payload rather than the misspelled flag (live-proven trigger:
1073
+ // `--provenance=stub --fiel <path>`, a one-character typo of --file).
1074
+ // Reject at the point of the typo instead — matches the sibling pattern
1075
+ // already correct in this same domain (packages/verify/cli.js:
1076
+ // `unknown argument: --version` -> exit 2, no stack, no fallthrough).
1077
+ // Scoped to `--`-prefixed tokens only (a bare positional stays in the
1078
+ // historical positionalArgs sink) because that is exactly the shape a
1079
+ // mistyped flag takes, and it is the shape the reachable repro above
1080
+ // hits BEFORE any value token is even read.
1081
+ emitCliErrorEvent({
1082
+ failedStage: 'cli_parse_args',
1083
+ correlationId: synthCorrelationId(),
1084
+ err: new Error(`unknown argument: ${arg}`),
1085
+ humanPrefix: 'invalid CLI invocation'
1086
+ });
1087
+ process.exit(2);
892
1088
  } else {
893
1089
  positionalArgs.push(args[i]);
894
1090
  }
@@ -923,7 +1119,12 @@ if (isMain) {
923
1119
  // operator asked for it, so a grep of the NDJSON shows how many breaks and
924
1120
  // orphans were found, not just the first break.
925
1121
  ...(Array.isArray(result.breaks) ? { break_count: result.breaks.length } : {}),
926
- ...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {})
1122
+ ...(Array.isArray(result.orphans) ? { orphan_count: result.orphans.length } : {}),
1123
+ // F-29134790: pre-adoption records are excluded from chain_ok but still
1124
+ // worth a grep-able count — an operator diffing orphan_count over time
1125
+ // should see the pre-chain figure hold steady while orphan_count reflects
1126
+ // only genuine torn writes.
1127
+ ...(Array.isArray(result.pre_adoption) ? { pre_adoption_count: result.pre_adoption.length } : {})
927
1128
  });
928
1129
  const lines = formatChainResult(result);
929
1130
  if (result.ok) {
package/verify-chain.js CHANGED
@@ -27,6 +27,17 @@
27
27
  * push that touched a record but not the chain), and it catches middle-deletion,
28
28
  * reorder, and forged-insertion (they break the seq run or the prev-link).
29
29
  *
30
+ * The `pre_adoption` exemption inside collectOrphanRecords (below) narrows the
31
+ * "push that touched a record but not the chain" promise one step further: it
32
+ * excuses an un-ledgered, no-integrity record from `ok` only when the record's
33
+ * OWN claimed timing.finished_at/verification.verified_at predates
34
+ * CHAIN_ADOPTION_DATE (F-1907b2bc, wave 14), so dropping a record with a
35
+ * fabricated RECENT date is still caught as a genuine orphan. Dropping one with
36
+ * a fabricated OLD date is not caught — that record is indistinguishable from
37
+ * genuine history, because both are self-reported fields the same privileged
38
+ * writer controls. That residual is the SAME already-disclosed ingest-write-
39
+ * credential trust boundary named two paragraphs up, not a new one.
40
+ *
30
41
  * KNOWN LIMITATION — tail truncation. Removing the most-recent entries (and
31
42
  * their record files) leaves a SHORTER but internally-consistent chain, which
32
43
  * verifies OK: the offline verifier has no external record of the expected head
@@ -72,7 +83,16 @@ import { findJsonFiles } from './rebuild-indexes.js';
72
83
  * @property {ChainBreak[]} [breaks] - All independent breaks (only present when
73
84
  * `collectAllBreaks` is set). The first element equals `break`.
74
85
  * @property {ChainOrphan[]} [orphans] - Records on disk but absent from the
75
- * ledger (only present when `collectOrphans` is set).
86
+ * ledger AND written by chain-aware code (only present when `collectOrphans`
87
+ * is set). These are genuine torn writes — persist-succeeded, ledger-append-
88
+ * missed — and they make `ok` false.
89
+ * @property {ChainOrphan[]} [pre_adoption] - Records on disk but absent from
90
+ * the ledger that predate the chain's adoption: no `integrity` block AND a
91
+ * self-reported timing.finished_at/verification.verified_at before
92
+ * CHAIN_ADOPTION_DATE (F-1907b2bc — a no-integrity record whose own claimed
93
+ * timestamp is on/after that date is a genuine `orphans` entry instead;
94
+ * only present when `collectOrphans` is set). Visible for operator
95
+ * awareness; does NOT make `ok` false.
76
96
  */
77
97
 
78
98
  /**
@@ -93,6 +113,10 @@ import { findJsonFiles } from './rebuild-indexes.js';
93
113
  * ledger (a torn write between record-write and ledger-append). An orphan
94
114
  * makes the result `ok: false` — the audit DETECTS the orphan instead of
95
115
  * silently passing while a real record lives outside the tamper-evident chain.
116
+ * F-29134790 (wave 12): also returns `pre_adoption[]` — records absent from
117
+ * the ledger that predate the chain's existence entirely (see
118
+ * collectOrphanRecords's adoption-boundary doc below). Pre-adoption records
119
+ * are visible for operator awareness but do NOT affect `ok`.
96
120
  * @returns {ChainVerifyResult}
97
121
  */
98
122
  export function verifyChain(repoRoot, opts = {}) {
@@ -112,7 +136,10 @@ export function verifyChain(repoRoot, opts = {}) {
112
136
  head_digest: GENESIS_DIGEST,
113
137
  break: brk,
114
138
  ...(collectAllBreaks ? { breaks: [brk] } : {}),
115
- ...(collectOrphans ? { orphans: [] } : {}),
139
+ // pre_adoption included alongside orphans (both empty here) so the
140
+ // result shape is consistent whenever collectOrphans is set, regardless
141
+ // of which early-return path produced it.
142
+ ...(collectOrphans ? { orphans: [], pre_adoption: [] } : {}),
116
143
  };
117
144
  }
118
145
 
@@ -191,8 +218,12 @@ export function verifyChain(repoRoot, opts = {}) {
191
218
  prevDigest = entry.submission_digest;
192
219
  }
193
220
 
194
- const orphans = collectOrphans ? collectOrphanRecords(repoRoot, entries) : null;
221
+ const reconciled = collectOrphans ? collectOrphanRecords(repoRoot, entries) : null;
222
+ const orphans = reconciled ? reconciled.orphans : null;
223
+ const preAdoption = reconciled ? reconciled.preAdoption : null;
195
224
 
225
+ // Only genuine orphans (post-adoption, ledger-eligible records) affect ok.
226
+ // pre_adoption is informational — see collectOrphanRecords's doc comment.
196
227
  const ok = breaks.length === 0 && (orphans === null || orphans.length === 0);
197
228
  return {
198
229
  ok,
@@ -200,36 +231,168 @@ export function verifyChain(repoRoot, opts = {}) {
200
231
  head_digest: headDigest,
201
232
  break: breaks.length > 0 ? breaks[0] : null,
202
233
  ...(collectAllBreaks ? { breaks } : {}),
203
- ...(collectOrphans ? { orphans } : {}),
234
+ ...(collectOrphans ? { orphans, pre_adoption: preAdoption } : {}),
204
235
  };
205
236
  }
206
237
 
238
+ /**
239
+ * The commit that gave persist.js's writeRecord its UNCONDITIONAL
240
+ * integrity-stamping logic — f4ca987 "feat(integrity): tamper-evident hash
241
+ * chain over records/", 2026-06-21T18:35:36-04:00 (2026-06-21T22:35:36Z).
242
+ * Every record genuinely written by writeRecord() from this date onward
243
+ * carries a fully-formed `integrity` block; a record with none at all can
244
+ * only predate it — see collectOrphanRecords' F-1907b2bc doc paragraph below.
245
+ *
246
+ * Set to the START of the commit's UTC calendar day, not its exact
247
+ * timestamp, so the boundary errs toward NOT excusing a borderline record
248
+ * (the fail-loud direction) rather than toward hair-splitting a commit
249
+ * second no real record is anywhere near — the actual gap in this repo's own
250
+ * history is five weeks wide (last genuine pre-chain record 2026-05-26,
251
+ * first genuine ledgered record 2026-07-03; see F-29134790's doc paragraph
252
+ * below), so this constant has a wide margin on both sides.
253
+ */
254
+ export const CHAIN_ADOPTION_DATE = new Date('2026-06-21T00:00:00Z');
255
+
256
+ /**
257
+ * The later of a record's own self-reported `timing.finished_at` /
258
+ * `verification.verified_at`, or `null` when neither is present and
259
+ * parseable. F-1907b2bc: used only to sanity-check a would-be `preAdoption`
260
+ * bucketing — this is NOT a trust boundary (the same privileged writer that
261
+ * can drop a record with no `integrity` block can also forge these fields;
262
+ * see the top-of-file threat-model paragraph). It only stops the tool from
263
+ * asserting "predates the chain" when the record's own data already
264
+ * contradicts that claim, or supports no claim at all.
265
+ *
266
+ * F-a85b7e84 (wave 16, LOW): a candidate must be a `string` before it is
267
+ * handed to `new Date()`. JSON has no Date type — every field a real
268
+ * writeRecord() call ever produces here is a string (Date.prototype.toJSON
269
+ * always serializes to one) or absent — so `typeof value === 'string'` costs
270
+ * nothing against genuine records. Without it, `new Date()`'s permissive
271
+ * ToPrimitive coercion accepts non-date JSON values that a bare falsy check
272
+ * (`if (!value) continue`) does not filter out: `new Date(true)` coerces the
273
+ * boolean to the number 1 (1970-01-01T00:00:00.001Z), a bare integer is read
274
+ * as an epoch-millisecond offset, and an array's comma-joined
275
+ * `toString()` (e.g. `[2026, 3, 1]` -> `'2026,3,1'`) fallback-parses as a
276
+ * real date — all landing well before CHAIN_ADOPTION_DATE with none of the
277
+ * "type a plausible fake ISO string" effort the surviving forgery residual
278
+ * (this function's own doc paragraph above) assumes. A non-string value now
279
+ * falls through to the same "no readable timestamp" orphan branch as an
280
+ * absent field, exactly like an unparseable string already did.
281
+ *
282
+ * @param {object} record
283
+ * @returns {Date|null}
284
+ */
285
+ function latestClaimedTimestamp(record) {
286
+ const candidates = [record.timing?.finished_at, record.verification?.verified_at];
287
+ let latest = null;
288
+ for (const value of candidates) {
289
+ if (typeof value !== 'string') continue;
290
+ const parsed = new Date(value);
291
+ if (isNaN(parsed.getTime())) continue;
292
+ if (latest === null || parsed.getTime() > latest.getTime()) latest = parsed;
293
+ }
294
+ return latest;
295
+ }
296
+
207
297
  /**
208
298
  * INGEST-PROACT-001 — reconcile on-disk records against the ledger.
209
299
  *
210
300
  * Walks every `*.json` under `records/` (accepted AND `_rejected/`) and flags
211
- * any record file whose `(run_id, seq)` identity is absent from the ledger.
212
- * Such a file is an ORPHAN: persist wrote the record but the ledger append did
213
- * not happen (a crash in the torn window between the two steps), so the record
301
+ * any record file whose repo-relative PATH is absent from the ledger. Such a
302
+ * file is an ORPHAN: persist wrote the record but the ledger append did not
303
+ * happen (a crash in the torn window between the two steps), so the record
214
304
  * exists OUTSIDE the tamper-evident chain and the plain chain-walk is blind to
215
305
  * it. The chain-manifest's own ledger lines are NOT under `records/`, so they
216
306
  * are never mistaken for orphans.
217
307
  *
308
+ * F-d4bcf5d0 (wave 10, HIGH): this used to ALSO fall back to a run_id-only
309
+ * match — "a record is accounted for if EITHER its path OR its run_id is
310
+ * ledgered anywhere". That fallback was unsound: a single run_id can
311
+ * legitimately own TWO on-disk records at TWO different paths (the
312
+ * accepted-after-a-prior-rejection resubmission flow persist.js's
313
+ * isRetryableRejection carve-out exists for — see
314
+ * F-0f9e4077/F-4036ae25/F-f8952a50), each needing its OWN ledger line. Once
315
+ * ANY entry for a run_id was ledgered, the fallback treated EVERY later
316
+ * record sharing that run_id as accounted for — including one at a
317
+ * genuinely un-ledgered path, exactly the torn-write shape this
318
+ * reconciliation pass exists to catch (proven live: a ledgered `_rejected`
319
+ * record followed by an accepted record for the SAME run_id whose OWN
320
+ * ledger line was then lost — the plain run_id match hid the orphan).
321
+ * Path identity is both the necessary AND the sufficient check: every ledger
322
+ * line carries an exact `path` field (lib/chain-manifest.js's documented
323
+ * line shape, always populated by persist.js's appendChainEntry call), and
324
+ * this function's own `relPath` computation (`relative(repoRoot,
325
+ * absPath).split(sep).join('/')`) is the identical normalization persist.js
326
+ * uses to produce that field. Dropping the run_id fallback removes the
327
+ * unsound generalization without weakening real coverage — see
328
+ * ingest-proact-001-004-verify-chain-reconcile-allbreaks.test.js's
329
+ * 'shared run_id, different paths' regression test.
330
+ *
331
+ * F-29134790 (wave 12, HIGH): path-identity membership is correct for records
332
+ * the chain COULD have ledgered, but on its own it has no ADOPTION BOUNDARY —
333
+ * every un-ledgered path reads identically, so a record that predates the
334
+ * chain feature's existence is indistinguishable from a genuine crash-window
335
+ * torn write. Proven live against this repo's own `records/` tree: 53 files
336
+ * with no ledger line, every one dated 2026-03-19 through 2026-05-26 — all
337
+ * BEFORE persist.js gained its integrity-stamping logic (f4ca987,
338
+ * 2026-06-21) — versus the 7 real ledgered records, all dated 2026-07-03
339
+ * onward. Pre-fix, `verifyChain({ collectOrphans: true })` reported all 53 as
340
+ * indistinguishable "torn persist" orphans (a 53:0 signal-to-noise ratio; a
341
+ * genuine orphan today would be the 54th unlabeled line).
342
+ *
343
+ * The fix needs no new marker file or backfill tool: persist.js's writeRecord
344
+ * already stamps `record.integrity = { submission_digest, prev_digest, seq }`
345
+ * UNCONDITIONALLY, in memory, BEFORE it writes the record file and BEFORE it
346
+ * appends the ledger line (see persist.js — the stamp happens ahead of
347
+ * validateRecord() and the atomic file write). That ordering guarantees a
348
+ * genuine post-adoption torn write (file landed, ledger append lost) still
349
+ * carries a fully-formed `integrity` block on disk; only a record written
350
+ * before persist.js gained this stamping logic can lack one entirely. So
351
+ * "does this record carry an `integrity` block" is not a proxy for the
352
+ * adoption boundary — it IS the adoption boundary, already recorded per-record
353
+ * by the exact code change that introduced the chain, for free. A record
354
+ * missing `integrity` is bucketed into the returned `preAdoption` array
355
+ * (visible, does not fail `ok`); a record WITH an `integrity` block that is
356
+ * still un-ledgered remains a genuine `orphans` entry exactly as before.
357
+ *
358
+ * F-1907b2bc (wave 14, MEDIUM): the paragraph above treats "does this record
359
+ * carry an `integrity` block" as the WHOLE adoption boundary, but that
360
+ * equivalence only holds for records that actually passed through the real
361
+ * writeRecord(). A record dropped directly onto disk — bypassing writeRecord
362
+ * entirely — also has no `integrity` block, and pre-fix was silently
363
+ * absorbed into `preAdoption` with a "predates the chain" reason string that
364
+ * is provably FALSE whenever the dropped record's own claimed timestamp is on
365
+ * or after CHAIN_ADOPTION_DATE (persist.js has stamped `integrity`
366
+ * UNCONDITIONALLY on every record it writes since that date — see the
367
+ * constant's doc comment above — so a record genuinely written from then on
368
+ * could not lack one). The fix cross-checks a would-be `preAdoption` record's
369
+ * own `timing.finished_at`/`verification.verified_at` (the later of the two
370
+ * — see `latestClaimedTimestamp`) against CHAIN_ADOPTION_DATE before
371
+ * accepting the "predates the chain" narrative; a record with no verifiable
372
+ * timestamp of its own, or one on/after the cutoff, is bucketed into
373
+ * `orphans` instead, with a reason string honest about why. This closes the
374
+ * LOW-effort version of dropping a forged record (not bothering to fake a
375
+ * plausible pre-adoption date) but not a DILIGENT one that also forges an old
376
+ * timing/verification block — no purely offline, record-content-based check
377
+ * can tell a forged old date from a genuine one; see the threat-model
378
+ * paragraph at the top of this file.
379
+ *
218
380
  * @param {string} repoRoot
219
381
  * @param {Array<object>} entries - The ledger entries (already read).
220
- * @returns {ChainOrphan[]}
382
+ * @returns {{ orphans: ChainOrphan[], preAdoption: ChainOrphan[] }} `orphans`
383
+ * are un-ledgered records stamped by chain-aware code — genuine torn
384
+ * writes. `preAdoption` are un-ledgered records with no `integrity` block —
385
+ * they predate the chain and were never ledgerable. Only `orphans` affects
386
+ * verifyChain's `ok`.
221
387
  */
222
388
  function collectOrphanRecords(repoRoot, entries) {
223
- // Index the ledger by run_id and by repo-relative path so a record matches if
224
- // EITHER identity is present — a record is ledgered as long as its line exists.
225
- const ledgerRunIds = new Set();
226
389
  const ledgerPaths = new Set();
227
390
  for (const e of entries) {
228
- if (e.run_id) ledgerRunIds.add(e.run_id);
229
391
  if (e.path) ledgerPaths.add(e.path);
230
392
  }
231
393
 
232
394
  const orphans = [];
395
+ const preAdoption = [];
233
396
  const recordsDir = join(repoRoot, 'records');
234
397
  for (const absPath of findJsonFiles(recordsDir)) {
235
398
  const relPath = relative(repoRoot, absPath).split(sep).join('/');
@@ -240,7 +403,9 @@ function collectOrphanRecords(repoRoot, entries) {
240
403
  record = JSON.parse(readFileSync(absPath, 'utf-8'));
241
404
  } catch {
242
405
  // A record file we cannot even parse, that is also absent from the ledger,
243
- // is still an orphan — report it with what little identity we have.
406
+ // is still an orphan — report it with what little identity we have. We
407
+ // cannot read `.integrity` to classify it as pre-adoption either, so this
408
+ // fails loud rather than being silently excused.
244
409
  orphans.push({
245
410
  run_id: null,
246
411
  seq: null,
@@ -250,11 +415,67 @@ function collectOrphanRecords(repoRoot, entries) {
250
415
  continue;
251
416
  }
252
417
 
253
- const runId = record.run_id ?? null;
254
- if (runId && ledgerRunIds.has(runId)) continue;
418
+ // A value that parses but is not a plain record object (null, an array, a
419
+ // bare primitive) is not something we can read `.integrity` off to
420
+ // classify as pre-adoption. Mirroring the not-valid-JSON branch above:
421
+ // when we cannot positively identify what the record actually IS, fail
422
+ // loud as a genuine orphan rather than silently excusing it into
423
+ // preAdoption — the adoption boundary is a claim we can only make about
424
+ // something shaped like a record.
425
+ if (record === null || typeof record !== 'object' || Array.isArray(record)) {
426
+ orphans.push({
427
+ run_id: null,
428
+ seq: null,
429
+ path: relPath,
430
+ reason: `record file present on disk but absent from the ledger, and its JSON content is not a record object`,
431
+ });
432
+ continue;
433
+ }
434
+
435
+ // Adoption boundary — see the F-29134790 doc paragraph above. "No
436
+ // integrity block" is NECESSARY for pre-adoption but, on its own, not
437
+ // SUFFICIENT — see the F-1907b2bc doc paragraph above. Cross-check the
438
+ // record's own claimed timestamp against CHAIN_ADOPTION_DATE before
439
+ // accepting the "predates the chain" narrative.
440
+ if (record.integrity == null) {
441
+ const claimedAt = latestClaimedTimestamp(record);
442
+ const isPreAdoption = claimedAt !== null && claimedAt.getTime() < CHAIN_ADOPTION_DATE.getTime();
443
+
444
+ if (isPreAdoption) {
445
+ preAdoption.push({
446
+ run_id: record.run_id ?? null,
447
+ seq: null,
448
+ path: relPath,
449
+ reason:
450
+ `record predates the integrity chain (no integrity block — written ` +
451
+ `before tamper-evident chaining was introduced); expected for ` +
452
+ `historical records, not counted as an audit failure.`,
453
+ });
454
+ continue;
455
+ }
456
+
457
+ orphans.push({
458
+ run_id: record.run_id ?? null,
459
+ seq: null,
460
+ path: relPath,
461
+ reason: claimedAt === null
462
+ ? `record has no integrity block, and no readable timing.finished_at ` +
463
+ `or verification.verified_at either — its "predates the chain" claim ` +
464
+ `cannot be supported by anything on the record itself, so it is ` +
465
+ `treated as a genuine orphan rather than silently excused as historical.`
466
+ : `record has no integrity block, but its own timing.finished_at / ` +
467
+ `verification.verified_at (latest: ${claimedAt.toISOString()}) is on or ` +
468
+ `after the chain-adoption date (${CHAIN_ADOPTION_DATE.toISOString()}) — ` +
469
+ `a record genuinely written by writeRecord() from that date onward ` +
470
+ `always carries an integrity block, so this one cannot genuinely ` +
471
+ `predate the chain. Re-run ingest for this record or investigate how ` +
472
+ `it reached disk without one.`,
473
+ });
474
+ continue;
475
+ }
255
476
 
256
477
  orphans.push({
257
- run_id: runId,
478
+ run_id: record.run_id ?? null,
258
479
  seq: record.integrity?.seq ?? null,
259
480
  path: relPath,
260
481
  reason:
@@ -264,7 +485,7 @@ function collectOrphanRecords(repoRoot, entries) {
264
485
  });
265
486
  }
266
487
 
267
- return orphans;
488
+ return { orphans, preAdoption };
268
489
  }
269
490
 
270
491
  /**
@@ -282,6 +503,13 @@ export function formatChainResult(result) {
282
503
  ];
283
504
  if (Array.isArray(result.orphans)) {
284
505
  lines.push(`reconciliation: 0 orphan record(s) on disk`);
506
+ // F-29134790: ok:true only guarantees zero GENUINE orphans — pre-adoption
507
+ // records (no integrity block) are excused from ok but still worth
508
+ // surfacing so an operator isn't left wondering why the disk record
509
+ // count exceeds the ledgered count.
510
+ if (Array.isArray(result.pre_adoption) && result.pre_adoption.length > 0) {
511
+ lines.push(` (${result.pre_adoption.length} pre-chain record(s) predate the integrity ledger and are excluded from reconciliation)`);
512
+ }
285
513
  }
286
514
  return lines;
287
515
  }
@@ -312,5 +540,13 @@ export function formatChainResult(result) {
312
540
  }
313
541
  }
314
542
 
543
+ // F-29134790: pre-adoption records never make ok false, but when the result
544
+ // IS not-ok for some other reason (a break, or a genuine orphan), still name
545
+ // the excluded count so an operator triaging the failure isn't misled into
546
+ // thinking every un-ledgered record on disk is part of the break.
547
+ if (Array.isArray(result.pre_adoption) && result.pre_adoption.length > 0) {
548
+ lines.push(`reconciliation: ${result.pre_adoption.length} pre-chain record(s) predate the integrity ledger (excluded from reconciliation, not an audit failure)`);
549
+ }
550
+
315
551
  return lines;
316
552
  }