@dogfood-lab/ingest 1.3.1 → 1.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dogfood-lab/ingest",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "type": "module",
   "description": "Ingestion pipeline for testing-os. Thin glue: dispatch → verifier → persist → indexes.",
   "main": "run.js",

package/rebuild-indexes.js

@@ -25,7 +25,7 @@
  */
 
 import { readdirSync, readFileSync, writeFileSync, mkdirSync, existsSync, unlinkSync } from 'node:fs';
-import { join, relative } from 'node:path';
+import { join, relative, sep } from 'node:path';
 import { randomBytes } from 'node:crypto';
 
 import { logStage as sharedLogStage } from '@dogfood-lab/dogfood-swarm/lib/log-stage.js';
@@ -49,14 +49,44 @@ function logStage(stage, fields = {}) {
 /**
  * Recursively find all .json files under a directory.
  *
+ * d3-ingest-B005 (Stage C humanization): the `readdirSync` is wrapped so an
+ * unreadable subtree (EACCES, ENOTDIR, a Windows lock — the same class
+ * renameWithRetry defends against) degrades to "that subtree's records are
+ * missing from this rebuild" instead of throwing and aborting the WHOLE scan.
+ * This restores parity with the per-FILE tolerance of `loadRecord` just below
+ * (module header: "does NOT crash on a single bad file") and mirrors the
+ * in-repo precedent in `packages/portfolio/lib/parse-regression-pins.js`'s
+ * `walkSourceFiles`. The skip is NOT silent: a structured
+ * `logStage('warn', { kind: 'dir_unreadable', path })` NDJSON line makes it
+ * greppable alongside the rest of the pipeline (`"kind":"dir_unreadable"`), so
+ * the operator can see which subtree was dropped and why the index is partial.
+ * Exported so the guard is unit-testable in isolation (the single wrapped
+ * `readdirSync` serves both the top-level call and every recursive descent).
+ *
  * @param {string} dir
  * @returns {string[]}
  */
-function findJsonFiles(dir) {
+export function findJsonFiles(dir) {
   const results = [];
   if (!existsSync(dir)) return results;
 
-  const entries = readdirSync(dir, { withFileTypes: true });
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch (err) {
+    // Unreadable directory (locked / permission-restricted / not a dir).
+    // Skip this subtree rather than sinking the whole rebuild, but emit a
+    // structured, greppable warn naming the path so the partial index is
+    // explained — not a silent swallow.
+    logStage('warn', {
+      kind: 'dir_unreadable',
+      reason: err && err.code ? err.code : 'readdir_failed',
+      path: dir,
+      error: err && err.message ? err.message : String(err),
+    });
+    return results;
+  }
+
   for (const entry of entries) {
     const fullPath = join(dir, entry.name);
     if (entry.isDirectory()) {
@@ -115,7 +145,17 @@ export function rebuildIndexes(repoRoot, options = {}) {
   const skipped = [];
 
   for (const f of [...acceptedFiles, ...rejectedFiles]) {
-    const relPath = relative(repoRoot, f);
+    // SEED-1 (d3-ingest-001) — posixify at the serialization boundary.
+    // `relative()` returns OS-native separators (backslashes on win32). This
+    // value becomes `record._path` and is serialized verbatim into all three
+    // committed index `path` fields (latest-by-repo / failing / stale), which
+    // downstream consumers read as raw.githubusercontent.com URL fragments
+    // (docs/policy-contract.md Gate F) — a backslash there is a broken URL.
+    // Normalize ONCE here, at the single source the whole family flows through,
+    // so every serialized path is forward-slash regardless of host OS. Mirrors
+    // the canonical transform proven in
+    // packages/portfolio/lib/parse-regression-pins.js:75. NEVER a win32-skip.
+    const relPath = relative(repoRoot, f).split(sep).join('/');
     const { record, error } = loadRecord(f);
     if (error) {
       corrupted.push({ path: relPath, error });

package/run.js

@@ -17,7 +17,7 @@
  * - regenerate indexes
  */
 
-import { resolve, dirname } from 'node:path';
+import { resolve, dirname, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { randomBytes } from 'node:crypto';
 
@@ -30,6 +30,27 @@ import { rebuildIndexes } from './rebuild-indexes.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
+/**
+ * SEED-1 (d3-ingest-003) — posixify a path-shaped value at the operator/log
+ * SERIALIZATION boundary. `computeRecordPath`/`writeRecord` return OS-native
+ * paths (backslash-separated, absolute, on win32) because those values are
+ * also used for real filesystem operations. But the moment a path crosses into
+ * CLI JSON output, NDJSON log lines, or a downstream report
+ * (dogfood-swarm persist.js records `report.dogfood.path`), it must be
+ * forward-slash so operators and log pivots see one canonical shape across
+ * OSes — and so a copy-paste into a raw.githubusercontent URL is not a broken
+ * link. We posixify ONLY here, at the emit sites, leaving the returned fs paths
+ * OS-native for the filesystem layer. Mirrors the boundary-normalize doctrine
+ * already used in rebuild-indexes.js and parse-regression-pins.js. NEVER a
+ * win32-skip.
+ *
+ * @param {string|null} p
+ * @returns {string|null}
+ */
+function posixifyPath(p) {
+  return typeof p === 'string' ? p.split(sep).join('/') : p;
+}
+
 /**
  * Emit a single structured stage-transition log line via the shared helper.
  *
@@ -238,7 +259,10 @@ export async function ingest(submission, options) {
   logStage('persist_complete', {
     submission_id: submissionId,
     correlation_id,
-    path,
+    // d3-ingest-003: posixify at the log boundary — `path` is OS-native from
+    // writeRecord (used for the fs write); the NDJSON log surface gets forward
+    // slashes so log pivots are byte-identical across OSes.
+    path: posixifyPath(path),
     written,
     duplicate: !written,
     duration_ms: Date.now() - persistStart
@@ -281,11 +305,12 @@ export async function ingest(submission, options) {
         failed_stage: 'rebuild_indexes',
         message: err.message,
         stack: truncatedStack,
-        record_persisted_at: path,
+        // d3-ingest-003: operator-facing path → posixify at the log boundary.
+        record_persisted_at: posixifyPath(path),
         recovery: 'next ingest will trigger a full rebuild of indexes/'
       });
       console.error(
-        `WARNING: record persisted at ${path}, but index rebuild failed: ${err.message}\n` +
+        `WARNING: record persisted at ${posixifyPath(path)}, but index rebuild failed: ${err.message}\n` +
         `         indexes/ may be stale until next ingest. To force rebuild now, re-run any test ingest.\n` +
         `         stack: ${stackPreview}`
       );
@@ -429,7 +454,10 @@ export async function verifyOnly(submission, options) {
     submission_id: submissionId,
     correlation_id,
     status: record.verification?.status ?? null,
-    would_persist_to
+    // d3-ingest-003: posixify at the log boundary. The returned
+    // `would_persist_to` below stays OS-native so callers that resolve it
+    // against the filesystem keep a real fs path.
+    would_persist_to: posixifyPath(would_persist_to)
   });
 
   return { record, would_persist_to, verify_only: true };
@@ -467,7 +495,19 @@ const isMain = process.argv[1] && resolve(process.argv[1]) === resolve(__dirname
 
 if (isMain) {
   const args = process.argv.slice(2);
-  const repoRoot = resolve(__dirname, '../..');
+  // SEED-2 (d3-ingest-002) — make the CLI's repoRoot overridable so callers
+  // (notably dogfood-swarm's commands/persist.js execSync, and any test
+  // harness) can redirect every record write + index rebuild into a sandbox
+  // instead of the REAL working tree. Without this the only way to sandbox was
+  // a brittle source-copy of this file (the setupTempRunJs run.js-copy in
+  // d1b-001-cli-toplevel-error-event.test.js) that rewrote __dirname's `../..`
+  // walk. A production caller passes the real root explicitly; a test passes a
+  // temp dir; the default preserves the historical behavior when the env var
+  // is unset. resolve() makes a relative override absolute so the downstream
+  // join()s stay anchored.
+  const repoRoot = process.env.INGEST_REPO_ROOT
+    ? resolve(process.env.INGEST_REPO_ROOT)
+    : resolve(__dirname, '../..');
 
   // Parse CLI flags
   let submissionJson;
@@ -476,9 +516,28 @@ if (isMain) {
   const positionalArgs = [];
 
   for (let i = 0; i < args.length; i++) {
-    if (args[i] === '--provenance' && args[i + 1]) {
-      provenanceMode = args[++i];
-    } else if (args[i] === '--file' && args[i + 1]) {
+    // Accept BOTH the space form (`--flag value`) and the equals form
+    // (`--flag=value`). The prior parser matched only the space form, so a
+    // caller passing `--provenance=stub --file=...` (the shape dogfood-swarm's
+    // commands/persist.js builds for its execSync invocation) fell through to
+    // positionalArgs — `--provenance` then read as missing and the CLI exited 2
+    // ("--provenance flag is required"), silently breaking `swarm persist
+    // --ingest` on every platform. (dogfood-swarm self-audit follow-up.)
+    let arg = args[i];
+    let inlineValue = null;
+    if (arg.startsWith('--')) {
+      const eq = arg.indexOf('=');
+      if (eq !== -1) {
+        inlineValue = arg.slice(eq + 1);
+        arg = arg.slice(0, eq);
+      }
+    }
+    const hasValue = inlineValue !== null || args[i + 1] !== undefined;
+    const takeValue = () => (inlineValue !== null ? inlineValue : args[++i]);
+
+    if (arg === '--provenance' && hasValue) {
+      provenanceMode = takeValue();
+    } else if (arg === '--file' && hasValue) {
       const { readFileSync } = await import('node:fs');
       // D1B-001 family (operator-legibility): a --file read failure
       // (ENOENT/EACCES) routes through the structured error event and exits 2
@@ -488,7 +547,7 @@ if (isMain) {
       // correlation id here (the same pivot the JSON.parse catch uses when
       // there is no submission to derive a run_id from yet).
       try {
-        submissionJson = readFileSync(resolve(args[++i]), 'utf-8');
+        submissionJson = readFileSync(resolve(takeValue()), 'utf-8');
       } catch (err) {
         emitCliErrorEvent({
           failedStage: 'cli_read_file',
@@ -498,9 +557,9 @@ if (isMain) {
         });
         process.exit(2);
       }
-    } else if (args[i] === '--payload' && args[i + 1]) {
-      submissionJson = args[++i];
-    } else if (args[i] === '--verify-only') {
+    } else if (arg === '--payload' && hasValue) {
+      submissionJson = takeValue();
+    } else if (arg === '--verify-only') {
       // F-252714-058: dry-run the pipeline without writing or rebuilding
       // indexes. CI / operators preview what WOULD have been persisted.
       verifyOnlyFlag = true;
@@ -625,7 +684,10 @@ if (isMain) {
         status: result.record.verification.status,
         run_id: result.record.run_id ?? null,
         verdict: result.record.overall_verdict?.verified ?? null,
-        would_persist_to: result.would_persist_to,
+        // d3-ingest-003: posixify path-shaped CLI output so the operator
+        // contract is identical across OSes (a Windows backslash here breaks
+        // any downstream URL-build/string-match).
+        would_persist_to: posixifyPath(result.would_persist_to),
         verify_only: true,
         rejection_reasons: result.record.verification.rejection_reasons ?? []
       }));
@@ -648,7 +710,9 @@ if (isMain) {
       status: result.record.verification.status,
       run_id: result.record.run_id ?? null,
       verdict: result.record.overall_verdict?.verified ?? null,
-      path: result.path,
+      // d3-ingest-003: posixify path-shaped CLI output (same family as
+      // would_persist_to above). dogfood-swarm's persist.js pivots on this.
+      path: posixifyPath(result.path),
       written: result.written,
       rejection_reasons: result.record.verification.rejection_reasons ?? []
     }));