@dogfood-lab/findings 1.2.3 → 1.3.0

package/lib/safe-yaml-load.js

@@ -0,0 +1,169 @@
+/**
+ * safe-yaml-load — single-source structured loader for the silent-loader family
+ * (F-721047-010 / D2 amend wave 1).
+ *
+ * **Why this helper exists.** Multiple sites in `packages/findings/**` were
+ * implementing the same pattern: walk a directory, parse `.yaml` (or `.json`)
+ * files, and silently swallow every error with `try { ... } catch {}`. A torn
+ * YAML file, an EACCES, a transient ENOENT — all disappeared, producing
+ * partial results that looked complete. The advisor verified four such sites
+ * (`review/event-log.js walkYaml`, `synthesis/recommendation-derivation.js
+ * loadAcceptedPatterns`, `synthesis/write-artifacts.js loadArtifacts`,
+ * `derive/load-records.js walkRecords/findRecordFile`) and surfaced two
+ * advisor-found siblings in `derive/load-records.js`. The audit's framing
+ * was that this was a "closed" anti-pattern but the header in
+ * `lib/atomic-write.js` actually says it's OPEN — so this fix closes a
+ * known-open pattern rather than regressing on a closure claim.
+ *
+ * **The contract.**
+ *   loadYamlFile(path) → { data, error }
+ *   loadJsonFile(path) → { data, error }
+ *   loadYamlDir(dir, { recursive }) → { entries: [{path, data}], skipped: [{path, error}] }
+ *   loadJsonDir(dir, { recursive }) → same shape
+ *
+ * - On success: `data` is the parsed payload, `error` is `null`.
+ * - On failure: `data` is `null`, `error` is a string ("YAML parse error: …",
+ *   "JSON parse error: …", "Read error: …" etc.). The caller can log the
+ *   structured skip, fail loud, or continue — but never silently disappear.
+ * - This is the same shape `validate.js parseFinding` returns, deliberately
+ *   so existing consumers can swap in this helper without reshaping.
+ *
+ * **What this is NOT.** This is not a schema validator. It only does the
+ * read+parse step. Schema validation lives in `validate.js` for findings
+ * and in `@dogfood-lab/schemas` for the other contracts.
+ *
+ * **Concurrency.** No locking — this is a pure read helper. Callers that
+ * need atomicity layer `withFileLock` on top (see `review/event-log.js
+ * appendEvent`).
+ */
+
+import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs';
+import { join, extname } from 'node:path';
+import yaml from 'js-yaml';
+
+/**
+ * Parse a YAML file. Returns `{ data, error }` — never throws for parse or
+ * read failures.
+ *
+ * @param {string} filePath - Absolute path to a YAML file.
+ * @returns {{ data: unknown, error: string | null }}
+ */
+export function loadYamlFile(filePath) {
+  let raw;
+  try {
+    raw = readFileSync(filePath, 'utf-8');
+  } catch (err) {
+    return { data: null, error: `Read error: ${err.message}` };
+  }
+  try {
+    const data = yaml.load(raw);
+    return { data, error: null };
+  } catch (err) {
+    return { data: null, error: `YAML parse error: ${err.message}` };
+  }
+}
+
+/**
+ * Parse a JSON file. Returns `{ data, error }` — never throws for parse or
+ * read failures.
+ *
+ * @param {string} filePath - Absolute path to a JSON file.
+ * @returns {{ data: unknown, error: string | null }}
+ */
+export function loadJsonFile(filePath) {
+  let raw;
+  try {
+    raw = readFileSync(filePath, 'utf-8');
+  } catch (err) {
+    return { data: null, error: `Read error: ${err.message}` };
+  }
+  try {
+    const data = JSON.parse(raw);
+    return { data, error: null };
+  } catch (err) {
+    return { data: null, error: `JSON parse error: ${err.message}` };
+  }
+}
+
+/**
+ * Walk a directory tree and load every matching file using `loadFile`.
+ *
+ * Returns `{ entries, skipped }`:
+ *   - `entries`: `[{ path, data }]` — successful loads.
+ *   - `skipped`: `[{ path, error }]` — files that couldn't be read or parsed,
+ *     each with a structured error string. The caller decides whether to
+ *     log, throw, or just count.
+ *
+ * If `dir` does not exist, returns `{ entries: [], skipped: [] }` (an empty
+ * directory is not an error).
+ *
+ * If `readdirSync` or `statSync` on a child throws, the child is recorded in
+ * `skipped` rather than aborting the whole walk. A torn permission on one
+ * subdir cannot make sibling subdirs disappear.
+ *
+ * @param {string} dir - Absolute path to the directory.
+ * @param {{ recursive?: boolean, ext?: string, loadFile?: (path: string) => { data: unknown, error: string | null } }} [opts]
+ * @returns {{ entries: Array<{ path: string, data: unknown }>, skipped: Array<{ path: string, error: string }> }}
+ */
+function walkDir(dir, { recursive = true, ext = '.yaml', loadFile = loadYamlFile } = {}) {
+  const entries = [];
+  const skipped = [];
+
+  if (!existsSync(dir)) return { entries, skipped };
+
+  function visit(d) {
+    let children;
+    try {
+      children = readdirSync(d);
+    } catch (err) {
+      skipped.push({ path: d, error: `readdir error: ${err.message}` });
+      return;
+    }
+    for (const child of children) {
+      const full = join(d, child);
+      let stat;
+      try {
+        stat = statSync(full);
+      } catch (err) {
+        skipped.push({ path: full, error: `stat error: ${err.message}` });
+        continue;
+      }
+      if (stat.isDirectory()) {
+        if (recursive) visit(full);
+        continue;
+      }
+      if (extname(child) !== ext) continue;
+      const { data, error } = loadFile(full);
+      if (error !== null) {
+        skipped.push({ path: full, error });
+      } else {
+        entries.push({ path: full, data });
+      }
+    }
+  }
+
+  visit(dir);
+  return { entries, skipped };
+}
+
+/**
+ * Walk a directory tree and load every `.yaml` file. See `walkDir` for shape.
+ *
+ * @param {string} dir
+ * @param {{ recursive?: boolean }} [opts]
+ * @returns {{ entries: Array<{ path: string, data: unknown }>, skipped: Array<{ path: string, error: string }> }}
+ */
+export function loadYamlDir(dir, opts = {}) {
+  return walkDir(dir, { ...opts, ext: '.yaml', loadFile: loadYamlFile });
+}
+
+/**
+ * Walk a directory tree and load every `.json` file. See `walkDir` for shape.
+ *
+ * @param {string} dir
+ * @param {{ recursive?: boolean }} [opts]
+ * @returns {{ entries: Array<{ path: string, data: unknown }>, skipped: Array<{ path: string, error: string }> }}
+ */
+export function loadJsonDir(dir, opts = {}) {
+  return walkDir(dir, { ...opts, ext: '.json', loadFile: loadJsonFile });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dogfood-lab/findings",
-  "version": "1.2.3",
+  "version": "1.3.0",
   "type": "module",
   "description": "Finding contract spine for testing-os. Validates, reads, lists, and queries evidence-bound findings — the fourth contract alongside record, scenario, and policy.",
   "main": "index.js",
@@ -18,7 +18,7 @@
     "findings": "./cli.js"
   },
   "scripts": {
-    "test": "node --test findings.test.js derive/derive.test.js review/review.test.js synthesis/synthesis.test.js advise/advise.test.js lib/atomic-write.test.js"
+    "test": "node --test findings.test.js derive/derive.test.js derive/load-records-skip.test.js derive/d2b-008-collision-guard.test.js derive/d2b-002-write-schema-gate.test.js review/review.test.js review/event-log-loader.test.js review/h4-engine-auto-reject-reason.test.js synthesis/synthesis.test.js synthesis/loaders-skip.test.js synthesis/d2b-001-derive-skipped-signal.test.js advise/advise.test.js lib/atomic-write.test.js lib/safe-yaml-load.test.js lib/d1b-002-findings-sleepsync.test.js"
   },
   "files": [
     "index.js",
@@ -41,8 +41,6 @@
   "dependencies": {
     "@dogfood-lab/ingest": "^1.2.0",
     "@dogfood-lab/schemas": "^1.2.0",
-    "ajv": "^8.18.0",
-    "ajv-formats": "^3.0.1",
     "js-yaml": "^4.1.0"
   },
   "engines": {

package/review/event-log.js

@@ -4,13 +4,14 @@
  * Events are stored as YAML arrays in reviews/<YYYY>/<date>-finding-review-log.yaml
  */
 
-import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync } from 'node:fs';
-import { resolve, dirname, join } from 'node:path';
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { resolve, dirname } from 'node:path';
 import { randomBytes } from 'node:crypto';
 import yaml from 'js-yaml';
 
 import { withFileLock } from '../lib/file-lock.js';
 import { renameWithRetry } from '../lib/rename-with-retry.js';
+import { loadYamlDir } from '../lib/safe-yaml-load.js';
 
 let _eventCounter = 0;
 
@@ -147,31 +148,56 @@ export function getEventsForFinding(rootDir, findingId) {
 
 /**
  * Read all events across all findings.
+ *
+ * Legacy shape: returns an array of events. Torn log files are NOT silently
+ * dropped — they surface as structured skip records via the sibling API
+ * `getAllEventsWithSkips()`. Callers that need to see which log files
+ * failed to load should use that API; this one is kept array-shaped for
+ * backward compatibility with the existing read sites (CLI display,
+ * `getEventsForFinding`, the review.test.js / event-log-race.test.js
+ * fixtures).
+ *
+ * H1 / F-721047-010 — silent-loader closure: previously the internal
+ * `walkYaml` helper wrapped the whole `readdir`/`statSync`/`readFileSync`/
+ * `yaml.load` loop in a single `try { ... } catch { /* skip bad files * / }`,
+ * which made torn YAML, EACCES, and ENOENT all disappear with no signal.
+ * The walk is now delegated to `loadYamlDir`, which returns
+ * `{ entries, skipped }` — every torn file is recorded with a structured
+ * error rather than dropped.
  */
 export function getAllEvents(rootDir) {
-  const reviewsDir = resolve(rootDir, 'reviews');
-  if (!existsSync(reviewsDir)) return [];
+  return getAllEventsWithSkips(rootDir).events;
+}
 
-  const events = [];
-  walkYaml(reviewsDir, data => {
-    if (Array.isArray(data)) events.push(...data);
-  });
+/**
+ * Read all events across all findings, plus a list of any log files that
+ * failed to load (torn YAML, EACCES, etc.). The structured-skip API the
+ * audit asked for — callers that care about pipeline honesty (rebuild
+ * scripts, doctor commands, CI checks) should consume this one rather
+ * than the array-shaped legacy `getAllEvents`.
+ *
+ * @param {string} rootDir
+ * @returns {{ events: object[], skipped: Array<{ path: string, error: string }> }}
+ */
+export function getAllEventsWithSkips(rootDir) {
+  const reviewsDir = resolve(rootDir, 'reviews');
+  if (!existsSync(reviewsDir)) return { events: [], skipped: [] };
 
-  return events.sort((a, b) => (a.timestamp || '').localeCompare(b.timestamp || ''));
-}
+  const { entries, skipped } = loadYamlDir(reviewsDir, { recursive: true });
 
-/** Walk directory tree for .yaml files, parse and call cb with data. */
-function walkYaml(dir, cb) {
-  for (const entry of readdirSync(dir)) {
-    const full = join(dir, entry);
-    try {
-      if (statSync(full).isDirectory()) {
-        walkYaml(full, cb);
-      } else if (entry.endsWith('.yaml')) {
-        const raw = readFileSync(full, 'utf-8');
-        const data = yaml.load(raw);
-        if (data) cb(data);
-      }
-    } catch { /* skip bad files */ }
+  const events = [];
+  for (const entry of entries) {
+    const data = entry.data;
+    if (!data) continue;
+    if (Array.isArray(data)) {
+      events.push(...data);
+    } else {
+      // A single-event YAML file (defensive: shouldn't happen for daily logs,
+      // but the original walkYaml tolerated either shape).
+      events.push(data);
+    }
   }
+
+  events.sort((a, b) => (a.timestamp || '').localeCompare(b.timestamp || ''));
+  return { events, skipped };
 }

package/review/review-engine.js

@@ -101,13 +101,34 @@ export function performAction(rootDir, params) {
 
   // Apply review metadata
   const now = new Date().toISOString();
+
+  // H4 / F-stage-a-h4 — auto-populate review.reject_reason whenever the
+  // target status is 'rejected'. Pre-amend, the engine only set the field
+  // for `action === 'reject'` and silently produced rejected findings on
+  // merge / supersede without a structured reason. The H4 schema if/then
+  // would then reject those findings at the contract boundary — but the
+  // engine was producing them. The fix:
+  //
+  //   - Operator-supplied `params.rejectReason` always wins (override).
+  //   - `merge` and `supersede` auto-default to `'merged_into_canonical'`
+  //     since lineage already carries `superseded_by` / `merged_from`.
+  //   - For `action === 'reject'`, the engine does NOT invent a default —
+  //     defaulting would erase operator intent. The schema enforces that
+  //     the field is set; an operator who forgets sees a validation error
+  //     downstream and learns to pass one. (This is the intentional
+  //     test-and-tell path in h4-engine-auto-reject-reason.test.js.)
+  let effectiveRejectReason = params.rejectReason;
+  if (!effectiveRejectReason && toStatus === 'rejected' && (action === 'merge' || action === 'supersede')) {
+    effectiveRejectReason = 'merged_into_canonical';
+  }
+
   finding.review = {
     reviewed_by: actor,
     reviewed_at: now,
     last_action: action,
     ...(params.reason ? { decision_reason: params.reason } : {}),
     ...(params.notes ? { review_notes: params.notes } : {}),
-    ...(params.rejectReason && action === 'reject' ? { reject_reason: params.rejectReason } : {})
+    ...(effectiveRejectReason ? { reject_reason: effectiveRejectReason } : {})
   };
 
   // Update timestamps

package/synthesis/doctrine-derivation.js

@@ -11,16 +11,23 @@
 import { readFileSync, readdirSync, existsSync } from 'node:fs';
 import { resolve, join } from 'node:path';
 import yaml from 'js-yaml';
-import { loadAcceptedPatterns } from './recommendation-derivation.js';
+import { loadAcceptedPatterns, loadAcceptedPatternsWithSkips } from './recommendation-derivation.js';
 
 /**
  * Derive doctrine from strong accepted patterns.
  *
+ * D2B-001 — return shape carries a `skipped: [{path, error}]` field so the
+ * silent-loader signal propagates through the derive API. A torn pattern
+ * that WOULD have been considered for doctrine synthesis no longer
+ * disappears with no signal. Legacy callers reading `doctrines` / `stats`
+ * are unaffected.
+ *
  * @param {string} rootDir - dogfood-labs repo root
- * @returns {{ doctrines: Array, stats: { patternsConsidered: number, doctrinesEmitted: number, belowThreshold: number } }}
+ * @returns {{ doctrines: Array, skipped: Array<{path: string, error: string}>, stats: { patternsConsidered: number, doctrinesEmitted: number, belowThreshold: number, patternsSkipped: number } }}
  */
 export function deriveDoctrine(rootDir) {
-  const patterns = loadAcceptedPatterns(rootDir);
+  const { entries, skipped } = loadAcceptedPatternsWithSkips(rootDir);
+  const patterns = entries.map(e => e.data);
   const strong = patterns.filter(p =>
     p.pattern_strength === 'strong' || p.pattern_strength === 'portfolio_stable'
   );
@@ -44,10 +51,12 @@ export function deriveDoctrine(rootDir) {
 
   return {
     doctrines,
+    skipped, // D2B-001: structured skip list for operator legibility
     stats: {
       patternsConsidered: patterns.length,
       doctrinesEmitted: doctrines.length,
-      belowThreshold
+      belowThreshold,
+      patternsSkipped: skipped.length
     }
   };
 }

package/synthesis/index.js

@@ -2,7 +2,11 @@
  * Synthesis layer exports.
  */
 export { derivePatterns } from './pattern-derivation.js';
-export { deriveRecommendations, loadAcceptedPatterns } from './recommendation-derivation.js';
+export { deriveRecommendations, loadAcceptedPatterns, loadAcceptedPatternsWithSkips } from './recommendation-derivation.js';
 export { deriveDoctrine } from './doctrine-derivation.js';
 export { validatePattern, validateRecommendation, validateDoctrine } from './validate-artifacts.js';
-export { writePattern, writeRecommendation, writeDoctrine, loadPatterns, loadRecommendations, loadDoctrines } from './write-artifacts.js';
+export {
+  writePattern, writeRecommendation, writeDoctrine,
+  writePatterns, writeRecommendations, writeDoctrines,
+  loadPatterns, loadRecommendations, loadDoctrines
+} from './write-artifacts.js';

package/synthesis/recommendation-derivation.js

@@ -5,18 +5,26 @@
  * pattern kind, dimensions, and transfer scope.
  */
 
-import { readFileSync, readdirSync, existsSync } from 'node:fs';
-import { resolve, join } from 'node:path';
-import yaml from 'js-yaml';
+import { resolve } from 'node:path';
+
+import { loadYamlDir } from '../lib/safe-yaml-load.js';
 
 /**
  * Derive recommendations from accepted patterns.
  *
+ * D2B-001 — return shape carries a `skipped: [{path, error}]` field so the
+ * silent-loader signal that Wave A1 wired through `loadYamlDir` now propagates
+ * through the derive API to the operator surface (CLI). Legacy callers that
+ * only read `recommendations` / `stats` are unaffected — the field is
+ * additive. A torn pattern that WOULD have been considered for recommendation
+ * synthesis no longer disappears with no signal.
+ *
  * @param {string} rootDir - dogfood-labs repo root
- * @returns {{ recommendations: Array, stats: { patternsConsidered: number, recommendationsEmitted: number } }}
+ * @returns {{ recommendations: Array, skipped: Array<{path: string, error: string}>, stats: { patternsConsidered: number, recommendationsEmitted: number, patternsSkipped: number } }}
  */
 export function deriveRecommendations(rootDir) {
-  const patterns = loadAcceptedPatterns(rootDir);
+  const { entries, skipped } = loadAcceptedPatternsWithSkips(rootDir);
+  const patterns = entries.map(e => e.data);
   const recommendations = [];
 
   for (const pat of patterns) {
@@ -26,9 +34,11 @@ export function deriveRecommendations(rootDir) {
 
   return {
     recommendations,
+    skipped, // D2B-001: structured skip list for operator legibility
     stats: {
       patternsConsidered: patterns.length,
-      recommendationsEmitted: recommendations.length
+      recommendationsEmitted: recommendations.length,
+      patternsSkipped: skipped.length
     }
   };
 }
@@ -137,20 +147,41 @@ function fmtSurfaces(pattern) {
 
 /**
  * Load accepted patterns from disk.
+ *
+ * Legacy array shape: returns only accepted patterns. Torn pattern YAML
+ * files are NO LONGER silently dropped — they surface via the sibling
+ * structured API `loadAcceptedPatternsWithSkips`. This call delegates to
+ * that one and discards the skipped list for callers that don't yet care
+ * about pipeline honesty.
+ *
+ * H2 / F-721047-010 — silent-loader closure: the previous implementation
+ * wrapped `yaml.load(readFileSync(...))` in a bare `try { ... } catch {}`,
+ * silently dropping every torn pattern. Now delegated to the shared
+ * `loadYamlDir` helper which returns structured skip records.
  */
 function loadAcceptedPatterns(rootDir) {
+  return loadAcceptedPatternsWithSkips(rootDir).entries.map(e => e.data);
+}
+
+/**
+ * Load accepted patterns from disk, surfacing structured skip records for
+ * any torn pattern file. The audit-honesty API.
+ *
+ * Only `entries[].data` whose `status === 'accepted'` are returned in
+ * entries — the legacy `loadAcceptedPatterns` filter is preserved. The
+ * `skipped` list captures every torn file (status unknowable) so a torn
+ * pattern that WOULD have been accepted does not silently vanish.
+ *
+ * @param {string} rootDir
+ * @returns {{ entries: Array<{ path: string, data: object }>, skipped: Array<{ path: string, error: string }> }}
+ */
+function loadAcceptedPatternsWithSkips(rootDir) {
   const dir = resolve(rootDir, 'patterns');
-  if (!existsSync(dir)) return [];
-
-  const patterns = [];
-  for (const file of readdirSync(dir)) {
-    if (!file.endsWith('.yaml')) continue;
-    try {
-      const data = yaml.load(readFileSync(join(dir, file), 'utf-8'));
-      if (data?.status === 'accepted') patterns.push(data);
-    } catch { /* skip */ }
-  }
-  return patterns;
+  const { entries, skipped } = loadYamlDir(dir, { recursive: false });
+  return {
+    entries: entries.filter(e => e.data?.status === 'accepted'),
+    skipped,
+  };
 }
 
-export { loadAcceptedPatterns };
+export { loadAcceptedPatterns, loadAcceptedPatternsWithSkips };

package/synthesis/validate-artifacts.js

@@ -1,46 +1,38 @@
 /**
  * Schema validation for pattern, recommendation, and doctrine artifacts.
+ *
+ * H3 hop 1: delegates to the canonical {@link validatePayload} from
+ * `@dogfood-lab/schemas`. Pre-H3 this module compiled its own
+ * Ajv2020 + ajv-formats instance per schema; that duplicated the
+ * verifier's compile path and created the C1 two-Ajv structural gap
+ * (same JSON Schema → two distinct compiled validators in two
+ * sibling packages). The migration collapses pattern, recommendation,
+ * and doctrine to the single cached validator the canonical seam
+ * shares with the rest of the workspace.
+ *
+ * Return contract preserved: `{ valid, errors: [{ path, message }] }`.
+ * Synthesis callers ignored `params` and `keyword` historically — we
+ * project the canonical ValidationError down to the narrower shape
+ * the synthesis layer actually uses.
  */
 
-import { readFileSync } from 'node:fs';
-import { dirname } from 'node:path';
-import { createRequire } from 'node:module';
-import Ajv2020 from 'ajv/dist/2020.js';
-import addFormats from 'ajv-formats';
-import yaml from 'js-yaml';
+import { validatePayload } from '@dogfood-lab/schemas';
 
-const require = createRequire(import.meta.url);
-// Resolve the schemas package's json directory via its subpath export.
-const SCHEMAS_DIR = dirname(
-  require.resolve('@dogfood-lab/schemas/json/dogfood-pattern.schema.json')
-);
-
-const _validators = {};
-
-function getValidator(schemaFile) {
-  if (!_validators[schemaFile]) {
-    const schema = JSON.parse(readFileSync(`${SCHEMAS_DIR}/${schemaFile}`, 'utf-8'));
-    const ajv = new Ajv2020({ allErrors: true, strict: false });
-    addFormats(ajv);
-    _validators[schemaFile] = ajv.compile(schema);
-  }
-  return _validators[schemaFile];
+function projectErrors(errors) {
+  return errors.map(e => ({ path: e.path, message: e.message }));
 }
 
 export function validatePattern(data) {
-  const validate = getValidator('dogfood-pattern.schema.json');
-  const valid = validate(data);
-  return { valid, errors: valid ? [] : (validate.errors || []).map(e => ({ path: e.instancePath || '/', message: e.message })) };
+  const result = validatePayload('pattern', data);
+  return { valid: result.valid, errors: projectErrors(result.errors) };
 }
 
 export function validateRecommendation(data) {
-  const validate = getValidator('dogfood-recommendation.schema.json');
-  const valid = validate(data);
-  return { valid, errors: valid ? [] : (validate.errors || []).map(e => ({ path: e.instancePath || '/', message: e.message })) };
+  const result = validatePayload('recommendation', data);
+  return { valid: result.valid, errors: projectErrors(result.errors) };
 }
 
 export function validateDoctrine(data) {
-  const validate = getValidator('dogfood-doctrine.schema.json');
-  const valid = validate(data);
-  return { valid, errors: valid ? [] : (validate.errors || []).map(e => ({ path: e.instancePath || '/', message: e.message })) };
+  const result = validatePayload('doctrine', data);
+  return { valid: result.valid, errors: projectErrors(result.errors) };
 }