@mcptoolshop/style-dataset-lab 3.3.0 → 3.4.0

package/lib/config.js

@@ -10,6 +10,7 @@ import { readFileSync, existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { inputError, runtimeError } from './errors.js';
+import { warn } from './log.js';
 
 function loadJsonFile(filePath) {
   if (!existsSync(filePath)) return null;
@@ -25,6 +26,80 @@ function loadJsonFile(filePath) {
   }
 }
 
+/**
+ * Validate a user-authored regex pattern string at CONFIG LOAD TIME rather
+ * than lazily inside detectLane()/detectGroup() on whatever record happens
+ * to process first.
+ *
+ * B02: detectLane()/detectGroup() build `new RegExp(pattern)` from strings
+ * loaded verbatim from lanes.json/terminology.json across 7 construction
+ * sites, with zero validation before this. One unescaped bracket
+ * (`"[freeport"`) used to abort a whole `sdlab bind` pass with a raw
+ * `SyntaxError: Invalid regular expression` naming neither the file, the
+ * lane/group, nor the offending pattern — finding it required manual
+ * bisection of the config. This throws ONE structured error naming all
+ * three, before any record is processed.
+ *
+ * @param {string} pattern
+ * @param {{file: string, context: string}} where
+ */
+function validateRegexPattern(pattern, { file, context }) {
+  try {
+    // eslint-disable-next-line no-new -- validation only, result unused
+    new RegExp(pattern);
+  } catch (err) {
+    throw inputError(
+      'INPUT_BAD_REGEX',
+      `Invalid regex pattern in ${file} (${context}): "${pattern}" — ${err.message}`,
+      'Fix or escape the pattern (e.g. escape literal brackets/parens/braces) and try again.'
+    );
+  }
+}
+
+/**
+ * Validate every id_patterns entry across all lanes in a loaded lanes.json.
+ * Fails fast on the FIRST invalid pattern (not an aggregate report) so the
+ * thrown error always names one concrete offending pattern.
+ */
+function validateLanePatterns(lanesData) {
+  for (const lane of lanesData.lanes || []) {
+    for (const pattern of lane.id_patterns || []) {
+      validateRegexPattern(pattern, { file: 'lanes.json', context: `lane "${lane.id}"` });
+    }
+  }
+}
+
+/**
+ * Validate every pattern site across a loaded terminology.json: per-group
+ * id_patterns/prompt_patterns, plus the four dict/array pattern collections
+ * detectGroup() also builds RegExp from (cross_group_patterns, edge_defaults,
+ * null_faction_patterns, id_fallbacks) — the full set of 6 sites detectGroup()
+ * itself constructs (the 7th is detectLane()'s single site, covered by
+ * validateLanePatterns above).
+ */
+function validateTerminologyPatterns(termData) {
+  for (const [groupName, group] of Object.entries(termData.groups || {})) {
+    for (const pattern of group?.id_patterns || []) {
+      validateRegexPattern(pattern, { file: 'terminology.json', context: `group "${groupName}" id_patterns` });
+    }
+    for (const pattern of group?.prompt_patterns || []) {
+      validateRegexPattern(pattern, { file: 'terminology.json', context: `group "${groupName}" prompt_patterns` });
+    }
+  }
+  for (const pattern of Object.keys(termData.cross_group_patterns || {})) {
+    validateRegexPattern(pattern, { file: 'terminology.json', context: 'cross_group_patterns' });
+  }
+  for (const pattern of Object.keys(termData.edge_defaults || {})) {
+    validateRegexPattern(pattern, { file: 'terminology.json', context: 'edge_defaults' });
+  }
+  for (const pattern of termData.null_faction_patterns || []) {
+    validateRegexPattern(pattern, { file: 'terminology.json', context: 'null_faction_patterns' });
+  }
+  for (const pattern of Object.keys(termData.id_fallbacks || {})) {
+    validateRegexPattern(pattern, { file: 'terminology.json', context: 'id_fallbacks' });
+  }
+}
+
 /**
  * Async-safe readFile + JSON.parse with structured error handling.
  * Optionally validates that required top-level fields are present.
@@ -75,8 +150,12 @@ export async function readJsonFile(filePath, opts = {}) {
  * Load project.json — project identity and generation defaults.
  */
 export function loadProjectMeta(projectRoot) {
-  const data = loadJsonFile(join(projectRoot, 'project.json'));
-  if (!data) {
+  const filePath = join(projectRoot, 'project.json');
+  if (!existsSync(filePath)) {
+    // B05: this loader used to substitute its default with zero logging —
+    // the module didn't even import lib/log.js. A dropped/never-created
+    // project.json now names its expected path instead of degrading silently.
+    warn(`project.json not found at ${filePath} — using built-in defaults (name: "unknown")`);
     return {
       name: 'unknown',
       display_name: 'Unknown Project',
@@ -85,7 +164,7 @@ export function loadProjectMeta(projectRoot) {
       defaults: {},
     };
   }
-  return data;
+  return loadJsonFile(filePath);
 }
 
 /**
@@ -111,8 +190,14 @@ export function resolveGpuModel(projectDefaults) {
  * Load constitution.json — array of rule objects.
  */
 export function loadConstitution(projectRoot) {
-  const data = loadJsonFile(join(projectRoot, 'constitution.json'));
-  if (!data) return { rules: [] };
+  const filePath = join(projectRoot, 'constitution.json');
+  if (!existsSync(filePath)) {
+    // B05: see loadProjectMeta above — warn once, naming the expected path,
+    // instead of silently substituting the empty-rules default.
+    warn(`constitution.json not found at ${filePath} — using built-in defaults (no rules configured)`);
+    return { rules: [] };
+  }
+  const data = loadJsonFile(filePath);
   if (Array.isArray(data)) return { rules: data };
   if (!data.rules) {
     throw inputError(
@@ -128,10 +213,14 @@ export function loadConstitution(projectRoot) {
  * Load lanes.json — lane definitions with detection patterns.
  */
 export function loadLanes(projectRoot) {
-  const data = loadJsonFile(join(projectRoot, 'lanes.json'));
-  if (!data) {
+  const filePath = join(projectRoot, 'lanes.json');
+  if (!existsSync(filePath)) {
+    // B05: warn once, naming the expected path, instead of silently
+    // substituting the no-lanes default.
+    warn(`lanes.json not found at ${filePath} — using built-in defaults (no lanes configured)`);
     return { default_lane: 'default', lanes: [] };
   }
+  const data = loadJsonFile(filePath);
   if (!data.lanes) {
     throw inputError(
       'INPUT_BAD_CONFIG',
@@ -139,6 +228,9 @@ export function loadLanes(projectRoot) {
       'Top-level shape: { "default_lane": "...", "lanes": [ ... ] }.'
     );
   }
+  // B02: validate every id_patterns regex now, before any record is ever
+  // tested against it — see validateRegexPattern's docstring above.
+  validateLanePatterns(data);
   return data;
 }
 
@@ -146,23 +238,31 @@ export function loadLanes(projectRoot) {
  * Load rubric.json — scoring dimensions, thresholds, failure modes.
  */
 export function loadRubric(projectRoot) {
-  const data = loadJsonFile(join(projectRoot, 'rubric.json'));
-  if (!data) {
+  const filePath = join(projectRoot, 'rubric.json');
+  if (!existsSync(filePath)) {
+    // B05: warn once, naming the expected path, instead of silently
+    // substituting the built-in thresholds default.
+    warn(`rubric.json not found at ${filePath} — using built-in defaults (pass=0.7/partial=0.5/fail_ceiling=0.4)`);
     return {
       dimensions: [],
       thresholds: { pass: 0.7, partial: 0.5, fail_ceiling: 0.4 },
       failure_to_rules: {},
     };
   }
-  return data;
+  return loadJsonFile(filePath);
 }
 
 /**
  * Load terminology.json — group vocabulary and detection patterns.
  */
 export function loadTerminology(projectRoot) {
-  const data = loadJsonFile(join(projectRoot, 'terminology.json'));
-  if (!data) {
+  const filePath = join(projectRoot, 'terminology.json');
+  if (!existsSync(filePath)) {
+    // B05: this is the finding's own example — a dropped terminology.json
+    // used to make detectGroup() return null for every record with zero
+    // signal. Warn once, naming the expected path, instead of substituting
+    // the empty-groups default silently.
+    warn(`terminology.json not found at ${filePath} — using built-in defaults (no groups configured)`);
     return {
       group_label: 'group',
       groups: {},
@@ -171,6 +271,10 @@ export function loadTerminology(projectRoot) {
       prompt_fallbacks: {},
     };
   }
+  const data = loadJsonFile(filePath);
+  // B02: validate every pattern site now, before any record is ever tested
+  // against it — see validateRegexPattern's docstring above.
+  validateTerminologyPatterns(data);
   return data;
 }
 

package/lib/critique-engine.js

@@ -255,6 +255,17 @@ export function critiqueCandidate({ brief, workflow, candidate, activeDimensions
     filename: candidate.filename,
     seed: candidate.seed,
     overall_fit: 'usable', // Default — human or LLM overrides
+    // H2: this is the ONLY assignment site for overall_fit anywhere in the
+    // codebase — nothing currently overrides it, so every candidate in
+    // every critique report is uniformly 'usable' by construction. That is
+    // a non-judgment ("nobody has looked at the image yet"), not a judgment
+    // ("looked, and they're all fine") — `reviewed: false` records which one
+    // this is so recommendAction() (below) and the CLI can tell the two
+    // apart instead of presenting keyword-drift tiebreaking as a considered
+    // recommendation. Flip to true at the (future, NOT built here — see the
+    // finding's explicit scoping note) point where a human or LLM judge
+    // actually assesses fit and overrides overall_fit.
+    reviewed: false,
     // EXTERNAL_VERIFIER: this rule-based engine is a different artifact class
     // from the image generator, so it is a legitimate cross-artifact verifier.
     judged_by_model: RULE_BASED_JUDGE,
@@ -301,12 +312,27 @@ export function triageCandidates(candidates, { driftThreshold = DEFAULT_DRIFT_TH
 
 /**
  * Derive a run-level recommended action from candidate notes.
+ *
+ * H2: `overall_fit: 'usable'` (critiqueCandidate's default, and — today —
+ * its ONLY assignment anywhere in the codebase) means "nobody has assessed
+ * fit yet", not "assessed and found uniformly usable". Before this fix,
+ * recommendAction() could not tell those two states apart, so an unreviewed
+ * run always took the same `usable.length > 0` branch a genuinely-reviewed
+ * run would, picking a "preferred" candidate by brief-keyword drift count —
+ * not by looking at the image — and presenting it with the same confidence
+ * either way. `reviewed` on the returned recommendation lets the CLI (see
+ * scripts/critique.js) qualify or suppress the auto "Next:" suggestion when
+ * nothing has actually been reviewed, without changing `mode` /
+ * `preferred_candidate` semantics for callers that don't look at it —
+ * critique.json's shape stays backward-compatible (additive field only).
  */
 export function recommendAction({ candidates }) {
   if (candidates.length === 0) {
-    return { mode: 'discard_run', reason: 'No candidates in run' };
+    return { mode: 'discard_run', reason: 'No candidates in run', reviewed: false };
   }
 
+  const anyReviewed = candidates.some(c => c.reviewed === true);
+
   const strong = candidates.filter(c => c.overall_fit === 'strong');
   const usable = candidates.filter(c => c.overall_fit === 'usable');
   const offModel = candidates.filter(c => c.overall_fit === 'off-model');
@@ -317,6 +343,7 @@ export function recommendAction({ candidates }) {
       mode: 'accept_one',
       preferred_candidate: strong[0].filename,
       reason: `${strong.length} strong candidate(s) — ${strong[0].filename} is the best fit`,
+      reviewed: anyReviewed,
     };
   }
 
@@ -327,7 +354,12 @@ export function recommendAction({ candidates }) {
     return {
       mode: 'refine_from_one',
       preferred_candidate: sorted[0].filename,
-      reason: `${usable.length} usable candidate(s) — ${sorted[0].filename} has fewest drift issues`,
+      reason: anyReviewed
+        ? `${usable.length} usable candidate(s) — ${sorted[0].filename} has fewest drift issues`
+        : `${usable.length} candidate(s) at the unreviewed default "usable" — nobody has assessed fit yet. ` +
+          `${sorted[0].filename} has the fewest rule-based drift issues, which is a tiebreaker, not a ` +
+          `judgment of the image.`,
+      reviewed: anyReviewed,
     };
   }
 
@@ -336,6 +368,7 @@ export function recommendAction({ candidates }) {
     return {
       mode: 'discard_run',
       reason: 'All candidates are off-model — brief may need fundamental revision',
+      reviewed: anyReviewed,
     };
   }
 
@@ -343,6 +376,7 @@ export function recommendAction({ candidates }) {
   return {
     mode: 'rerun_broader',
     reason: 'No clear winner — try broader seed range or adjusted parameters',
+    reviewed: anyReviewed,
   };
 }
 

package/lib/eligibility.js

@@ -6,6 +6,28 @@
  */
 
 import { detectLane, detectGroup } from './config.js';
+import { warn } from './log.js';
+
+/**
+ * F5: stable, machine-readable codes — one per exclusion reason, in the
+ * exact same order/count as the human-readable `reasons` strings pushed
+ * below. Added so categorizeExclusions() can categorize by code instead
+ * of depending on prose matching (rewording a `reasons` string used to
+ * silently break categorization). Purely additive: `reasons` keeps its
+ * exact original shape and text, this is a NEW parallel field
+ * (`reason_codes` on the returned object) — any existing caller that only
+ * reads `.eligible` / `.reasons` sees no change at all.
+ */
+export const REASON_CODES = {
+  NO_JUDGMENT: 'no_judgment',
+  WRONG_STATUS: 'wrong_status',
+  NOT_CANON_BOUND: 'not_canon_bound',
+  UNMEASURED_PASS_COUNT: 'unmeasured_pass_count',
+  LOW_PASS_RATIO: 'low_pass_ratio',
+  EXCLUDED_LANE: 'excluded_lane',
+  RIGHTS_FILTER: 'rights_filter',
+  EXCLUDED_TAGS: 'excluded_tags',
+};
 
 /**
  * Evaluate a single record against a selection profile.
@@ -13,15 +35,18 @@ import { detectLane, detectGroup } from './config.js';
  * @param {Object} profile — selection profile
  * @param {Object} lanesConfig — from lanes.json
  * @param {Object} termConfig — from terminology.json
- * @returns {{ eligible: boolean, reasons: string[] }}
+ * @returns {{ eligible: boolean, reasons: string[], reason_codes: string[] }}
  */
 export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
   const reasons = [];
+  // F5: parallel array — reasonCodes[i] is the stable code for reasons[i].
+  const reasonCodes = [];
 
   // 1. Judgment required
   if (profile.require_judgment) {
     if (!record.judgment) {
       reasons.push('no judgment (record has not been reviewed)');
+      reasonCodes.push(REASON_CODES.NO_JUDGMENT);
     }
   }
 
@@ -29,6 +54,7 @@ export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
   if (profile.require_status?.length > 0 && record.judgment) {
     if (!profile.require_status.includes(record.judgment.status)) {
       reasons.push(`status "${record.judgment.status}" not in [${profile.require_status.join(', ')}]`);
+      reasonCodes.push(REASON_CODES.WRONG_STATUS);
     }
   }
 
@@ -36,14 +62,31 @@ export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
   if (profile.require_canon_bound) {
     if (!record.canon?.assertions || record.canon.assertions.length === 0) {
       reasons.push('not canon-bound (no assertions)');
+      reasonCodes.push(REASON_CODES.NOT_CANON_BOUND);
     }
   }
 
   // 4. Minimum pass ratio
   if (profile.minimum_pass_ratio != null && record.canon?.assertion_count > 0) {
-    const ratio = record.canon.pass_count / record.canon.assertion_count;
-    if (ratio < profile.minimum_pass_ratio) {
-      reasons.push(`pass ratio ${(ratio * 100).toFixed(0)}% below minimum ${(profile.minimum_pass_ratio * 100).toFixed(0)}%`);
+    const passCount = record.canon.pass_count;
+    // pass_count missing/non-numeric while assertion_count > 0 must NOT
+    // silently compute NaN: `NaN < minimum_pass_ratio` is always false, so
+    // the exclusion would never fire and a record whose pass ratio was never
+    // actually measured would be treated as having cleared a bar it never
+    // touched. Law 2 (inclusion is explainable) requires this be its own
+    // named reason, not a silent pass.
+    if (typeof passCount !== 'number' || !Number.isFinite(passCount)) {
+      reasons.push(
+        `pass_count is missing or not a number (assertion_count is ${record.canon.assertion_count}); ` +
+          `cannot verify pass ratio, so it is excluded rather than assumed passing`
+      );
+      reasonCodes.push(REASON_CODES.UNMEASURED_PASS_COUNT);
+    } else {
+      const ratio = passCount / record.canon.assertion_count;
+      if (ratio < profile.minimum_pass_ratio) {
+        reasons.push(`pass ratio ${(ratio * 100).toFixed(0)}% below minimum ${(profile.minimum_pass_ratio * 100).toFixed(0)}%`);
+        reasonCodes.push(REASON_CODES.LOW_PASS_RATIO);
+      }
     }
   }
 
@@ -53,6 +96,7 @@ export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
     const lane = detectLane(record.id, prompt, lanesConfig);
     if (profile.exclude_lanes.includes(lane)) {
       reasons.push(`lane "${lane}" is excluded`);
+      reasonCodes.push(REASON_CODES.EXCLUDED_LANE);
     }
   }
 
@@ -61,6 +105,7 @@ export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
     const rights = record.rights_status || record.training_rights_status || null;
     if (rights !== profile.rights_filter) {
       reasons.push(`rights "${rights || 'unknown'}" does not match required "${profile.rights_filter}"`);
+      reasonCodes.push(REASON_CODES.RIGHTS_FILTER);
     }
   }
 
@@ -69,41 +114,108 @@ export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
     const excluded = record.dataset_tags.filter(t => profile.exclude_tags.includes(t));
     if (excluded.length > 0) {
       reasons.push(`has excluded tags: ${excluded.join(', ')}`);
+      reasonCodes.push(REASON_CODES.EXCLUDED_TAGS);
     }
   }
 
   return {
     eligible: reasons.length === 0,
     reasons,
+    reason_codes: reasonCodes,
   };
 }
 
 /**
  * Categorize exclusion reasons for a set of records.
- * @param {Array<{record_id: string, reasons: string[]}>} excluded
- * @returns {Object} — category → count mapping
+ *
+ * F5: every reason is guaranteed to land in exactly one bucket — the
+ * mutually-exclusive if/else-if chain below now ends in a trailing `else`
+ * that increments `uncategorized` instead of silently matching nothing.
+ * Previously, rewording a `reasons` string in evaluateEligibility(), or
+ * adding a ninth reason with no matching branch, meant those exclusions
+ * were counted in NO category at all — a gap invisible in the output
+ * (the category sum would just quietly fall short of the true count).
+ *
+ * When an `excluded` entry carries `reason_codes` (evaluateEligibility's
+ * new parallel array — see REASON_CODES above), categorization uses the
+ * stable code directly instead of prose-matching `reasons`. Entries
+ * without `reason_codes` (older excluded.jsonl written before F5, or a
+ * caller that only forwards `reasons`) fall back to the original
+ * free-text matching, unchanged.
+ *
+ * @param {Array<{record_id: string, reasons: string[], reason_codes?: string[]}>} excluded
+ * @returns {Object} — category → count mapping (includes `uncategorized`)
  */
 export function categorizeExclusions(excluded) {
   const categories = {
     no_judgment: 0,
     wrong_status: 0,
     not_canon_bound: 0,
+    unmeasured_pass_count: 0,
     low_pass_ratio: 0,
     excluded_lane: 0,
     rights_filter: 0,
     excluded_tags: 0,
+    uncategorized: 0,
   };
+  const KNOWN_CODES = new Set(Object.values(REASON_CODES));
 
-  for (const { reasons } of excluded) {
-    for (const r of reasons) {
+  // F5: sample of the actual uncategorized reason strings, surfaced via
+  // warn() below so an operator sees WHAT needs a new branch/code instead
+  // of just a suspicious gap in the numbers.
+  const uncategorizedSamples = [];
+  let totalReasons = 0;
+
+  for (const { reasons, reason_codes: reasonCodes } of excluded) {
+    reasons.forEach((r, i) => {
+      totalReasons++;
+      const code = reasonCodes?.[i];
+      if (code && KNOWN_CODES.has(code)) {
+        categories[code]++;
+        return;
+      }
       if (r.includes('no judgment')) categories.no_judgment++;
       else if (r.includes('status')) categories.wrong_status++;
       else if (r.includes('canon-bound')) categories.not_canon_bound++;
+      // Order matters: "pass_count missing" must be checked before the
+      // generic "pass ratio" match below, since an unmeasured pass_count
+      // is a distinct exclusion reason from a measured-and-low ratio (Law 2
+      // — inclusion must be explainable, so "never measured" cannot be
+      // folded into "measured and failed").
+      else if (r.includes('pass_count')) categories.unmeasured_pass_count++;
       else if (r.includes('pass ratio')) categories.low_pass_ratio++;
       else if (r.includes('lane')) categories.excluded_lane++;
       else if (r.includes('rights')) categories.rights_filter++;
       else if (r.includes('tags')) categories.excluded_tags++;
-    }
+      else {
+        categories.uncategorized++;
+        if (uncategorizedSamples.length < 10) uncategorizedSamples.push(r);
+      }
+    });
+  }
+
+  if (categories.uncategorized > 0) {
+    const shown = uncategorizedSamples.slice(0, 5);
+    warn(
+      `eligibility: ${categories.uncategorized} exclusion reason(s) matched no known category ` +
+      `(first ${shown.length}: ${shown.join(' | ')})` +
+      (uncategorizedSamples.length > shown.length ? ', ...' : '') +
+      ' — add a matching reason_code/branch, or these will keep landing in "uncategorized".',
+    );
+  }
+
+  // F5: defensive tripwire, not a normal-path check — the if/else-if chain
+  // above is mutually exclusive and the trailing else always catches
+  // anything unmatched, so this sum is mathematically guaranteed to equal
+  // totalReasons for any input. A mismatch would mean a future edit broke
+  // that mutual-exclusivity guarantee (e.g. turned an `else if` into a
+  // bare `if`, allowing one reason to double-count).
+  const categorizedSum = Object.values(categories).reduce((a, b) => a + b, 0);
+  if (categorizedSum !== totalReasons) {
+    warn(
+      `eligibility: categorization invariant violated — categorized ${categorizedSum} reason(s) but ` +
+      `${totalReasons} were processed. This indicates a bug in categorizeExclusions itself.`,
+    );
   }
 
   return categories;

package/lib/eval-pack.js

@@ -10,14 +10,13 @@
  * 4. Subject continuity — same-subject records for identity testing
  */
 
-import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
+import { readdir, readFile, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { existsSync } from 'node:fs';
 import { randomBytes } from 'node:crypto';
 import { loadProjectConfig, detectLane, detectGroup } from './config.js';
 import { loadRecord } from './records.js';
-import { computeConfigFingerprint, SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
-import { inputError } from './errors.js';
+import { computeConfigFingerprint, SCHEMA_VERSION, checkManifestVersion, claimIdDir } from './snapshot.js';
 
 /**
  * Generate an eval-pack ID: eval-YYYYMMDD-HHMMSS-XXXX
@@ -73,17 +72,18 @@ export async function buildEvalPack(projectRoot, options = {}) {
   const subjectContinuity = buildSubjectContinuity(records, maxContinuity);
 
   // Write eval pack
-  const evalId = generateEvalId();
-  const evalDir = join(projectRoot, 'eval-packs', evalId);
-  // D-005: refuse to silently overwrite on ID collision.
-  if (existsSync(evalDir)) {
-    throw inputError(
-      'EVAL_PACK_ID_COLLISION',
-      `Eval pack directory already exists: ${evalDir}`,
-      'Retry to generate a fresh eval pack ID.',
-    );
-  }
-  await mkdir(evalDir, { recursive: true });
+  // SDL-M10: atomically claim the eval-pack id/dir — closes the
+  // check-then-act race the old existsSync+recursive-mkdir pattern had
+  // (D-005's intent, now actually race-proof; see claimIdDir in snapshot.js).
+  const { id: evalId, dir: evalDir } = await claimIdDir(
+    join(projectRoot, 'eval-packs'),
+    generateEvalId,
+    {
+      code: 'EVAL_PACK_ID_COLLISION',
+      message: 'Could not claim a unique eval-pack ID — every candidate collided with an existing eval-pack directory.',
+      hint: 'Retry; if this persists, another sdlab process may be creating eval packs in a tight loop.',
+    },
+  );
 
   // DB-014: compute `records_available` so the manifest reports sampled vs.
   // available. Operators can see when they're leaving records on the table.

package/lib/eval-runs.js

@@ -14,6 +14,7 @@ import { loadTrainingManifest } from './training-manifests.js';
 import { loadEvalPack } from './eval-pack.js';
 import { loadProjectConfig, detectLane, detectGroup } from './config.js';
 import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
+import { inputError } from './errors.js';
 
 // DB-008: cap sample record_ids per failure bucket so scorecards stay small
 // but the operator has enough detail to start investigation.
@@ -91,6 +92,18 @@ export async function scoreEvalRun(projectRoot, evalRunId, outputsPath) {
 
   // Load eval pack tasks
   const evalPackDir = join(projectRoot, 'eval-packs', runManifest.eval_pack_id);
+  // F2: createEvalRun only ever stores the eval_pack_id string — it never
+  // re-verifies at scoring time that the pack directory still exists. If
+  // the pack was deleted (or moved) between run creation and scoring, this
+  // used to fall straight through to the all-four-skip / vacuous-pass path
+  // below. Fail loudly here with the exact missing path instead.
+  if (!existsSync(evalPackDir)) {
+    throw inputError(
+      'EVAL_PACK_DIR_MISSING',
+      `Eval pack directory not found: ${evalPackDir} (eval run "${evalRunId}" references eval_pack_id "${runManifest.eval_pack_id}").`,
+      'The eval pack this run was created against may have been deleted or moved. Rebuild the eval pack, or create a fresh eval run against one that still exists.',
+    );
+  }
   const taskFiles = {
     'lane-coverage': 'lane-coverage.jsonl',
     'forbidden-drift': 'forbidden-drift.jsonl',
@@ -102,10 +115,18 @@ export async function scoreEvalRun(projectRoot, evalRunId, outputsPath) {
 
   // Score each task type
   const scores = {};
+  // F2: count task files that actually resolved. Object.values({}).every(...)
+  // on an EMPTY scores object is vacuously true in JS — with zero task files
+  // found, `scores` stays `{}` and the verdict below silently computed
+  // overall_verdict:'pass', overall_score:0.000 for a model that was never
+  // evaluated against anything. resolvedTaskFiles turns that into a named,
+  // structured failure instead of a green rubber stamp.
+  let resolvedTaskFiles = 0;
 
   for (const [taskName, fileName] of Object.entries(taskFiles)) {
     const taskPath = join(evalPackDir, fileName);
     if (!existsSync(taskPath)) continue;
+    resolvedTaskFiles++;
 
     const taskEntries = (await readFile(taskPath, 'utf-8'))
       .trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
@@ -121,6 +142,18 @@ export async function scoreEvalRun(projectRoot, evalRunId, outputsPath) {
     }
   }
 
+  // F2: belt-and-suspenders — even with the directory-existence guard above,
+  // a pack directory that exists but has had every individual task file
+  // deleted out from under it must hit the same guard, not the vacuous
+  // Object.values({}).every(...) === true path.
+  if (resolvedTaskFiles === 0) {
+    throw inputError(
+      'EVAL_PACK_NO_TASK_FILES',
+      `Eval pack "${runManifest.eval_pack_id}" at ${evalPackDir} has no resolvable task files — nothing was scored for eval run "${evalRunId}".`,
+      `Expected at least one of: ${Object.values(taskFiles).join(', ')}. Verify the eval pack was built correctly and its directory was not partially deleted.`,
+    );
+  }
+
   // Compute overall verdict
   const allPassed = Object.values(scores).every(s => s.passed);
   const overallScore = Object.values(scores).reduce((sum, s) => sum + (s.score || 0), 0) /