@mcptoolshop/style-dataset-lab 3.3.0 → 3.4.0

package/lib/split.js

@@ -8,14 +8,23 @@
  * 4. Every split produces an audit trail proving no leakage
  */
 
-import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises';
+import { readFile, writeFile, readdir } from 'node:fs/promises';
 import { join } from 'node:path';
 import { existsSync } from 'node:fs';
 import { createHash, randomBytes } from 'node:crypto';
 import { loadProjectConfig, loadSplitProfile, detectLane } from './config.js';
-import { loadSnapshot, loadSnapshotIncluded, SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
+import { loadSnapshot, loadSnapshotIncluded, SCHEMA_VERSION, checkManifestVersion, claimIdDir } from './snapshot.js';
 import { loadRecord } from './records.js';
-import { inputError } from './errors.js';
+import { warn } from './log.js';
+
+/**
+ * SDL-C2: share of subject families (0–1) resolved via the id-stripping
+ * fallback above which the guess is considered "meaningful" enough that
+ * lib/card.js must withhold its "None (verified)" leakage claim. Chosen
+ * conservatively low — even a modest guessed share means confidence rests
+ * on an unverified naming heuristic, not authored identity.
+ */
+export const GUESSED_FAMILY_MEANINGFUL_SHARE = 0.1;
 
 /**
  * Mulberry32 — deterministic 32-bit PRNG.
@@ -54,35 +63,84 @@ function generateSplitId() {
   return `split-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
 }
 
+/**
+ * SDL-C2: normalize a record ID into a canonical subject stem.
+ *
+ * Used by the id-stripping fallback (resolveSubjectFamily case 3) and by
+ * the independent leakage cross-check (auditStemCollisions). The naive
+ * `.replace(/_[vs]\d+$/, '')` regex it replaces had two gaps that let one
+ * real subject split into two families:
+ *
+ *   - case sensitivity: "fooV2" and "foo_V2" normalized to two different
+ *     stems instead of one.
+ *   - suffix-only matching: real id schemes embed the version/seed marker
+ *     as an INFIX token on one segment (`styleset_p04v2_crane_tower`), not
+ *     only as a trailing suffix (`styleset_p04_crane_tower_v1`). The old
+ *     regex only ever matched the latter shape.
+ *
+ * Approach: lowercase, split on `_`, strip a trailing `v<digits>` /
+ * `s<digits>` marker from EACH segment individually (not just the whole
+ * id), drop any segment that normalizes to empty (a segment that WAS only
+ * the marker), and rejoin. Both example ids above reduce to
+ * "styleset_p04_crane_tower".
+ */
+export function normalizeIdStem(id) {
+  const segments = String(id).toLowerCase().split('_');
+  const cleaned = segments
+    .map(seg => seg.replace(/[vs]\d+$/, ''))
+    .filter(seg => seg.length > 0);
+  // Every segment stripped to nothing (pathological all-marker id) — fall
+  // back to the lowercased original rather than collapsing to ''.
+  return cleaned.length > 0 ? cleaned.join('_') : String(id).toLowerCase();
+}
+
 /**
  * Determine the subject family for a record.
  *
  * Priority:
  * 1. identity.subject_name (named characters like renna_vasik)
  * 2. lineage.derived_from_record_id (walk chain to root ancestor)
- * 3. Strip _v\d+$, _s\d+$ suffixes from record ID
+ * 3. Strip version/seed markers from record ID (normalizeIdStem)
+ *
+ * SDL-C2: case 3 is a GUESS, not an authored fact — on real data (every
+ * salt-road record has zero identity.subject_name) it fires for 100% of
+ * records. Callers must not treat a guessed resolution as equivalent in
+ * confidence to an authored one; the `guessed` flag lets them track and
+ * surface that distinction instead of guessing silently.
+ *
+ * @returns {{family: string, guessed: boolean}}
  */
 function resolveSubjectFamily(record) {
   // 1. Named subject
   if (record.identity?.subject_name) {
-    return record.identity.subject_name;
+    return { family: record.identity.subject_name, guessed: false };
   }
 
-  // 2. Lineage chain — use root ancestor ID as family
+  // 2. Lineage chain — use root ancestor ID as family. Authored data (the
+  // pipeline recorded this derivation), not a guess.
   if (record.lineage?.derived_from_record_id) {
     // We return the ancestor ID; the caller's ancestor map handles chain walking
-    return record.lineage.derived_from_record_id;
+    return { family: record.lineage.derived_from_record_id, guessed: false };
   }
 
-  // 3. Strip version/seed suffixes from record ID
-  return record.id.replace(/_[vs]\d+$/, '');
+  // 3. Strip version/seed markers from record ID — case-insensitive,
+  // infix-aware. Nothing authored says these records share a subject;
+  // we are inferring it from a naming convention.
+  return { family: normalizeIdStem(record.id), guessed: true };
 }
 
 /**
  * Walk lineage chains to find root ancestors.
- * Returns a map: recordId → rootAncestorFamily
+ * Returns a map: recordId → {family, guessed}
+ *
+ * F7: exported (previously module-private) so its O(n) behavior can be
+ * measured directly in tests without paying for a full
+ * createSnapshot+createSplit round trip's file I/O on every record —
+ * buildAncestorMap itself is a pure function of `records` (no I/O, no
+ * side effects), so this is a safe, low-risk addition to the module's
+ * export surface purely for testability.
  */
-function buildAncestorMap(records) {
+export function buildAncestorMap(records) {
   const parentOf = new Map();
   const idToRecord = new Map();
 
@@ -93,13 +151,85 @@ function buildAncestorMap(records) {
     }
   }
 
-  // Walk each chain to root (with cycle protection)
-  function findRoot(id, visited = new Set()) {
-    if (visited.has(id)) return id; // cycle — stop
-    visited.add(id);
-    const parent = parentOf.get(id);
-    if (!parent) return id;
-    return findRoot(parent, visited);
+  // F7: memoize resolved roots into a map shared across every findRoot()
+  // call below, as the walk proceeds. Before this, findRoot() got a FRESH
+  // per-call `visited` Set with no memoization shared across calls — for a
+  // linear derivation chain of length k (this pipeline's wave-over-wave
+  // iteration model produces exactly that shape: record 2 derived from
+  // record 1, record 3 derived from record 2, ...), the outer loop below
+  // calls findRoot() once per record, and EACH call re-walked an
+  // ever-longer prefix of the same chain from scratch — resolving all k
+  // records cost ~k²/2 lookups instead of k.
+  //
+  // rootCache maps id -> its fully-resolved root id. Every id visited
+  // during a walk gets pinned to the same final root once found (path
+  // compression): the first call to walk the whole chain still costs O(k),
+  // but every subsequent call for any id already visited on that walk (or
+  // whose walk crosses a previously-cached id partway through) is an O(1)
+  // cache hit instead of a re-walk.
+  //
+  // Written as an explicit loop rather than recursion: profiling this fix
+  // against a synthetic 20,000-record linear chain found the ORIGINAL
+  // recursive walk (recursion depth == chain length, once per call) blows
+  // V8's call stack ("RangeError: Maximum call stack size exceeded") well
+  // before memoization would even matter — a chain long enough to make the
+  // O(n^2) cost bite is also long enough to crash outright. Fixing the
+  // algorithmic complexity without also fixing the recursion-depth ceiling
+  // would leave a strictly worse failure mode (a crash) sitting right next
+  // to the fix, so both are closed together here.
+  //
+  // This does not change results for the acyclic chains this field
+  // actually models — for a pathological cycle (which
+  // lineage.derived_from_record_id should never legitimately contain) it
+  // still terminates safely and deterministically; it just picks whichever
+  // member of the cycle the outer loop reaches first as the shared "root"
+  // for the whole cycle, rather than each cyclic member resolving to
+  // itself independently the way a fully-independent (non-memoized) walk
+  // would. Cycle protection itself (a node revisited within the SAME walk)
+  // is otherwise semantically unchanged from the original.
+  const rootCache = new Map();
+
+  function findRoot(startId) {
+    const cached = rootCache.get(startId);
+    if (cached !== undefined) return cached;
+
+    // Walk forward from startId, collecting the path as we go, until we
+    // hit: (a) a node with an already-cached root (reuse it — this is the
+    // O(1) fast path for everything after the first walk through a given
+    // chain), (b) a node with no parent (it IS the root), or (c) a node
+    // already seen ON THIS WALK (a cycle — stop there, matching the
+    // original recursive version's "cycle stops at the repeated node"
+    // behavior exactly).
+    const path = [];
+    const onPath = new Set();
+    let current = startId;
+    let root;
+
+    while (true) {
+      const cachedRoot = rootCache.get(current);
+      if (cachedRoot !== undefined) {
+        root = cachedRoot;
+        break;
+      }
+      if (onPath.has(current)) {
+        root = current; // cycle — stop at the repeated node
+        break;
+      }
+      onPath.add(current);
+      path.push(current);
+      const parent = parentOf.get(current);
+      if (!parent) {
+        root = current;
+        break;
+      }
+      current = parent;
+    }
+
+    // Path compression: pin every node visited on THIS walk directly to
+    // the resolved root, so any future call for any of them — from this
+    // top-level findRoot() call or a later one — is an O(1) lookup.
+    for (const id of path) rootCache.set(id, root);
+    return root;
   }
 
   const ancestorMap = new Map();
@@ -111,8 +241,10 @@ function buildAncestorMap(records) {
       if (rootRecord) {
         ancestorMap.set(record.id, resolveSubjectFamily(rootRecord));
       } else {
-        // Root not in dataset — strip suffixes from root ID
-        ancestorMap.set(record.id, rootId.replace(/_[vs]\d+$/, ''));
+        // Root not in dataset — strip markers from root ID (SDL-C2: same
+        // case-insensitive, infix-aware normalization as case 3 above;
+        // this is a guess too).
+        ancestorMap.set(record.id, { family: normalizeIdStem(rootId), guessed: true });
       }
     }
   }
@@ -201,28 +333,65 @@ export async function createSplit(projectRoot, snapshotId, profile, options = {}
   const snapshot = await loadSnapshot(projectRoot, snapshotId);
   const includedEntries = await loadSnapshotIncluded(projectRoot, snapshotId);
   const recordsDir = join(projectRoot, 'records');
+  // Declared here (rather than where the pre-existing lane/family warnings
+  // are pushed further below) so the F3 missing-records warning shares the
+  // same array — and therefore the same manifest.json / audit.json
+  // `warnings` field — as every other warning this function produces.
+  const warnings = [];
 
-  // Load full records for included entries
+  // Load full records for included entries.
+  //
+  // F3: a record the (frozen, Law-1) snapshot included whose file was
+  // later deleted from records/ must never silently drop out of the split
+  // with no counter, no warning, no id recorded. loadRecord() returns null
+  // on ENOENT (lib/records.js); previously `if (record) records.push(record)`
+  // was the ENTIRE handling for that case — a missing record just shrank
+  // `records` by one, invisibly. WARN + ledger here rather than hard-fail:
+  // a split is an intermediate artifact, not itself the Law-4
+  // ("exports are reproducible") contract-bearing one — export.js and
+  // training-packages.js are, and hard-fail on the identical situation
+  // (see their EXPORT_MISSING_RECORDS / TRAINING_PACKAGE_MISSING_RECORDS
+  // guards) rather than warning.
   const records = [];
+  const missingRecords = [];
   for (const entry of includedEntries) {
     const record = await loadRecord(recordsDir, entry.record_id);
-    if (record) records.push(record);
+    if (record) {
+      records.push(record);
+    } else {
+      missingRecords.push({ record_id: entry.record_id, stage: 'split' });
+    }
+  }
+  if (missingRecords.length > 0) {
+    const sample = missingRecords.slice(0, 10).map(m => m.record_id);
+    const msg =
+      `split: ${missingRecords.length} of ${includedEntries.length} record(s) referenced by snapshot ` +
+      `"${snapshotId}" could not be loaded from ${recordsDir} — this split is built from the ` +
+      `${records.length} record(s) actually found` +
+      (missingRecords.length > 10 ? ` (first 10: ${sample.join(', ')}, ...)` : ` (${sample.join(', ')})`);
+    warn(msg);
+    warnings.push(msg);
   }
 
   // Build ancestor map for lineage chain resolution
   const ancestorMap = buildAncestorMap(records);
 
-  // Assign each record to a subject family
+  // Assign each record to a subject family. Resolution is cached per
+  // record-id (recordResolution) so the second assignment pass below reuses
+  // the exact same {family, guessed} decision rather than re-deriving it —
+  // the two passes must never be able to disagree.
   const familyMap = new Map(); // family → [record]
+  const familyGuessed = new Map(); // family → true if ANY member arrived via id-stripping fallback
+  const recordResolution = new Map(); // record.id → {family, guessed}
   for (const record of records) {
-    let family;
-    if (ancestorMap.has(record.id)) {
-      family = ancestorMap.get(record.id);
-    } else {
-      family = resolveSubjectFamily(record);
-    }
+    const resolution = ancestorMap.has(record.id)
+      ? ancestorMap.get(record.id)
+      : resolveSubjectFamily(record);
+    recordResolution.set(record.id, resolution);
+    const { family, guessed } = resolution;
     if (!familyMap.has(family)) familyMap.set(family, []);
     familyMap.get(family).push(record);
+    if (guessed) familyGuessed.set(family, true);
   }
 
   // D-002: Deterministic primary lane per family = majority vote; ties
@@ -267,7 +436,11 @@ export async function createSplit(projectRoot, snapshotId, profile, options = {}
   const trainFamilies = new Set();
   const valFamilies = new Set();
   const testFamilies = new Set();
-  const warnings = [];
+  // F3: `warnings` is declared once, up near the top of createSplit (ahead
+  // of the missing-records check), and reused here — NOT redeclared — so
+  // every warning this function produces (missing records, lane-family
+  // count, guessed-family share, stem-collision) lands in one array and
+  // therefore one manifest.json / audit.json `warnings` field.
 
   const sortedLanes = [...laneGroups.keys()].sort();
   for (const lane of sortedLanes) {
@@ -312,12 +485,10 @@ export async function createSplit(projectRoot, snapshotId, profile, options = {}
   const testRecords = [];
 
   for (const record of records) {
-    let family;
-    if (ancestorMap.has(record.id)) {
-      family = ancestorMap.get(record.id);
-    } else {
-      family = resolveSubjectFamily(record);
-    }
+    // Reuse the exact resolution cached during the family-map build pass
+    // above — recomputing here would risk the two passes silently
+    // disagreeing (D-003's whole point is to catch that class of bug).
+    const { family } = recordResolution.get(record.id);
 
     if (trainFamilies.has(family)) trainRecords.push(record.id);
     else if (valFamilies.has(family)) valRecords.push(record.id);
@@ -337,29 +508,90 @@ export async function createSplit(projectRoot, snapshotId, profile, options = {}
   valRecords.sort();
   testRecords.sort();
 
-  // Leakage audit
+  // Leakage audit — checks whether the assignment's OWN family map spans
+  // partitions. By construction this can never fire (the assignment loop
+  // groups by family before splitting) unless there's a bug in the
+  // assignment step itself; it cannot see a bad family RESOLUTION (SDL-C2).
   const leakageIssues = auditLeakage(familyMap, trainRecords, valRecords, testRecords);
 
+  // SDL-C2b: independent cross-check. Never reads familyMap/ancestorMap —
+  // re-derives a canonical id stem per record from scratch and flags any
+  // case where records sharing a stem (i.e. very likely one real subject)
+  // ended up placed in more than one partition. This is the only check
+  // that can catch SDL-C2-shaped bugs, including future regressions of
+  // resolveSubjectFamily/normalizeIdStem that the leakage_check above
+  // would stay blind to.
+  const stemCollisionIssues = auditStemCollisions(records, trainRecords, valRecords, testRecords);
+
+  // SDL-C2: surface how much of the split's leakage-freedom claim rests on
+  // guessed (id-stripped) family resolution vs. authored subject_name /
+  // lineage. On real project data (salt-road: 99/99 records, zero
+  // identity.subject_name) this is 100% — silent guessing is how SDL-C2
+  // stayed invisible. Never silent again: counted here, warned on below,
+  // and consumed by lib/card.js to gate the "None (verified)" claim.
+  let guessedFamilies = 0;
+  for (const family of familyMap.keys()) {
+    if (familyGuessed.get(family)) guessedFamilies++;
+  }
+  let guessedRecords = 0;
+  for (const resolution of recordResolution.values()) {
+    if (resolution.guessed) guessedRecords++;
+  }
+  const totalFamilies = familyMap.size;
+  const guessedFamilyRatio = totalFamilies > 0 ? guessedFamilies / totalFamilies : 0;
+  // Conservatively low: even a modest guessed share means the leakage-free
+  // claim is resting on an unverified naming-convention heuristic rather
+  // than authored identity, which is worth a caveat rather than silence.
+  const meaningfulGuessShare = guessedFamilyRatio > GUESSED_FAMILY_MEANINGFUL_SHARE;
+  const familyResolution = {
+    total_families: totalFamilies,
+    guessed_families: guessedFamilies,
+    guessed_family_ratio: +guessedFamilyRatio.toFixed(3),
+    total_records: records.length,
+    guessed_records: guessedRecords,
+    guessed_record_ratio: records.length > 0 ? +(guessedRecords / records.length).toFixed(3) : 0,
+    meaningful_guess_share_threshold: GUESSED_FAMILY_MEANINGFUL_SHARE,
+    meaningful_guess_share: meaningfulGuessShare,
+  };
+  if (meaningfulGuessShare) {
+    warnings.push(
+      `${guessedFamilies} of ${totalFamilies} subject families ` +
+      `(${(guessedFamilyRatio * 100).toFixed(1)}%) were inferred by stripping version/seed ` +
+      `markers from record IDs — no identity.subject_name or lineage authored. ` +
+      `Verify subject grouping manually; the leakage-free claim on this split is unverified for these families.`,
+    );
+  }
+  if (stemCollisionIssues.length > 0) {
+    warnings.push(
+      `subject-stem cross-check found ${stemCollisionIssues.length} case(s) where records sharing ` +
+      `a canonical id stem were resolved to different families — see audit.json stem_collision_check.`,
+    );
+  }
+
   // Lane balance audit (DB-005: includes deviation from profile ratios)
   const laneBalance = auditLaneBalance(
     records, trainRecords, valRecords, testRecords, config.lanes, profile
   );
 
   // Write output (skip if dry run)
-  const splitId = generateSplitId();
   if (options.dryRun) {
+    // Dry run never touches disk — nothing to atomically claim; a fresh
+    // preview id is enough to characterize the shape of the result.
+    const splitId = generateSplitId();
     return { splitId, train: trainRecords.length, val: valRecords.length, test: testRecords.length };
   }
-  const splitDir = join(projectRoot, 'splits', splitId);
-  // D-005: refuse to silently overwrite on ID collision.
-  if (existsSync(splitDir)) {
-    throw inputError(
-      'SPLIT_ID_COLLISION',
-      `Split directory already exists: ${splitDir}`,
-      'Retry to generate a fresh split ID.',
-    );
-  }
-  await mkdir(splitDir, { recursive: true });
+  // SDL-M10: atomically claim the split id/dir — closes the check-then-act
+  // race the old existsSync+recursive-mkdir pattern had (D-005's intent,
+  // now actually race-proof; see claimIdDir in snapshot.js).
+  const { id: splitId, dir: splitDir } = await claimIdDir(
+    join(projectRoot, 'splits'),
+    generateSplitId,
+    {
+      code: 'SPLIT_ID_COLLISION',
+      message: 'Could not claim a unique split ID — every candidate collided with an existing split directory.',
+      hint: 'Retry; if this persists, another sdlab process may be creating splits in a tight loop.',
+    },
+  );
 
   const manifest = {
     // DB-001 / DB-006: stamp manifest schema version
@@ -376,7 +608,16 @@ export async function createSplit(projectRoot, snapshotId, profile, options = {}
       train: trainRecords.length,
       val: valRecords.length,
       test: testRecords.length,
+      // F3: how many snapshot-included records could not be loaded from
+      // records/ at split time. 0 in the common case; see missing_records
+      // below for the actual ids.
+      missing_records: missingRecords.length,
     },
+    // F3: ledger of every snapshot-included record_id that failed to load
+    // (mirrors the images_failed pattern export.js established one layer
+    // down at export time) — record_id + which stage recorded the miss, so
+    // an operator can trace it without re-diffing snapshot vs. records/.
+    missing_records: missingRecords,
     warnings,
   };
 
@@ -386,6 +627,18 @@ export async function createSplit(projectRoot, snapshotId, profile, options = {}
       passed: leakageIssues.length === 0,
       issues: leakageIssues,
     },
+    // SDL-C2b: distinct from leakage_check above — independently re-derived,
+    // does not reuse familyMap, and is the only check that can detect a
+    // SDL-C2-shaped family-resolution bug. Read this key (not
+    // leakage_check) to know whether the independent cross-check fired.
+    stem_collision_check: {
+      passed: stemCollisionIssues.length === 0,
+      issues: stemCollisionIssues,
+    },
+    // SDL-C2: how much of family resolution rests on the id-stripping
+    // guess vs. authored identity/lineage. lib/card.js reads this to
+    // decide whether "None (verified)" is honest to print.
+    family_resolution: familyResolution,
     lane_balance: laneBalance,
     family_count: familyMap.size,
     families_per_split: {
@@ -436,6 +689,69 @@ function auditLeakage(familyMap, trainIds, valIds, testIds) {
   return issues;
 }
 
+/**
+ * SDL-C2b: independent leakage cross-check.
+ *
+ * auditLeakage() above can only ever confirm that the assignment loop's OWN
+ * familyMap is internally consistent — by construction every record with a
+ * given family key was grouped and placed together, so that check can never
+ * observe a case where family RESOLUTION itself was wrong (two records that
+ * are really the same subject but got resolved to two different family
+ * keys — SDL-C2). This function never reads familyMap or ancestorMap.
+ *
+ * It re-derives a canonical id stem for every record directly from the
+ * record's own id (normalizeIdStem — the same normalization the fixed
+ * resolveSubjectFamily now uses), independent of whatever family label the
+ * assignment computed. It then asks a question the assignment's own audit
+ * structurally cannot ask: did records that share a canonical stem (i.e.
+ * are very likely one real subject) end up placed in more than one
+ * partition? A "yes" is real, unambiguous leakage regardless of what family
+ * label the assignment used internally — including if a future regression
+ * reintroduces a SDL-C2-shaped bug in family resolution.
+ *
+ * Returns an array of issues (empty = clean). Each issue also carries
+ * `assigned_families` — the (possibly multiple) family labels the members
+ * were actually resolved to, purely as an operator diagnostic; it is never
+ * consulted for the pass/fail decision above.
+ */
+function auditStemCollisions(records, trainIds, valIds, testIds) {
+  const trainSet = new Set(trainIds);
+  const valSet = new Set(valIds);
+  const testSet = new Set(testIds);
+
+  const byStem = new Map(); // canonical stem → [record]
+  for (const record of records) {
+    const stem = normalizeIdStem(record.id);
+    if (!byStem.has(stem)) byStem.set(stem, []);
+    byStem.get(stem).push(record);
+  }
+
+  const issues = [];
+  for (const [stem, recs] of byStem) {
+    if (recs.length < 2) continue; // nothing to collide with
+
+    const partitions = new Set();
+    for (const r of recs) {
+      if (trainSet.has(r.id)) partitions.add('train');
+      if (valSet.has(r.id)) partitions.add('val');
+      if (testSet.has(r.id)) partitions.add('test');
+    }
+    if (partitions.size <= 1) continue; // same stem, same partition — fine
+
+    const assignedFamilies = [...new Set(recs.map(r => resolveSubjectFamily(r).family))].sort();
+
+    issues.push({
+      canonical_stem: stem,
+      record_ids: recs.map(r => r.id).sort(),
+      spans_partitions: [...partitions].sort(),
+      assigned_families: assignedFamilies,
+    });
+  }
+
+  issues.sort((a, b) => a.canonical_stem.localeCompare(b.canonical_stem));
+  return issues;
+}
+
 /**
  * Audit lane balance across splits.
  *