@mcptoolshop/style-dataset-lab 3.2.0 → 3.4.0

package/lib/snapshot.js

@@ -19,8 +19,14 @@ import { warn } from './log.js';
  * snapshot/split/export/eval-pack/eval-run/training-manifest/training-package/
  * impl-pack stamps this version; readers log a WARNING on version mismatch
  * (best-effort compatibility).
+ *
+ * 2.3.0 (v3.3.0): PIN_PER_STEP — the ComfyUI run manifest now also stamps this
+ * version and carries a `pinning` block (comfy_workflow_sha + model/LoRA content
+ * hashes + seed_policy; see lib/run-manifest.js). The block is additive and
+ * optional, so a legacy manifest with no schema_version / no pinning still loads
+ * (checkManifestVersion only warns; it never throws).
  */
-export const SCHEMA_VERSION = '2.2.0';
+export const SCHEMA_VERSION = '2.3.0';
 
 /**
  * Warn if a loaded manifest's schema_version does not match the reader.
@@ -35,6 +41,96 @@ export function checkManifestVersion(manifest, kind) {
   }
 }
 
+/**
+ * SDL-M10: atomically claim a fresh, unused SUBDIRECTORY of `parentDir`.
+ *
+ * The pattern this replaces — `existsSync(dir)` then throw-if-true,
+ * followed later by `mkdir(dir, {recursive:true})` — is check-then-act: a
+ * second concurrent process can create the same directory in the window
+ * between the check and the mkdir, and recursive mkdir never throws on an
+ * already-existing directory, so the race is silently invisible (two
+ * builders interleave writes into one snapshot/split/export/etc. dir —
+ * Law 1 exposure: "snapshots frozen").
+ *
+ * Mirrors lib/selections.js's claimSelectionId: a non-recursive `mkdir`
+ * is atomic at the filesystem level (POSIX and Windows both guarantee
+ * this) — it either creates the directory and returns, or fails with
+ * EEXIST if something else claimed that exact name first. On EEXIST we
+ * regenerate a fresh id and retry; for the timestamp+random ids used
+ * throughout this module, two consecutive real collisions is already
+ * vanishingly unlikely.
+ *
+ * Creating `parentDir` itself (e.g. "snapshots/") is NOT part of the race
+ * being closed here — two processes racing to create the same shared
+ * parent with {recursive:true} is harmless (both succeed; no caller's
+ * data lives directly in the parent, only under their own distinctly
+ * named leaf directory).
+ *
+ * @param {string} parentDir — parent directory, created recursively if missing
+ * @param {() => string} generateId — produces a candidate leaf id; called
+ *   again on every retry so concurrent claimants converge on different ids
+ * @param {{code: string, message: string, hint: string, maxAttempts?: number}} onExhausted
+ *   — error identity thrown if every attempt collides (kept caller-specific
+ *   so existing codes like SNAPSHOT_ID_COLLISION stay externally stable)
+ * @returns {Promise<{id: string, dir: string}>}
+ */
+export async function claimIdDir(parentDir, generateId, onExhausted) {
+  const maxAttempts = onExhausted.maxAttempts ?? 25;
+  if (!existsSync(parentDir)) {
+    await mkdir(parentDir, { recursive: true });
+  }
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const id = generateId();
+    const dir = join(parentDir, id);
+    try {
+      await mkdir(dir); // non-recursive — throws EEXIST if already claimed
+      return { id, dir };
+    } catch (err) {
+      if (err.code === 'EEXIST') continue;
+      throw err;
+    }
+  }
+  throw inputError(onExhausted.code, onExhausted.message, onExhausted.hint);
+}
+
+/**
+ * SDL-M10: atomically claim a fresh, unused FILE `<parentDir>/<id>.json`.
+ *
+ * Same TOCTOU problem as claimIdDir, but for the one artifact in this
+ * domain stored as a flat file rather than a directory (training
+ * manifests: `training/manifests/<id>.json`). The atomic primitive here
+ * is `writeFile(..., {flag: 'wx'})` — exclusive create, which fails with
+ * EEXIST rather than silently overwriting — so claim-and-write happen in
+ * one step instead of racing a separate existsSync check against it.
+ *
+ * @param {string} parentDir — parent directory, created recursively if missing
+ * @param {() => string} generateId — produces a candidate id; called again
+ *   on every retry
+ * @param {(id: string) => string} buildContent — renders the file content
+ *   for a given candidate id (called fresh per attempt since the content
+ *   embeds the id)
+ * @param {{code: string, message: string, hint: string, maxAttempts?: number}} onExhausted
+ * @returns {Promise<{id: string, filePath: string}>}
+ */
+export async function claimIdFile(parentDir, generateId, buildContent, onExhausted) {
+  const maxAttempts = onExhausted.maxAttempts ?? 25;
+  if (!existsSync(parentDir)) {
+    await mkdir(parentDir, { recursive: true });
+  }
+  for (let attempt = 0; attempt < maxAttempts; attempt++) {
+    const id = generateId();
+    const filePath = join(parentDir, `${id}.json`);
+    try {
+      await writeFile(filePath, buildContent(id), { flag: 'wx' }); // exclusive create — atomic
+      return { id, filePath };
+    } catch (err) {
+      if (err.code === 'EEXIST') continue;
+      throw err;
+    }
+  }
+  throw inputError(onExhausted.code, onExhausted.message, onExhausted.hint);
+}
+
 /**
  * Derive which rule names were evaluated against a record for a given profile.
  * Mirrors the order of checks in evaluateEligibility() so the trace is stable.
@@ -95,18 +191,6 @@ export function computeConfigFingerprint(projectRoot) {
 export async function createSnapshot(projectRoot, profile, options = {}) {
   const config = loadProjectConfig(projectRoot);
   const recordsDir = join(projectRoot, 'records');
-  const snapshotId = generateSnapshotId();
-  const snapshotDir = join(projectRoot, 'snapshots', snapshotId);
-
-  // D-005: refuse to proceed if the generated ID has collided with an existing
-  // snapshot dir — silent overwrite would break Law 1 (snapshots are frozen).
-  if (!options.dryRun && existsSync(snapshotDir)) {
-    throw inputError(
-      'SNAPSHOT_ID_COLLISION',
-      `Snapshot directory already exists: ${snapshotDir}`,
-      'Retry to generate a fresh snapshot ID.',
-    );
-  }
 
   const configFingerprint = computeConfigFingerprint(projectRoot);
   const profileId = profile.profile_id || profile.id || 'unknown';
@@ -141,7 +225,7 @@ export async function createSnapshot(projectRoot, profile, options = {}) {
       continue;
     }
 
-    const { eligible, reasons } = evaluateEligibility(
+    const { eligible, reasons, reason_codes: reasonCodes } = evaluateEligibility(
       record, profile, config.lanes, config.terminology
     );
 
@@ -175,7 +259,13 @@ export async function createSnapshot(projectRoot, profile, options = {}) {
         }
       }
     } else {
-      excluded.push({ record_id: record.id, reasons });
+      // F5: forward the stable reason_codes alongside the human-readable
+      // reasons — purely additive on excluded.jsonl's on-disk shape (one
+      // new key), so old readers/old files that only expect
+      // {record_id, reasons} are unaffected. Lets categorizeExclusions
+      // (lib/eligibility.js) categorize this snapshot's exclusions by
+      // code instead of prose-matching `reasons`.
+      excluded.push({ record_id: record.id, reasons, reason_codes: reasonCodes });
     }
   }
 
@@ -191,9 +281,24 @@ export async function createSnapshot(projectRoot, profile, options = {}) {
 
   // Write snapshot files (skip if dry run)
   if (options.dryRun) {
+    // Dry run never touches disk, so there is nothing to atomically claim —
+    // a fresh preview id is enough to characterize the shape of the result.
+    const snapshotId = generateSnapshotId();
     return { snapshotId, included: included.length, excluded: excluded.length, unreadable: unreadable.length };
   }
-  await mkdir(snapshotDir, { recursive: true });
+
+  // SDL-M10: atomically claim the snapshot id/dir — closes the
+  // check-then-act race the old existsSync+recursive-mkdir pattern had
+  // (D-005's intent, now actually race-proof; see claimIdDir above).
+  const { id: snapshotId, dir: snapshotDir } = await claimIdDir(
+    join(projectRoot, 'snapshots'),
+    generateSnapshotId,
+    {
+      code: 'SNAPSHOT_ID_COLLISION',
+      message: 'Could not claim a unique snapshot ID — every candidate collided with an existing snapshot directory.',
+      hint: 'Retry; if this persists, another sdlab process may be creating snapshots in a tight loop.',
+    },
+  );
 
   const manifest = {
     // DB-001 / DB-006: stamp manifest schema version (reader checks on load)