@mcptoolshop/style-dataset-lab 3.0.0 → 3.1.0

package/lib/export.js

@@ -6,14 +6,17 @@
  */
 
 import { readFile, writeFile, mkdir, readdir, symlink, copyFile, stat } from 'node:fs/promises';
-import { join, relative, resolve } from 'node:path';
+import { join, relative, resolve, sep, basename, isAbsolute } from 'node:path';
 import { existsSync } from 'node:fs';
+import { platform } from 'node:os';
 import { createHash, randomBytes } from 'node:crypto';
 import { loadProjectConfig, loadExportProfile, detectLane, detectGroup } from './config.js';
-import { loadSnapshot, loadSnapshotIncluded, computeConfigFingerprint } from './snapshot.js';
+import { loadSnapshot, loadSnapshotIncluded, computeConfigFingerprint, SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
 import { loadSplit, loadSplitPartition } from './split.js';
 import { loadRecord } from './records.js';
 import { generateCard } from './card.js';
+import { inputError } from './errors.js';
+import { warn } from './log.js';
 
 /**
  * Generate an export ID: export-YYYYMMDD-HHMMSS-XXXX
@@ -54,6 +57,29 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
 
   const exportId = generateExportId();
   const exportDir = join(projectRoot, 'exports', exportId);
+
+  // D-005: existsSync guard prevents silent overwrite on ID collision
+  if (existsSync(exportDir)) {
+    throw inputError(
+      'EXPORT_ID_COLLISION',
+      `Export directory already exists: ${exportDir}`,
+      'Retry to generate a fresh export ID, or delete the existing directory if this was intentional.',
+    );
+  }
+
+  // D-001: Verify fingerprint has not drifted since snapshot creation.
+  // We inherit the snapshot's fingerprint in the export manifest; if a caller
+  // has edited project config mid-flight we surface that loudly rather than
+  // silently embedding a new fingerprint that masks reproducibility loss.
+  const currentFingerprint = computeConfigFingerprint(projectRoot);
+  if (snapshot.config_fingerprint && currentFingerprint !== snapshot.config_fingerprint) {
+    throw inputError(
+      'FINGERPRINT_DRIFT',
+      `Project config fingerprint has changed since snapshot "${snapshotId}" was created.`,
+      `Snapshot fingerprint: ${snapshot.config_fingerprint.slice(0, 16)}..., current: ${currentFingerprint.slice(0, 16)}.... Create a fresh snapshot from the updated config, or revert the config edits.`,
+    );
+  }
+
   await mkdir(exportDir, { recursive: true });
 
   // Build metadata + collect image paths
@@ -61,6 +87,8 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
   const imagePaths = [];
   const checksumEntries = [];
 
+  const projectRootResolved = resolve(projectRoot);
+
   for (const entry of included) {
     const record = await loadRecord(recordsDir, entry.record_id);
     if (!record) continue;
@@ -87,10 +115,33 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
 
     // Track image for linking
     if (profile.include_images && record.asset_path) {
+      // D-007: validate asset_path does not escape projectRoot, use
+      // platform-safe basename, and name output files by record.id
+      // (D-009) to avoid collisions on shared basenames.
+      if (isAbsolute(record.asset_path)) {
+        throw inputError(
+          'ASSET_PATH_INVALID',
+          `Record "${record.id}" has absolute asset_path: ${record.asset_path}`,
+          'asset_path must be relative to the project root.',
+        );
+      }
+      const resolvedSource = resolve(projectRootResolved, record.asset_path);
+      if (resolvedSource !== projectRootResolved &&
+          !resolvedSource.startsWith(projectRootResolved + sep)) {
+        throw inputError(
+          'ASSET_PATH_TRAVERSAL',
+          `Record "${record.id}" asset_path escapes projectRoot: ${record.asset_path}`,
+          'Reject path traversal segments (e.g. "../") in asset_path.',
+        );
+      }
+      const sourceBasename = basename(record.asset_path);
+      const extMatch = sourceBasename.match(/\.[^.]+$/);
+      const ext = extMatch ? extMatch[0] : '';
+      const safeFilename = `${record.id}${ext}`;
       imagePaths.push({
         record_id: record.id,
-        source: join(projectRoot, record.asset_path),
-        filename: record.asset_path.split('/').pop(),
+        source: resolvedSource,
+        filename: safeFilename,
       });
     }
   }
@@ -100,27 +151,70 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
   await writeFile(metadataPath, metadata.map(r => JSON.stringify(r)).join('\n') + '\n');
 
   // Images directory
+  const warnings = [];
+  // DB-002: separate accounting for expected vs. actually placed vs. failed.
+  // Previously `counts.images` reported `sources_that_existed`, which hid
+  // per-file symlink/copy failures and made partial exports look complete.
+  let imagesExpected = 0;
+  let imagesActual = 0;
+  const imagesFailed = []; // [{record_id, reason}]
   if (profile.include_images && imagePaths.length > 0) {
+    // D-008: Windows symlink semantics differ (require admin or dev mode);
+    // refuse symlink mode on Windows and require --copy for portability.
+    if (!options.copy && platform() === 'win32') {
+      throw inputError(
+        'SYMLINK_UNSUPPORTED_WINDOWS',
+        'Symlink mode is not supported on Windows export (target semantics differ and break on archive).',
+        'Re-run with --copy to produce a portable, self-contained export.',
+      );
+    }
+    if (!options.copy) {
+      warnings.push(
+        'Export uses relative symlinks — package is NOT portable across machines. ' +
+        'Use --copy for cross-machine transfer or archival.',
+      );
+    }
+
     const imagesDir = join(exportDir, 'images');
     await mkdir(imagesDir, { recursive: true });
 
-    let linked = 0;
+    imagesExpected = imagePaths.length;
     for (const img of imagePaths) {
       const dest = join(imagesDir, img.filename);
-      if (existsSync(img.source)) {
-        try {
-          if (options.copy) {
-            await copyFile(img.source, dest);
-          } else {
-            const relPath = relative(imagesDir, img.source);
-            await symlink(relPath, dest);
-          }
-          linked++;
-        } catch {
-          // Skip individual failures (e.g. duplicate filenames)
+      if (!existsSync(img.source)) {
+        // DB-002: source-missing is a distinct failure mode from link/copy error.
+        imagesFailed.push({ record_id: img.record_id, reason: 'source_missing' });
+        continue;
+      }
+      try {
+        if (options.copy) {
+          await copyFile(img.source, dest);
+        } else {
+          const relPath = relative(imagesDir, img.source);
+          await symlink(relPath, dest);
         }
+        imagesActual++;
+      } catch (err) {
+        // DB-002: capture the specific placement failure for operator triage.
+        imagesFailed.push({
+          record_id: img.record_id,
+          reason: options.copy ? 'copy_failed' : 'link_failed',
+          detail: err.message,
+        });
       }
     }
+
+    // DB-002: surface partial placement failures loudly — silent `catch {}`
+    // previously hid this and made exports appear complete when 20% of
+    // images had actually been dropped.
+    if (imagesFailed.length > 0) {
+      const sample = imagesFailed.slice(0, 10).map(f => `${f.record_id} (${f.reason})`);
+      const msg =
+        `export: ${imagesFailed.length} of ${imagesExpected} images failed to place` +
+        (imagesFailed.length > 10 ? ` — first 10: ${sample.join(', ')}, ...` : ` — ${sample.join(', ')}`);
+      warn(msg);
+      warnings.push(msg);
+    }
   }
 
   // Splits
@@ -144,34 +238,35 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
     await writeFile(join(exportDir, 'dataset-card.json'), JSON.stringify(json, null, 2) + '\n');
   }
 
-  // Checksums
-  if (profile.include_checksums) {
-    // Checksum all files in the export dir (except checksums.txt itself)
-    const allFiles = await collectFiles(exportDir);
-    for (const filePath of allFiles.sort()) {
-      const relPath = relative(exportDir, filePath);
-      if (relPath === 'checksums.txt') continue;
-      const hash = await sha256File(filePath);
-      checksumEntries.push(`SHA256 (${relPath}) = ${hash}`);
-    }
-    await writeFile(join(exportDir, 'checksums.txt'), checksumEntries.join('\n') + '\n');
-  }
-
-  // Manifest (written last — references everything)
+  // D-015: Write manifest + summary FIRST so they are covered by checksums.
+  // D-001: Inherit snapshot's config_fingerprint (we verified above it matches
+  // current state) — never recompute independently at export time.
   const manifest = {
+    // DB-001 / DB-006: stamp manifest schema version
+    schema_version: SCHEMA_VERSION,
     export_id: exportId,
     created_at: new Date().toISOString(),
     created_by: 'sdlab-export-v1',
     project: config.meta.name,
     snapshot_id: snapshotId,
     split_id: splitId,
-    config_fingerprint: computeConfigFingerprint(projectRoot),
+    config_fingerprint: snapshot.config_fingerprint,
     profile,
+    warnings,
     counts: {
       records: metadata.length,
-      images: imagePaths.filter(i => existsSync(i.source)).length,
-      checksums: checksumEntries.length,
+      // DB-002: images count is the actual placed count (not source-exists).
+      // Keep top-level `images` for backward compat with older manifest readers
+      // that indexed it as the summary number, but add structured detail.
+      images: imagesActual,
+      images_expected: imagesExpected,
+      images_actual: imagesActual,
+      images_failed: imagesFailed.length,
+      checksums: 0, // updated after checksum pass below
     },
+    // DB-002: record-level failure detail so operators can investigate
+    // without re-opening the directory to discover what's missing.
+    images_failed: imagesFailed,
   };
   await writeFile(join(exportDir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
 
@@ -188,9 +283,22 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
       test: split.counts.test,
     },
     config_fingerprint: manifest.config_fingerprint,
+    warnings,
   };
   await writeFile(join(exportDir, 'summary.json'), JSON.stringify(summary, null, 2) + '\n');
 
+  // Checksums (computed AFTER manifest+summary so they are included)
+  if (profile.include_checksums) {
+    const allFiles = await collectFiles(exportDir);
+    for (const filePath of allFiles.sort()) {
+      const relPath = relative(exportDir, filePath);
+      if (relPath === 'checksums.txt') continue;
+      const hash = await sha256File(filePath);
+      checksumEntries.push(`SHA256 (${relPath}) = ${hash}`);
+    }
+    await writeFile(join(exportDir, 'checksums.txt'), checksumEntries.join('\n') + '\n');
+  }
+
   return {
     exportId,
     recordCount: metadata.length,
@@ -255,5 +363,7 @@ export async function loadExport(projectRoot, exportId) {
   if (!existsSync(path)) {
     throw new Error(`Export "${exportId}" not found at ${path}`);
   }
-  return JSON.parse(await readFile(path, 'utf-8'));
+  const manifest = JSON.parse(await readFile(path, 'utf-8'));
+  checkManifestVersion(manifest, 'export');
+  return manifest;
 }

package/lib/implementation-packs.js

@@ -17,6 +17,8 @@ import { loadTrainingProfile } from './training-profiles.js';
 import { loadProjectConfig, detectLane, detectGroup } from './config.js';
 import { loadSplitPartition } from './split.js';
 import { loadRecord } from './records.js';
+import { inputError } from './errors.js';
+import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
 
 function generateImplId() {
   const now = new Date();
@@ -138,9 +140,19 @@ export async function buildImplementationPack(projectRoot, manifestId) {
   // Write implementation pack
   const implId = generateImplId();
   const implDir = join(projectRoot, 'training', 'implementations', implId);
+  // D-005: refuse to silently overwrite on ID collision.
+  if (existsSync(implDir)) {
+    throw inputError(
+      'IMPL_PACK_ID_COLLISION',
+      `Implementation pack directory already exists: ${implDir}`,
+      'Retry to generate a fresh implementation pack ID.',
+    );
+  }
   await mkdir(implDir, { recursive: true });
 
   const implManifest = {
+    // DB-001 / DB-006: stamp manifest schema version
+    schema_version: SCHEMA_VERSION,
     implementation_pack_id: implId,
     created_at: new Date().toISOString(),
     created_by: 'sdlab-implementation-v1',
@@ -312,5 +324,7 @@ export async function loadImplementationPack(projectRoot, implId) {
   if (!existsSync(path)) {
     throw new Error(`Implementation pack "${implId}" not found at ${path}`);
   }
-  return JSON.parse(await readFile(path, 'utf-8'));
+  const manifest = JSON.parse(await readFile(path, 'utf-8'));
+  checkManifestVersion(manifest, 'implementation-pack');
+  return manifest;
 }

package/lib/log.js

@@ -3,6 +3,13 @@
  *
  * Levels: quiet < normal < verbose
  * Set via --quiet or --verbose flags (parsed by caller).
+ *
+ * Context prefix:
+ *   info('generate', 'starting wave 1')  → '[generate] starting wave 1'
+ *   info('starting wave 1')               → 'starting wave 1' (no prefix)
+ * The first argument is treated as a command/context label when it is a
+ * short single-token string with no whitespace; otherwise the call is
+ * prefix-less and backwards-compatible.
  */
 
 let level = 'normal'; // quiet | normal | verbose
@@ -15,22 +22,52 @@ export function getLogLevel() {
   return level;
 }
 
+// Accept a leading short-token context label as the first arg.
+// Heuristic: string, no whitespace, ≤ 32 chars, and there is at least
+// one additional argument after it. This keeps existing call sites
+// (where args is typically a full message) unchanged.
+function applyContextPrefix(args) {
+  if (args.length < 2) return args;
+  const first = args[0];
+  if (typeof first !== 'string') return args;
+  if (first.length > 32) return args;
+  if (/\s/.test(first)) return args;
+  // Avoid treating a message that happens to start with a format token as
+  // a context: require it to look like an identifier (word chars, dots, dashes).
+  if (!/^[A-Za-z][\w.:-]*$/.test(first)) return args;
+  return [`[${first}]`, ...args.slice(1)];
+}
+
 /** Always shown (errors, final results). */
 export function log(...args) {
-  console.log(...args);
+  console.log(...applyContextPrefix(args));
+}
+
+/**
+ * Artifact / final-output path helper.
+ * Always prints regardless of verbosity (survives --quiet) so scripts
+ * that produce files have a machine-parseable path on stdout.
+ */
+export function result(...args) {
+  console.log(...applyContextPrefix(args));
+}
+
+/** Always shown (used for final success lines). */
+export function success(...args) {
+  console.log(...applyContextPrefix(args));
 }
 
 /** Shown at normal and verbose levels (progress, summaries). */
 export function info(...args) {
-  if (level !== 'quiet') console.log(...args);
+  if (level !== 'quiet') console.log(...applyContextPrefix(args));
 }
 
 /** Shown only at verbose level (per-record details, debug context). */
 export function verbose(...args) {
-  if (level === 'verbose') console.log(...args);
+  if (level === 'verbose') console.log(...applyContextPrefix(args));
 }
 
 /** Always shown on stderr. */
 export function warn(...args) {
-  console.error('\x1b[33m⚠\x1b[0m', ...args);
+  console.error('\x1b[33m⚠\x1b[0m', ...applyContextPrefix(args));
 }

package/lib/paths.js

@@ -3,8 +3,9 @@
  */
 
 import { fileURLToPath } from 'node:url';
-import { dirname, join } from 'node:path';
+import { dirname, join, resolve, relative, isAbsolute } from 'node:path';
 import { existsSync } from 'node:fs';
+import { inputError } from './errors.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -18,10 +19,18 @@ export const REPO_ROOT = join(__dirname, '..');
  * @returns {string} absolute path to projects/<name>/
  */
 export function getProjectRoot(name) {
+  if (!name || typeof name !== 'string') {
+    throw inputError(
+      'INPUT_MISSING_PROJECT',
+      'Project name is required.',
+      'Pass --project <name>.'
+    );
+  }
   const root = join(REPO_ROOT, 'projects', name);
   if (!existsSync(root)) {
-    throw new Error(
-      `Project "${name}" not found at ${root}.\n` +
+    throw inputError(
+      'INPUT_UNKNOWN_PROJECT',
+      `Project "${name}" not found at ${root}.`,
       `Run: sdlab init ${name} --domain <domain>`
     );
   }
@@ -37,6 +46,35 @@ export function resolveProjectPath(name, ...segments) {
   return join(getProjectRoot(name), ...segments);
 }
 
+/**
+ * Resolve a user-supplied path relative to a project root and verify
+ * it does not escape the root (no `..` traversal, no absolute-path escapes
+ * out of root). Returns the absolute resolved path.
+ *
+ * @param {string} projectRoot — absolute project root (trusted)
+ * @param {string} userPath — user-supplied, possibly malicious path
+ * @param {{baseRoot?:string, flagName?:string}} [opts]
+ * @returns {string} absolute path guaranteed to be within baseRoot
+ */
+export function resolveSafeProjectPath(projectRoot, userPath, opts = {}) {
+  const flagName = opts.flagName || 'path';
+  if (userPath == null || userPath === '') {
+    throw inputError('INPUT_MISSING_VALUE', `Flag --${flagName} requires a path.`);
+  }
+  const baseRoot = opts.baseRoot || projectRoot;
+  const absBase = resolve(baseRoot);
+  const resolved = isAbsolute(userPath) ? resolve(userPath) : resolve(absBase, userPath);
+  const rel = relative(absBase, resolved);
+  if (rel.startsWith('..') || (isAbsolute(rel) && rel !== '')) {
+    throw inputError(
+      'INPUT_PATH_TRAVERSAL',
+      `Path "${userPath}" resolves outside of ${absBase}.`,
+      `Paths passed via --${flagName} must stay within the project tree.`
+    );
+  }
+  return resolved;
+}
+
 /**
  * Return the workflow profiles directory for a project root.
  * @param {string} projectRoot — absolute path to project

package/lib/refine-briefs.js

@@ -73,19 +73,26 @@ export function renderRefinedPrompt(parentPrompt, delta) {
  *   - Append suppress items that aren't already present
  */
 export function renderRefinedNegative(parentNegative, delta) {
-  const existing = parentNegative.split(',').map(s => s.trim().toLowerCase());
+  // PB-007: coerce missing/null/non-string negatives to '' so we don't
+  // throw on legacy briefs or hand-edited JSON that dropped the field.
+  const parent = typeof parentNegative === 'string' ? parentNegative : '';
+  const existing = parent
+    .split(',')
+    .map(s => s.trim().toLowerCase())
+    .filter(s => s.length > 0);
   const additions = [];
 
   for (const item of delta.suppress || []) {
-    const lower = item.toLowerCase();
-    // Don't duplicate if already in negative
-    if (!existing.some(e => e.includes(lower) || lower.includes(e))) {
+    const lower = item.toLowerCase().trim();
+    // Don't duplicate if already in negative (token-exact match)
+    if (lower.length > 0 && !existing.includes(lower)) {
       additions.push(item);
     }
   }
 
-  if (additions.length === 0) return parentNegative;
-  return `${parentNegative}, ${additions.join(', ')}`;
+  if (additions.length === 0) return parent;
+  if (parent.length === 0) return additions.join(', ');
+  return `${parent}, ${additions.join(', ')}`;
 }
 
 // ─── Delta extraction from critique ──────────────────────────────
@@ -166,9 +173,9 @@ export function extractDelta(critique, pickedFilename, cliOverrides = {}) {
 export function buildRefinedBrief({ parentBrief, critique, pickedCandidate, delta, runId, refineDir }) {
   const briefId = generateRefinedBriefId(refineDir, parentBrief.brief_id);
 
-  // Render updated prompts
-  const prompt = renderRefinedPrompt(parentBrief.prompt, delta);
-  const negativePrompt = renderRefinedNegative(parentBrief.negative_prompt, delta);
+  // Render updated prompts — PB-007 guards missing fields upstream.
+  const prompt = renderRefinedPrompt(parentBrief.prompt ?? '', delta);
+  const negativePrompt = renderRefinedNegative(parentBrief.negative_prompt ?? '', delta);
 
   const refined = {
     brief_id: briefId,

package/lib/reingest-selected.js

@@ -71,10 +71,21 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
   const skipped = [];
   const provenanceLines = [];
 
-  for (const item of manifest.items) {
-    // Determine record ID from selection + item index
-    const idx = String(manifest.items.indexOf(item) + 1).padStart(3, '0');
+  // D-010: use entries() — not indexOf() — to avoid O(n^2) behavior and
+  // silent duplicate-ID bugs when manifest.items contains structurally
+  // equal or repeated references. Assert uniqueness of generated IDs.
+  const seenIds = new Set();
+  for (const [i, item] of manifest.items.entries()) {
+    const idx = String(i + 1).padStart(3, '0');
     const recordId = `gen_${selectionId}_${idx}`;
+    if (seenIds.has(recordId)) {
+      throw inputError(
+        'REINGEST_DUPLICATE_RECORD_ID',
+        `Duplicate record ID generated during re-ingest: ${recordId}`,
+        'This indicates a bug in record ID derivation; report this along with the selection manifest.',
+      );
+    }
+    seenIds.add(recordId);
     const recordPath = join(recordsDir, `${recordId}.json`);
 
     if (existsSync(recordPath)) {

package/lib/reingest.js

@@ -7,7 +7,7 @@
  * canon-bound like everything else.
  */
 
-import { readFile, writeFile, readdir, stat } from 'node:fs/promises';
+import { readFile, writeFile, readdir, stat, mkdir, copyFile } from 'node:fs/promises';
 import { join, basename, extname } from 'node:path';
 import { existsSync } from 'node:fs';
 import { loadTrainingManifest } from './training-manifests.js';
@@ -33,6 +33,12 @@ const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
 export async function reingestGenerated(projectRoot, sourcePath, manifestId, options = {}) {
   const manifest = await loadTrainingManifest(projectRoot, manifestId);
   const recordsDir = join(projectRoot, 'records');
+  // D-020: ensure outputs/candidates/ exists so the record's asset_path
+  // contract ("outputs/candidates/<file>") actually points at the image.
+  const candidatesDir = join(projectRoot, 'outputs', 'candidates');
+  if (!options.dryRun) {
+    await mkdir(candidatesDir, { recursive: true });
+  }
 
   // Find image files in source directory
   const entries = await readdir(sourcePath);
@@ -80,6 +86,14 @@ export async function reingestGenerated(projectRoot, sourcePath, manifestId, opt
     record.schema_version = '2.1.0';
 
     if (!options.dryRun) {
+      // D-020: copy the image into outputs/candidates/ so the record's
+      // project-relative asset_path contract is satisfied. Without this,
+      // downstream readers (export, adapters) would silently skip the
+      // record because the file does not exist at the declared path.
+      const destPath = join(candidatesDir, file);
+      if (!existsSync(destPath)) {
+        await copyFile(filePath, destPath);
+      }
       await writeFile(recordPath, JSON.stringify(record, null, 2) + '\n');
     }
 

package/lib/run-summary.js

@@ -63,10 +63,11 @@ export function renderRunMarkdown(manifest) {
   lines.push('');
 
   // Errors
-  if (manifest.errors.length > 0) {
+  const errors = manifest.errors || [];
+  if (errors.length > 0) {
     lines.push('## Errors');
     lines.push('');
-    for (const err of manifest.errors) {
+    for (const err of errors) {
       lines.push(`- **Image ${err.index + 1}** (seed ${err.seed}): ${err.error}`);
     }
     lines.push('');