@mcptoolshop/style-dataset-lab 3.0.0 → 3.0.1

package/lib/comfyui.js

@@ -6,9 +6,15 @@
  */
 
 import { readFile } from 'node:fs/promises';
+import { info } from './log.js';
 
 const DEFAULT_URL = 'http://127.0.0.1:8188';
 const MAX_POLL_MS = 600_000; // 10 minutes
+const HEARTBEAT_MS = 15_000; // emit a heartbeat at most every 15s
+// PB-004: if we see an unfamiliar status shape for this many consecutive
+// polls without completed/error, bail with a descriptive message rather
+// than hang until MAX_POLL_MS.
+const UNKNOWN_STATUS_BAIL_POLLS = 30; // ~30s of unrecognized shape
 
 /**
  * Check if ComfyUI is reachable.
@@ -48,21 +54,102 @@ export async function submitAndWait(workflow, comfyUrl = DEFAULT_URL, opts = {})
 
   const { prompt_id: promptId } = await res.json();
 
+  const MAX_CONSECUTIVE_HISTORY_FAILURES = 5;
+  let consecutiveHistoryFailures = 0;
+  // PB-001: heartbeats
+  let lastHeartbeatAt = Date.now();
+  let announcedStart = false;
+  let seenEntry = false;
+  // PB-004: count polls where entry exists but status shape is unfamiliar
+  let unknownStatusPolls = 0;
+  let lastObservedStatus = null;
   const start = Date.now();
   while (true) {
-    if (Date.now() - start > timeoutMs) {
+    const now = Date.now();
+    if (now - start > timeoutMs) {
       throw new Error(`ComfyUI poll timeout after ${timeoutMs / 1000}s for prompt ${promptId}`);
     }
     await new Promise(r => setTimeout(r, 1000));
-    const histRes = await fetch(`${comfyUrl}/history/${promptId}`);
-    if (!histRes.ok) continue;
+    let histRes;
+    try {
+      histRes = await fetch(`${comfyUrl}/history/${promptId}`);
+    } catch (err) {
+      consecutiveHistoryFailures++;
+      if (consecutiveHistoryFailures >= MAX_CONSECUTIVE_HISTORY_FAILURES) {
+        throw new Error(
+          `ComfyUI history endpoint unreachable after ${MAX_CONSECUTIVE_HISTORY_FAILURES} consecutive attempts for prompt ${promptId}: ${err.message}`
+        );
+      }
+      continue;
+    }
+    if (!histRes.ok) {
+      consecutiveHistoryFailures++;
+      if (consecutiveHistoryFailures >= MAX_CONSECUTIVE_HISTORY_FAILURES) {
+        throw new Error(
+          `ComfyUI history endpoint returned ${histRes.status} ${MAX_CONSECUTIVE_HISTORY_FAILURES} times in a row for prompt ${promptId}`
+        );
+      }
+      continue;
+    }
+    consecutiveHistoryFailures = 0;
     const history = await histRes.json();
     const entry = history[promptId];
-    if (!entry) continue;
-    if (entry.status?.completed) return entry;
+
+    // PB-001: emit heartbeats so the user knows the pipeline isn't hung.
+    const elapsedSec = Math.round((Date.now() - start) / 1000);
+    const heartbeatDue = Date.now() - lastHeartbeatAt >= HEARTBEAT_MS;
+    if (!entry) {
+      // Queued on ComfyUI but not yet visible in /history.
+      if (!announcedStart) {
+        info(`ComfyUI: queued (prompt ${promptId})`);
+        announcedStart = true;
+        lastHeartbeatAt = Date.now();
+      } else if (heartbeatDue) {
+        info(`ComfyUI: queued, waiting... (${elapsedSec}s)`);
+        lastHeartbeatAt = Date.now();
+      }
+      continue;
+    }
+
+    if (!seenEntry) {
+      seenEntry = true;
+      info(`ComfyUI: generating... (prompt ${promptId})`);
+      lastHeartbeatAt = Date.now();
+    } else if (heartbeatDue) {
+      info(`ComfyUI: generating... (${elapsedSec}s elapsed)`);
+      lastHeartbeatAt = Date.now();
+    }
+
+    // PB-004: success signals — accept both completed===true and status_str==='success'
+    // (future-proof for schema drift where one may change without the other).
+    if (entry.status?.completed === true || entry.status?.status_str === 'success') {
+      info(`ComfyUI: completed in ${elapsedSec}s`);
+      return entry;
+    }
+
+    // PB-004: known failure signals — bail immediately.
     if (entry.status?.status_str === 'error') {
       throw new Error(`ComfyUI generation failed: ${JSON.stringify(entry.status)}`);
     }
+    if (entry.status?.execution_error || entry.execution_error) {
+      const errObj = entry.status?.execution_error || entry.execution_error;
+      throw new Error(`ComfyUI execution error for prompt ${promptId}: ${JSON.stringify(errObj)}`);
+    }
+
+    // PB-004: the entry exists but matches neither a known success nor
+    // a known failure. This is the schema-drift hang scenario — count
+    // consecutive unknown-shape polls and bail with context.
+    const statusStr = entry.status?.status_str ?? null;
+    lastObservedStatus = entry.status ?? null;
+    unknownStatusPolls++;
+    if (unknownStatusPolls >= UNKNOWN_STATUS_BAIL_POLLS) {
+      throw new Error(
+        `ComfyUI returned an unrecognized status shape for prompt ${promptId} ` +
+        `after ${unknownStatusPolls} polls (status_str=${JSON.stringify(statusStr)}). ` +
+        `This usually means the ComfyUI history schema changed. ` +
+        `Observed status: ${JSON.stringify(lastObservedStatus)}`
+      );
+    }
   }
 }
 
@@ -75,7 +162,14 @@ export async function submitAndWait(workflow, comfyUrl = DEFAULT_URL, opts = {})
  * @returns {Promise<Buffer>}
  */
 export async function downloadImage(filename, subfolder, comfyUrl = DEFAULT_URL, type = 'output') {
-  const url = `${comfyUrl}/view?filename=${encodeURIComponent(filename)}&subfolder=${encodeURIComponent(subfolder || '')}&type=${type}`;
+  const ALLOWED_TYPES = ['output', 'input', 'temp'];
+  if (!ALLOWED_TYPES.includes(type)) {
+    throw new Error(`Invalid ComfyUI image type "${type}" — must be one of: ${ALLOWED_TYPES.join(', ')}`);
+  }
+  if (typeof comfyUrl !== 'string' || comfyUrl.length === 0 || !/^https?:\/\//i.test(comfyUrl)) {
+    throw new Error(`Invalid comfyUrl "${comfyUrl}" — must start with http:// or https://`);
+  }
+  const url = `${comfyUrl}/view?filename=${encodeURIComponent(filename)}&subfolder=${encodeURIComponent(subfolder || '')}&type=${encodeURIComponent(type)}`;
   const res = await fetch(url);
   if (!res.ok) throw new Error(`Download failed: ${res.status}`);
   return Buffer.from(await res.arrayBuffer());

package/lib/config.js

@@ -7,17 +7,70 @@
  */
 
 import { readFileSync, existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
+import { inputError, runtimeError } from './errors.js';
 
 function loadJsonFile(filePath) {
   if (!existsSync(filePath)) return null;
   try {
     return JSON.parse(readFileSync(filePath, 'utf-8'));
   } catch (err) {
-    throw new Error(`Failed to parse ${filePath}: ${err.message}`);
+    throw runtimeError(
+      'INPUT_INVALID_JSON',
+      `Failed to parse ${filePath}: ${err.message}`,
+      'Check the file for JSON syntax errors (trailing commas, unquoted keys, etc).',
+      err
+    );
   }
 }
 
+/**
+ * Async-safe readFile + JSON.parse with structured error handling.
+ * Optionally validates that required top-level fields are present.
+ *
+ * @param {string} filePath — absolute path to a JSON file
+ * @param {{requiredFields?: string[]}} [opts]
+ */
+export async function readJsonFile(filePath, opts = {}) {
+  let raw;
+  try {
+    raw = await readFile(filePath, 'utf-8');
+  } catch (err) {
+    if (err && err.code === 'ENOENT') {
+      throw inputError(
+        'INPUT_FILE_NOT_FOUND',
+        `File not found: ${filePath}`,
+        'Check the path and try again.'
+      );
+    }
+    throw runtimeError('RUNTIME_READ_FAILED', `Could not read ${filePath}: ${err.message}`, null, err);
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw runtimeError(
+      'INPUT_INVALID_JSON',
+      `Failed to parse ${filePath}: ${err.message}`,
+      'Check the file for JSON syntax errors.',
+      err
+    );
+  }
+  const required = opts.requiredFields || [];
+  if (required.length) {
+    const missing = required.filter((f) => parsed == null || parsed[f] === undefined);
+    if (missing.length) {
+      throw inputError(
+        'INPUT_MISSING_FIELDS',
+        `${filePath} is missing required fields: ${missing.join(', ')}`,
+        `Add the fields to the file, or use a valid example as template.`
+      );
+    }
+  }
+  return parsed;
+}
+
 /**
  * Load project.json — project identity and generation defaults.
  */
@@ -42,7 +95,13 @@ export function loadConstitution(projectRoot) {
   const data = loadJsonFile(join(projectRoot, 'constitution.json'));
   if (!data) return { rules: [] };
   if (Array.isArray(data)) return { rules: data };
-  if (!data.rules) throw new Error(`constitution.json must have a "rules" array`);
+  if (!data.rules) {
+    throw inputError(
+      'INPUT_BAD_CONFIG',
+      `constitution.json must have a "rules" array`,
+      'Wrap your rules in { "rules": [...] } or use an array at the top level.'
+    );
+  }
   return data;
 }
 
@@ -54,7 +113,13 @@ export function loadLanes(projectRoot) {
   if (!data) {
     return { default_lane: 'default', lanes: [] };
   }
-  if (!data.lanes) throw new Error(`lanes.json must have a "lanes" array`);
+  if (!data.lanes) {
+    throw inputError(
+      'INPUT_BAD_CONFIG',
+      `lanes.json must have a "lanes" array`,
+      'Top-level shape: { "default_lane": "...", "lanes": [ ... ] }.'
+    );
+  }
   return data;
 }
 

package/lib/deprecation.js

@@ -9,6 +9,6 @@ export function warnDeprecated(oldFlag, newFlag) {
   if (warned.has(key)) return;
   warned.add(key);
   process.stderr.write(
-    `\x1b[33m⚠ --${oldFlag} is deprecated and will be removed in v3.0. Use --${newFlag} instead.\x1b[0m\n`
+    `\x1b[33m⚠ --${oldFlag} is deprecated and will be removed in v4.0. Use --${newFlag} instead.\x1b[0m\n`
   );
 }

package/lib/eval-pack.js

@@ -16,6 +16,8 @@ import { existsSync } from 'node:fs';
 import { randomBytes } from 'node:crypto';
 import { loadProjectConfig, detectLane, detectGroup } from './config.js';
 import { loadRecord } from './records.js';
+import { computeConfigFingerprint, SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
+import { inputError } from './errors.js';
 
 /**
  * Generate an eval-pack ID: eval-YYYYMMDD-HHMMSS-XXXX
@@ -44,11 +46,18 @@ export async function buildEvalPack(projectRoot, options = {}) {
   const maxDrift = options.maxDrift || 20;
   const maxContinuity = options.maxContinuity || 10;
 
-  // Load all records
+  // Load all records (DB-003-adjacent: skip unparseable files rather than
+  // aborting the entire eval-pack build — same graceful-degradation shape
+  // snapshot.js uses).
   const records = [];
   for (const file of files) {
-    const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
-    records.push(record);
+    try {
+      const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+      records.push(record);
+    } catch {
+      // Skip unreadable records; eval pack is a best-effort selection and
+      // one bad file must not kill the whole build.
+    }
   }
 
   // Task 1: Lane coverage
@@ -66,19 +75,60 @@ export async function buildEvalPack(projectRoot, options = {}) {
   // Write eval pack
   const evalId = generateEvalId();
   const evalDir = join(projectRoot, 'eval-packs', evalId);
+  // D-005: refuse to silently overwrite on ID collision.
+  if (existsSync(evalDir)) {
+    throw inputError(
+      'EVAL_PACK_ID_COLLISION',
+      `Eval pack directory already exists: ${evalDir}`,
+      'Retry to generate a fresh eval pack ID.',
+    );
+  }
   await mkdir(evalDir, { recursive: true });
 
+  // DB-014: compute `records_available` so the manifest reports sampled vs.
+  // available. Operators can see when they're leaving records on the table.
+  const laneAvailable = countLaneCoverageCandidates(records, config);
+  const driftAvailable = countForbiddenDriftCandidates(records);
+  const anchorAvailable = countAnchorGoldCandidates(records, config);
+  const continuityAvailable = countSubjectContinuityCandidates(records);
+
   const manifest = {
+    // DB-001 / DB-006: stamp manifest schema version
+    schema_version: SCHEMA_VERSION,
     eval_id: evalId,
     created_at: new Date().toISOString(),
     created_by: 'sdlab-eval-v1',
     project: config.meta.name,
+    // D-017: capture config fingerprint so eval verdicts can detect canon drift.
+    config_fingerprint: computeConfigFingerprint(projectRoot),
     task_counts: {
       lane_coverage: laneCoverage.length,
       forbidden_drift: forbiddenDrift.length,
       anchor_gold: anchorGold.length,
       subject_continuity: subjectContinuity.length,
     },
+    // DB-014: coverage — sampled vs. available per task
+    records_sampled: {
+      lane_coverage: laneCoverage.length,
+      forbidden_drift: forbiddenDrift.length,
+      anchor_gold: anchorGold.length,
+      subject_continuity: subjectContinuity.length,
+      total: laneCoverage.length + forbiddenDrift.length +
+        anchorGold.length + subjectContinuity.length,
+    },
+    records_available: {
+      lane_coverage: laneAvailable,
+      forbidden_drift: driftAvailable,
+      anchor_gold: anchorAvailable,
+      subject_continuity: continuityAvailable,
+      total: laneAvailable + driftAvailable + anchorAvailable + continuityAvailable,
+    },
+    caps: {
+      max_per_lane: maxPerLane,
+      max_per_faction: maxPerFaction,
+      max_drift: maxDrift,
+      max_continuity: maxContinuity,
+    },
     total_records: laneCoverage.length + forbiddenDrift.length +
       anchorGold.length + subjectContinuity.length,
   };
@@ -95,6 +145,44 @@ export async function buildEvalPack(projectRoot, options = {}) {
   };
 }
 
+/** DB-014: count approved+canon-bound records (the lane-coverage candidate pool). */
+function countLaneCoverageCandidates(records, _config) {
+  let n = 0;
+  for (const r of records) {
+    if (r.judgment?.status === 'approved' && r.canon?.assertions?.length) n++;
+  }
+  return n;
+}
+
+/** DB-014: count records that qualify as forbidden-drift candidates. */
+function countForbiddenDriftCandidates(records) {
+  let n = 0;
+  for (const r of records) {
+    const status = r.judgment?.status;
+    if (status !== 'rejected' && status !== 'borderline') continue;
+    const failureModes = r.judgment?.failure_modes || [];
+    const failedAssertions = (r.canon?.assertions || [])
+      .filter(a => a.verdict === 'fail').length;
+    if (failureModes.length > 0 || failedAssertions > 0 || r.judgment?.explanation) n++;
+  }
+  return n;
+}
+
+/** DB-014: count approved+canon-bound records (also anchor-gold pool). */
+function countAnchorGoldCandidates(records, _config) {
+  return countLaneCoverageCandidates(records, _config);
+}
+
+/** DB-014: count named subjects (subject-continuity pool). */
+function countSubjectContinuityCandidates(records) {
+  const subjects = new Set();
+  for (const r of records) {
+    const s = r.identity?.subject_name;
+    if (s) subjects.add(s);
+  }
+  return subjects.size;
+}
+
 /**
  * Task 1: Lane coverage — best approved record per lane.
  */
@@ -279,5 +367,7 @@ export async function loadEvalPack(projectRoot, evalId) {
   if (!existsSync(path)) {
     throw new Error(`Eval pack "${evalId}" not found at ${path}`);
   }
-  return JSON.parse(await readFile(path, 'utf-8'));
+  const manifest = JSON.parse(await readFile(path, 'utf-8'));
+  checkManifestVersion(manifest, 'eval-pack');
+  return manifest;
 }

package/lib/eval-runs.js

@@ -13,6 +13,11 @@ import { randomBytes } from 'node:crypto';
 import { loadTrainingManifest } from './training-manifests.js';
 import { loadEvalPack } from './eval-pack.js';
 import { loadProjectConfig, detectLane, detectGroup } from './config.js';
+import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
+
+// DB-008: cap sample record_ids per failure bucket so scorecards stay small
+// but the operator has enough detail to start investigation.
+const SAMPLE_LIMIT = 5;
 
 /**
  * Generate an eval run ID: er-YYYYMMDD-HHMMSS-XXXX
@@ -42,6 +47,8 @@ export async function createEvalRun(projectRoot, manifestId, evalPackId) {
   await mkdir(evalRunDir, { recursive: true });
 
   const runManifest = {
+    // DB-001 / DB-006: stamp manifest schema version
+    schema_version: SCHEMA_VERSION,
     eval_run_id: evalRunId,
     created_at: new Date().toISOString(),
     created_by: 'sdlab-eval-run-v1',
@@ -148,16 +155,25 @@ export async function scoreEvalRun(projectRoot, evalRunId, outputsPath) {
 function scoreLaneCoverage(tasks, outputMap) {
   let covered = 0;
   const lanes = new Set();
+  // DB-008: collect record_ids that missed coverage so the operator can drill
+  // down from "covered: 7/10" to "which 3 weren't covered".
+  const missedSamples = [];
   for (const task of tasks) {
     lanes.add(task.lane);
     const output = outputMap.get(task.record_id);
-    if (output?.accept) covered++;
+    if (output?.accept) {
+      covered++;
+    } else if (missedSamples.length < SAMPLE_LIMIT) {
+      missedSamples.push(task.record_id);
+    }
   }
   const score = tasks.length > 0 ? covered / tasks.length : 0;
   return {
     task: 'lane_coverage',
     total: tasks.length,
     covered,
+    missed: tasks.length - covered,
+    missed_sample_record_ids: missedSamples,
     lanes_represented: lanes.size,
     score: +score.toFixed(3),
     passed: score >= 0.7,
@@ -168,12 +184,19 @@ function scoreForbiddenDrift(tasks, outputMap) {
   // For forbidden drift, we check that rejected/borderline records are NOT accepted
   let correctRejections = 0;
   let falseAcceptances = 0;
+  // DB-008: name the records that were wrongly accepted. Without this, a
+  // scorecard that says "false_acceptances: 3" forces the operator to diff
+  // outputs.jsonl against the eval pack manually.
+  const falseAcceptanceSamples = [];
   for (const task of tasks) {
     const output = outputMap.get(task.record_id);
     if (!output || !output.accept) {
       correctRejections++;
     } else {
       falseAcceptances++;
+      if (falseAcceptanceSamples.length < SAMPLE_LIMIT) {
+        falseAcceptanceSamples.push(task.record_id);
+      }
     }
   }
   const score = tasks.length > 0 ? correctRejections / tasks.length : 1;
@@ -182,6 +205,7 @@ function scoreForbiddenDrift(tasks, outputMap) {
     total: tasks.length,
     correct_rejections: correctRejections,
     false_acceptances: falseAcceptances,
+    false_acceptance_sample_record_ids: falseAcceptanceSamples,
     score: +score.toFixed(3),
     passed: falseAcceptances === 0,
   };
@@ -189,15 +213,23 @@ function scoreForbiddenDrift(tasks, outputMap) {
 
 function scoreAnchorGold(tasks, outputMap) {
   let matched = 0;
+  // DB-008: which anchor records failed to match?
+  const missedSamples = [];
   for (const task of tasks) {
     const output = outputMap.get(task.record_id);
-    if (output?.accept) matched++;
+    if (output?.accept) {
+      matched++;
+    } else if (missedSamples.length < SAMPLE_LIMIT) {
+      missedSamples.push(task.record_id);
+    }
   }
   const score = tasks.length > 0 ? matched / tasks.length : 0;
   return {
     task: 'anchor_gold',
     total: tasks.length,
     matched,
+    missed: tasks.length - matched,
+    missed_sample_record_ids: missedSamples,
     score: +score.toFixed(3),
     passed: score >= 0.8,
   };
@@ -205,16 +237,23 @@ function scoreAnchorGold(tasks, outputMap) {
 
 function scoreSubjectContinuity(tasks, outputMap) {
   let subjectsWithCoverage = 0;
+  // DB-008: which named subjects lost continuity coverage?
+  const missedSamples = [];
   for (const task of tasks) {
     const recordIds = task.record_ids || [];
     const accepted = recordIds.filter(id => outputMap.get(id)?.accept).length;
-    if (accepted >= 2) subjectsWithCoverage++;
+    if (accepted >= 2) {
+      subjectsWithCoverage++;
+    } else if (missedSamples.length < SAMPLE_LIMIT) {
+      missedSamples.push(task.subject_name || recordIds[0] || 'unknown');
+    }
   }
   const score = tasks.length > 0 ? subjectsWithCoverage / tasks.length : 0;
   return {
     task: 'subject_continuity',
     total_subjects: tasks.length,
     subjects_with_coverage: subjectsWithCoverage,
+    missed_subject_samples: missedSamples,
     score: +score.toFixed(3),
     passed: score >= 0.6,
   };