@mcptoolshop/style-dataset-lab 2.0.0 → 2.2.0

package/lib/eval-runs.js

@@ -0,0 +1,285 @@
+/**
+ * Eval run management.
+ *
+ * An eval run scores generated outputs against Phase 2 eval packs.
+ * Every eval run links back to a training manifest and eval pack —
+ * no orphan evals.
+ */
+
+import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { loadTrainingManifest } from './training-manifests.js';
+import { loadEvalPack } from './eval-pack.js';
+import { loadProjectConfig, detectLane, detectGroup } from './config.js';
+
+/**
+ * Generate an eval run ID: er-YYYYMMDD-HHMMSS-XXXX
+ */
+function generateEvalRunId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `er-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Create an eval run — links a training manifest to an eval pack.
+ *
+ * @param {string} projectRoot
+ * @param {string} manifestId — training manifest ID
+ * @param {string} evalPackId — eval pack ID
+ * @returns {Promise<{evalRunId: string}>}
+ */
+export async function createEvalRun(projectRoot, manifestId, evalPackId) {
+  // Verify both exist
+  const manifest = await loadTrainingManifest(projectRoot, manifestId);
+  const evalPack = await loadEvalPack(projectRoot, evalPackId);
+
+  const evalRunId = generateEvalRunId();
+  const evalRunDir = join(projectRoot, 'training', 'eval-runs', evalRunId);
+  await mkdir(evalRunDir, { recursive: true });
+
+  const runManifest = {
+    eval_run_id: evalRunId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-eval-run-v1',
+    training_manifest_id: manifestId,
+    eval_pack_id: evalPackId,
+    source_export_id: manifest.source_export_id,
+    training_profile_id: manifest.training_profile_id,
+    adapter_target: manifest.adapter_target,
+    status: 'created',
+    scorecard: null,
+  };
+
+  await writeFile(join(evalRunDir, 'manifest.json'), JSON.stringify(runManifest, null, 2) + '\n');
+
+  return { evalRunId };
+}
+
+/**
+ * Score an eval run — compare generated outputs against eval pack tasks.
+ *
+ * Outputs are provided as a JSONL file where each line has:
+ *   { record_id, generated_path, accept: true|false, notes? }
+ *
+ * @param {string} projectRoot
+ * @param {string} evalRunId
+ * @param {string} outputsPath — path to outputs index JSONL
+ * @returns {Promise<{scorecard: Object}>}
+ */
+export async function scoreEvalRun(projectRoot, evalRunId, outputsPath) {
+  const evalRunDir = join(projectRoot, 'training', 'eval-runs', evalRunId);
+  const runManifest = JSON.parse(await readFile(join(evalRunDir, 'manifest.json'), 'utf-8'));
+  const config = loadProjectConfig(projectRoot);
+
+  // Load outputs
+  const outputsRaw = await readFile(outputsPath, 'utf-8');
+  const outputs = outputsRaw.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
+
+  // Copy outputs index into eval run
+  await writeFile(join(evalRunDir, 'output-index.jsonl'), outputsRaw);
+
+  // Load eval pack tasks
+  const evalPackDir = join(projectRoot, 'eval-packs', runManifest.eval_pack_id);
+  const taskFiles = {
+    'lane-coverage': 'lane-coverage.jsonl',
+    'forbidden-drift': 'forbidden-drift.jsonl',
+    'anchor-gold': 'anchor-gold.jsonl',
+    'subject-continuity': 'subject-continuity.jsonl',
+  };
+
+  const outputMap = new Map(outputs.map(o => [o.record_id, o]));
+
+  // Score each task type
+  const scores = {};
+
+  for (const [taskName, fileName] of Object.entries(taskFiles)) {
+    const taskPath = join(evalPackDir, fileName);
+    if (!existsSync(taskPath)) continue;
+
+    const taskEntries = (await readFile(taskPath, 'utf-8'))
+      .trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
+
+    if (taskName === 'lane-coverage') {
+      scores.lane_coverage = scoreLaneCoverage(taskEntries, outputMap);
+    } else if (taskName === 'forbidden-drift') {
+      scores.forbidden_drift = scoreForbiddenDrift(taskEntries, outputMap);
+    } else if (taskName === 'anchor-gold') {
+      scores.anchor_gold = scoreAnchorGold(taskEntries, outputMap);
+    } else if (taskName === 'subject-continuity') {
+      scores.subject_continuity = scoreSubjectContinuity(taskEntries, outputMap);
+    }
+  }
+
+  // Compute overall verdict
+  const allPassed = Object.values(scores).every(s => s.passed);
+  const overallScore = Object.values(scores).reduce((sum, s) => sum + (s.score || 0), 0) /
+    Math.max(Object.values(scores).length, 1);
+
+  const scorecard = {
+    eval_run_id: evalRunId,
+    scored_at: new Date().toISOString(),
+    overall_verdict: allPassed ? 'pass' : 'fail',
+    overall_score: +overallScore.toFixed(3),
+    output_count: outputs.length,
+    accepted_count: outputs.filter(o => o.accept).length,
+    rejected_count: outputs.filter(o => !o.accept).length,
+    tasks: scores,
+  };
+
+  // Write scorecard
+  await writeFile(join(evalRunDir, 'scorecard.json'), JSON.stringify(scorecard, null, 2) + '\n');
+
+  // Write summary markdown
+  const summary = renderSummary(scorecard, runManifest);
+  await writeFile(join(evalRunDir, 'summary.md'), summary);
+
+  // Update manifest status
+  runManifest.status = 'scored';
+  runManifest.scorecard = scorecard;
+  await writeFile(join(evalRunDir, 'manifest.json'), JSON.stringify(runManifest, null, 2) + '\n');
+
+  return { scorecard };
+}
+
+function scoreLaneCoverage(tasks, outputMap) {
+  let covered = 0;
+  const lanes = new Set();
+  for (const task of tasks) {
+    lanes.add(task.lane);
+    const output = outputMap.get(task.record_id);
+    if (output?.accept) covered++;
+  }
+  const score = tasks.length > 0 ? covered / tasks.length : 0;
+  return {
+    task: 'lane_coverage',
+    total: tasks.length,
+    covered,
+    lanes_represented: lanes.size,
+    score: +score.toFixed(3),
+    passed: score >= 0.7,
+  };
+}
+
+function scoreForbiddenDrift(tasks, outputMap) {
+  // For forbidden drift, we check that rejected/borderline records are NOT accepted
+  let correctRejections = 0;
+  let falseAcceptances = 0;
+  for (const task of tasks) {
+    const output = outputMap.get(task.record_id);
+    if (!output || !output.accept) {
+      correctRejections++;
+    } else {
+      falseAcceptances++;
+    }
+  }
+  const score = tasks.length > 0 ? correctRejections / tasks.length : 1;
+  return {
+    task: 'forbidden_drift',
+    total: tasks.length,
+    correct_rejections: correctRejections,
+    false_acceptances: falseAcceptances,
+    score: +score.toFixed(3),
+    passed: falseAcceptances === 0,
+  };
+}
+
+function scoreAnchorGold(tasks, outputMap) {
+  let matched = 0;
+  for (const task of tasks) {
+    const output = outputMap.get(task.record_id);
+    if (output?.accept) matched++;
+  }
+  const score = tasks.length > 0 ? matched / tasks.length : 0;
+  return {
+    task: 'anchor_gold',
+    total: tasks.length,
+    matched,
+    score: +score.toFixed(3),
+    passed: score >= 0.8,
+  };
+}
+
+function scoreSubjectContinuity(tasks, outputMap) {
+  let subjectsWithCoverage = 0;
+  for (const task of tasks) {
+    const recordIds = task.record_ids || [];
+    const accepted = recordIds.filter(id => outputMap.get(id)?.accept).length;
+    if (accepted >= 2) subjectsWithCoverage++;
+  }
+  const score = tasks.length > 0 ? subjectsWithCoverage / tasks.length : 0;
+  return {
+    task: 'subject_continuity',
+    total_subjects: tasks.length,
+    subjects_with_coverage: subjectsWithCoverage,
+    score: +score.toFixed(3),
+    passed: score >= 0.6,
+  };
+}
+
+function renderSummary(scorecard, manifest) {
+  const lines = [];
+  lines.push(`# Eval Run: ${scorecard.eval_run_id}`);
+  lines.push('');
+  lines.push(`**Training manifest:** ${manifest.training_manifest_id}`);
+  lines.push(`**Eval pack:** ${manifest.eval_pack_id}`);
+  lines.push(`**Scored:** ${scorecard.scored_at}`);
+  lines.push(`**Verdict:** ${scorecard.overall_verdict.toUpperCase()}`);
+  lines.push(`**Overall score:** ${(scorecard.overall_score * 100).toFixed(1)}%`);
+  lines.push('');
+  lines.push(`**Outputs:** ${scorecard.output_count} total, ${scorecard.accepted_count} accepted, ${scorecard.rejected_count} rejected`);
+  lines.push('');
+  lines.push('## Task scores');
+  lines.push('');
+  lines.push('| Task | Score | Passed |');
+  lines.push('|------|-------|--------|');
+  for (const [name, task] of Object.entries(scorecard.tasks)) {
+    lines.push(`| ${name} | ${(task.score * 100).toFixed(1)}% | ${task.passed ? 'YES' : 'NO'} |`);
+  }
+  lines.push('');
+  return lines.join('\n');
+}
+
+/**
+ * Load an eval run manifest.
+ */
+export async function loadEvalRun(projectRoot, evalRunId) {
+  const path = join(projectRoot, 'training', 'eval-runs', evalRunId, 'manifest.json');
+  if (!existsSync(path)) {
+    throw new Error(`Eval run "${evalRunId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}
+
+/**
+ * List all eval runs.
+ */
+export async function listEvalRuns(projectRoot) {
+  const dir = join(projectRoot, 'training', 'eval-runs');
+  if (!existsSync(dir)) return [];
+
+  const entries = await readdir(dir, { withFileTypes: true });
+  const runs = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('er-')) continue;
+    const manifestPath = join(dir, entry.name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const data = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      runs.push({
+        id: data.eval_run_id,
+        created_at: data.created_at,
+        status: data.status,
+        manifest: data.training_manifest_id,
+        eval_pack: data.eval_pack_id,
+        verdict: data.scorecard?.overall_verdict || null,
+      });
+    } catch { /* skip */ }
+  }
+
+  return runs.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}

package/lib/export.js

@@ -0,0 +1,259 @@
+/**
+ * Export package builder — self-contained, reproducible dataset exports.
+ *
+ * An export package contains everything needed to use or rebuild the dataset:
+ * manifest, metadata, images (symlinked or copied), splits, card, and checksums.
+ */
+
+import { readFile, writeFile, mkdir, readdir, symlink, copyFile, stat } from 'node:fs/promises';
+import { join, relative, resolve } from 'node:path';
+import { existsSync } from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { loadProjectConfig, loadExportProfile, detectLane, detectGroup } from './config.js';
+import { loadSnapshot, loadSnapshotIncluded, computeConfigFingerprint } from './snapshot.js';
+import { loadSplit, loadSplitPartition } from './split.js';
+import { loadRecord } from './records.js';
+import { generateCard } from './card.js';
+
+/**
+ * Generate an export ID: export-YYYYMMDD-HHMMSS-XXXX
+ */
+function generateExportId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `export-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Compute SHA-256 of a file.
+ */
+async function sha256File(filePath) {
+  const data = await readFile(filePath);
+  return createHash('sha256').update(data).digest('hex');
+}
+
+/**
+ * Build an export package.
+ *
+ * @param {string} projectRoot
+ * @param {string} snapshotId
+ * @param {string} splitId
+ * @param {Object} options — { profileId?, copy? }
+ * @returns {Promise<{exportId: string, recordCount: number, imageCount: number}>}
+ */
+export async function buildExport(projectRoot, snapshotId, splitId, options = {}) {
+  const config = loadProjectConfig(projectRoot);
+  const profile = loadExportProfile(projectRoot, options.profileId || null);
+  const snapshot = await loadSnapshot(projectRoot, snapshotId);
+  const split = await loadSplit(projectRoot, splitId);
+
+  // Load all included records
+  const included = await loadSnapshotIncluded(projectRoot, snapshotId);
+  const recordsDir = join(projectRoot, 'records');
+
+  const exportId = generateExportId();
+  const exportDir = join(projectRoot, 'exports', exportId);
+  await mkdir(exportDir, { recursive: true });
+
+  // Build metadata + collect image paths
+  const metadata = [];
+  const imagePaths = [];
+  const checksumEntries = [];
+
+  for (const entry of included) {
+    const record = await loadRecord(recordsDir, entry.record_id);
+    if (!record) continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const lane = detectLane(record.id, prompt, config.lanes);
+    const group = detectGroup(record.id, prompt, config.terminology);
+
+    // Build metadata row with requested fields
+    const row = {};
+    for (const field of profile.metadata_fields) {
+      if (field === 'id') row.id = record.id;
+      else if (field === 'asset_path') row.asset_path = record.asset_path;
+      else if (field === 'provenance') row.provenance = record.provenance;
+      else if (field === 'judgment') row.judgment = record.judgment;
+      else if (field === 'canon') row.canon = record.canon;
+      else if (field === 'identity') row.identity = record.identity || null;
+      else if (field === 'lineage') row.lineage = record.lineage || null;
+      else if (record[field] !== undefined) row[field] = record[field];
+    }
+    row.detected_lane = lane;
+    row.detected_group = group;
+    metadata.push(row);
+
+    // Track image for linking
+    if (profile.include_images && record.asset_path) {
+      imagePaths.push({
+        record_id: record.id,
+        source: join(projectRoot, record.asset_path),
+        filename: record.asset_path.split('/').pop(),
+      });
+    }
+  }
+
+  // Write metadata.jsonl
+  const metadataPath = join(exportDir, 'metadata.jsonl');
+  await writeFile(metadataPath, metadata.map(r => JSON.stringify(r)).join('\n') + '\n');
+
+  // Images directory
+  if (profile.include_images && imagePaths.length > 0) {
+    const imagesDir = join(exportDir, 'images');
+    await mkdir(imagesDir, { recursive: true });
+
+    let linked = 0;
+    for (const img of imagePaths) {
+      const dest = join(imagesDir, img.filename);
+      if (existsSync(img.source)) {
+        try {
+          if (options.copy) {
+            await copyFile(img.source, dest);
+          } else {
+            const relPath = relative(imagesDir, img.source);
+            await symlink(relPath, dest);
+          }
+          linked++;
+        } catch {
+          // Skip individual failures (e.g. duplicate filenames)
+        }
+      }
+    }
+  }
+
+  // Splits
+  if (profile.include_splits) {
+    const splitsDir = join(exportDir, 'splits');
+    await mkdir(splitsDir, { recursive: true });
+
+    for (const partition of ['train', 'val', 'test']) {
+      const entries = await loadSplitPartition(projectRoot, splitId, partition);
+      await writeFile(
+        join(splitsDir, `${partition}.jsonl`),
+        entries.map(e => JSON.stringify(e)).join('\n') + '\n'
+      );
+    }
+  }
+
+  // Dataset card
+  if (profile.include_card) {
+    const { markdown, json } = await generateCard(projectRoot, snapshotId, splitId);
+    await writeFile(join(exportDir, 'dataset-card.md'), markdown);
+    await writeFile(join(exportDir, 'dataset-card.json'), JSON.stringify(json, null, 2) + '\n');
+  }
+
+  // Checksums
+  if (profile.include_checksums) {
+    // Checksum all files in the export dir (except checksums.txt itself)
+    const allFiles = await collectFiles(exportDir);
+    for (const filePath of allFiles.sort()) {
+      const relPath = relative(exportDir, filePath);
+      if (relPath === 'checksums.txt') continue;
+      const hash = await sha256File(filePath);
+      checksumEntries.push(`SHA256 (${relPath}) = ${hash}`);
+    }
+    await writeFile(join(exportDir, 'checksums.txt'), checksumEntries.join('\n') + '\n');
+  }
+
+  // Manifest (written last — references everything)
+  const manifest = {
+    export_id: exportId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-export-v1',
+    project: config.meta.name,
+    snapshot_id: snapshotId,
+    split_id: splitId,
+    config_fingerprint: computeConfigFingerprint(projectRoot),
+    profile,
+    counts: {
+      records: metadata.length,
+      images: imagePaths.filter(i => existsSync(i.source)).length,
+      checksums: checksumEntries.length,
+    },
+  };
+  await writeFile(join(exportDir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+
+  // Summary
+  const summary = {
+    export_id: exportId,
+    snapshot_id: snapshotId,
+    split_id: splitId,
+    record_count: metadata.length,
+    image_count: manifest.counts.images,
+    partitions: {
+      train: split.counts.train,
+      val: split.counts.val,
+      test: split.counts.test,
+    },
+    config_fingerprint: manifest.config_fingerprint,
+  };
+  await writeFile(join(exportDir, 'summary.json'), JSON.stringify(summary, null, 2) + '\n');
+
+  return {
+    exportId,
+    recordCount: metadata.length,
+    imageCount: manifest.counts.images,
+  };
+}
+
+/**
+ * Recursively collect all files in a directory.
+ */
+async function collectFiles(dir, base = dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await collectFiles(full, base));
+    } else {
+      files.push(full);
+    }
+  }
+  return files;
+}
+
+/**
+ * List all exports in a project.
+ */
+export async function listExports(projectRoot) {
+  const exportsDir = join(projectRoot, 'exports');
+  if (!existsSync(exportsDir)) return [];
+
+  const entries = await readdir(exportsDir, { withFileTypes: true });
+  const exports = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('export-')) continue;
+    const manifestPath = join(exportsDir, entry.name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      exports.push({
+        id: manifest.export_id,
+        created_at: manifest.created_at,
+        snapshot_id: manifest.snapshot_id,
+        split_id: manifest.split_id,
+        records: manifest.counts.records,
+        images: manifest.counts.images,
+      });
+    } catch {
+      // Skip malformed
+    }
+  }
+
+  return exports.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}
+
+/**
+ * Load an export manifest.
+ */
+export async function loadExport(projectRoot, exportId) {
+  const path = join(projectRoot, 'exports', exportId, 'manifest.json');
+  if (!existsSync(path)) {
+    throw new Error(`Export "${exportId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}