@mcptoolshop/style-dataset-lab 2.0.0 → 2.1.0

package/lib/eval-pack.js

@@ -0,0 +1,283 @@
+/**
+ * Canon-aware eval pack builder.
+ *
+ * Eval packs are curated test instruments that verify a model's
+ * relationship with project canon. Four task types:
+ *
+ * 1. Lane coverage — representative approved records per lane
+ * 2. Forbidden drift — rejected/borderline records paired with violated rules
+ * 3. Anchor/gold — highest pass-ratio records per faction
+ * 4. Subject continuity — same-subject records for identity testing
+ */
+
+import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { loadProjectConfig, detectLane, detectGroup } from './config.js';
+import { loadRecord } from './records.js';
+
+/**
+ * Generate an eval-pack ID: eval-YYYYMMDD-HHMMSS-XXXX
+ */
+function generateEvalId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `eval-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Build an eval pack from project records.
+ *
+ * @param {string} projectRoot
+ * @param {Object} options — { maxPerLane?, maxPerFaction?, maxDrift?, maxContinuity? }
+ * @returns {Promise<{evalId: string, tasks: Object}>}
+ */
+export async function buildEvalPack(projectRoot, options = {}) {
+  const config = loadProjectConfig(projectRoot);
+  const recordsDir = join(projectRoot, 'records');
+  const files = (await readdir(recordsDir)).filter(f => f.endsWith('.json')).sort();
+
+  const maxPerLane = options.maxPerLane || 5;
+  const maxPerFaction = options.maxPerFaction || 3;
+  const maxDrift = options.maxDrift || 20;
+  const maxContinuity = options.maxContinuity || 10;
+
+  // Load all records
+  const records = [];
+  for (const file of files) {
+    const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+    records.push(record);
+  }
+
+  // Task 1: Lane coverage
+  const laneCoverage = buildLaneCoverage(records, config, maxPerLane);
+
+  // Task 2: Forbidden drift
+  const forbiddenDrift = buildForbiddenDrift(records, config, maxDrift);
+
+  // Task 3: Anchor/gold
+  const anchorGold = buildAnchorGold(records, config, maxPerFaction);
+
+  // Task 4: Subject continuity
+  const subjectContinuity = buildSubjectContinuity(records, maxContinuity);
+
+  // Write eval pack
+  const evalId = generateEvalId();
+  const evalDir = join(projectRoot, 'eval-packs', evalId);
+  await mkdir(evalDir, { recursive: true });
+
+  const manifest = {
+    eval_id: evalId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-eval-v1',
+    project: config.meta.name,
+    task_counts: {
+      lane_coverage: laneCoverage.length,
+      forbidden_drift: forbiddenDrift.length,
+      anchor_gold: anchorGold.length,
+      subject_continuity: subjectContinuity.length,
+    },
+    total_records: laneCoverage.length + forbiddenDrift.length +
+      anchorGold.length + subjectContinuity.length,
+  };
+
+  await writeFile(join(evalDir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+  await writeFile(join(evalDir, 'lane-coverage.jsonl'), laneCoverage.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(evalDir, 'forbidden-drift.jsonl'), forbiddenDrift.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(evalDir, 'anchor-gold.jsonl'), anchorGold.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(evalDir, 'subject-continuity.jsonl'), subjectContinuity.map(r => JSON.stringify(r)).join('\n') + '\n');
+
+  return {
+    evalId,
+    tasks: manifest.task_counts,
+  };
+}
+
+/**
+ * Task 1: Lane coverage — best approved record per lane.
+ */
+function buildLaneCoverage(records, config, maxPerLane) {
+  const byLane = new Map();
+
+  for (const record of records) {
+    if (record.judgment?.status !== 'approved') continue;
+    if (!record.canon?.assertions?.length) continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const lane = detectLane(record.id, prompt, config.lanes);
+    if (!byLane.has(lane)) byLane.set(lane, []);
+    byLane.get(lane).push(record);
+  }
+
+  const result = [];
+  for (const [lane, recs] of [...byLane].sort((a, b) => a[0].localeCompare(b[0]))) {
+    // Sort by pass ratio descending
+    const sorted = recs.sort((a, b) => {
+      const ratioA = a.canon.assertion_count > 0 ? a.canon.pass_count / a.canon.assertion_count : 0;
+      const ratioB = b.canon.assertion_count > 0 ? b.canon.pass_count / b.canon.assertion_count : 0;
+      return ratioB - ratioA;
+    });
+
+    for (const record of sorted.slice(0, maxPerLane)) {
+      result.push({
+        task: 'lane_coverage',
+        record_id: record.id,
+        lane,
+        pass_ratio: record.canon.assertion_count > 0
+          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : 0,
+        asset_path: record.asset_path,
+      });
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Task 2: Forbidden drift — rejected/borderline records with violated rules.
+ */
+function buildForbiddenDrift(records, config, maxDrift) {
+  const candidates = [];
+
+  for (const record of records) {
+    const status = record.judgment?.status;
+    if (status !== 'rejected' && status !== 'borderline') continue;
+
+    // Collect failure modes from judgment
+    const failureModes = record.judgment?.failure_modes || [];
+    // Collect failed canon assertions
+    const failedAssertions = (record.canon?.assertions || [])
+      .filter(a => a.verdict === 'fail')
+      .map(a => a.rule_id || a.rule);
+
+    const violations = [...new Set([...failureModes, ...failedAssertions])];
+    if (violations.length === 0 && !record.judgment?.explanation) continue;
+
+    candidates.push({
+      task: 'forbidden_drift',
+      record_id: record.id,
+      status,
+      violations,
+      explanation: record.judgment?.explanation || null,
+      asset_path: record.asset_path,
+    });
+  }
+
+  // Sort by number of violations descending (most instructive first)
+  candidates.sort((a, b) => b.violations.length - a.violations.length);
+  return candidates.slice(0, maxDrift);
+}
+
+/**
+ * Task 3: Anchor/gold — highest pass-ratio records per faction.
+ */
+function buildAnchorGold(records, config, maxPerFaction) {
+  const byFaction = new Map();
+
+  for (const record of records) {
+    if (record.judgment?.status !== 'approved') continue;
+    if (!record.canon?.assertions?.length) continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const faction = detectGroup(record.id, prompt, config.terminology) || 'unknown';
+    if (!byFaction.has(faction)) byFaction.set(faction, []);
+    byFaction.get(faction).push(record);
+  }
+
+  const result = [];
+  for (const [faction, recs] of [...byFaction].sort((a, b) => a[0].localeCompare(b[0]))) {
+    const sorted = recs.sort((a, b) => {
+      const ratioA = a.canon.assertion_count > 0 ? a.canon.pass_count / a.canon.assertion_count : 0;
+      const ratioB = b.canon.assertion_count > 0 ? b.canon.pass_count / b.canon.assertion_count : 0;
+      return ratioB - ratioA;
+    });
+
+    for (const record of sorted.slice(0, maxPerFaction)) {
+      result.push({
+        task: 'anchor_gold',
+        record_id: record.id,
+        faction,
+        pass_ratio: record.canon.assertion_count > 0
+          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : 0,
+        assertion_count: record.canon.assertion_count,
+        asset_path: record.asset_path,
+      });
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Task 4: Subject continuity — groups of same-subject records for identity testing.
+ */
+function buildSubjectContinuity(records, maxGroups) {
+  const bySubject = new Map();
+
+  for (const record of records) {
+    const subject = record.identity?.subject_name;
+    if (!subject) continue;
+    if (!bySubject.has(subject)) bySubject.set(subject, []);
+    bySubject.get(subject).push(record);
+  }
+
+  const result = [];
+  // Sort subjects by record count descending (richest clusters first)
+  const sorted = [...bySubject].sort((a, b) => b[1].length - a[1].length);
+
+  for (const [subject, recs] of sorted.slice(0, maxGroups)) {
+    result.push({
+      task: 'subject_continuity',
+      subject_name: subject,
+      record_count: recs.length,
+      record_ids: recs.map(r => r.id).sort(),
+      subject_type: recs[0].identity?.subject_type || 'unknown',
+      faction: recs[0].identity?.faction || null,
+    });
+  }
+
+  return result;
+}
+
+/**
+ * List all eval packs in a project.
+ */
+export async function listEvalPacks(projectRoot) {
+  const evalDir = join(projectRoot, 'eval-packs');
+  if (!existsSync(evalDir)) return [];
+
+  const entries = await readdir(evalDir, { withFileTypes: true });
+  const packs = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('eval-')) continue;
+    const manifestPath = join(evalDir, entry.name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      packs.push({
+        id: manifest.eval_id,
+        created_at: manifest.created_at,
+        total: manifest.total_records,
+        tasks: manifest.task_counts,
+      });
+    } catch {
+      // Skip malformed
+    }
+  }
+
+  return packs.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}
+
+/**
+ * Load an eval pack manifest.
+ */
+export async function loadEvalPack(projectRoot, evalId) {
+  const path = join(projectRoot, 'eval-packs', evalId, 'manifest.json');
+  if (!existsSync(path)) {
+    throw new Error(`Eval pack "${evalId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}

package/lib/export.js

@@ -0,0 +1,259 @@
+/**
+ * Export package builder — self-contained, reproducible dataset exports.
+ *
+ * An export package contains everything needed to use or rebuild the dataset:
+ * manifest, metadata, images (symlinked or copied), splits, card, and checksums.
+ */
+
+import { readFile, writeFile, mkdir, readdir, symlink, copyFile, stat } from 'node:fs/promises';
+import { join, relative, resolve } from 'node:path';
+import { existsSync } from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { loadProjectConfig, loadExportProfile, detectLane, detectGroup } from './config.js';
+import { loadSnapshot, loadSnapshotIncluded, computeConfigFingerprint } from './snapshot.js';
+import { loadSplit, loadSplitPartition } from './split.js';
+import { loadRecord } from './records.js';
+import { generateCard } from './card.js';
+
+/**
+ * Generate an export ID: export-YYYYMMDD-HHMMSS-XXXX
+ */
+function generateExportId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `export-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Compute SHA-256 of a file.
+ */
+async function sha256File(filePath) {
+  const data = await readFile(filePath);
+  return createHash('sha256').update(data).digest('hex');
+}
+
+/**
+ * Build an export package.
+ *
+ * @param {string} projectRoot
+ * @param {string} snapshotId
+ * @param {string} splitId
+ * @param {Object} options — { profileId?, copy? }
+ * @returns {Promise<{exportId: string, recordCount: number, imageCount: number}>}
+ */
+export async function buildExport(projectRoot, snapshotId, splitId, options = {}) {
+  const config = loadProjectConfig(projectRoot);
+  const profile = loadExportProfile(projectRoot, options.profileId || null);
+  const snapshot = await loadSnapshot(projectRoot, snapshotId);
+  const split = await loadSplit(projectRoot, splitId);
+
+  // Load all included records
+  const included = await loadSnapshotIncluded(projectRoot, snapshotId);
+  const recordsDir = join(projectRoot, 'records');
+
+  const exportId = generateExportId();
+  const exportDir = join(projectRoot, 'exports', exportId);
+  await mkdir(exportDir, { recursive: true });
+
+  // Build metadata + collect image paths
+  const metadata = [];
+  const imagePaths = [];
+  const checksumEntries = [];
+
+  for (const entry of included) {
+    const record = await loadRecord(recordsDir, entry.record_id);
+    if (!record) continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const lane = detectLane(record.id, prompt, config.lanes);
+    const group = detectGroup(record.id, prompt, config.terminology);
+
+    // Build metadata row with requested fields
+    const row = {};
+    for (const field of profile.metadata_fields) {
+      if (field === 'id') row.id = record.id;
+      else if (field === 'asset_path') row.asset_path = record.asset_path;
+      else if (field === 'provenance') row.provenance = record.provenance;
+      else if (field === 'judgment') row.judgment = record.judgment;
+      else if (field === 'canon') row.canon = record.canon;
+      else if (field === 'identity') row.identity = record.identity || null;
+      else if (field === 'lineage') row.lineage = record.lineage || null;
+      else if (record[field] !== undefined) row[field] = record[field];
+    }
+    row.detected_lane = lane;
+    row.detected_group = group;
+    metadata.push(row);
+
+    // Track image for linking
+    if (profile.include_images && record.asset_path) {
+      imagePaths.push({
+        record_id: record.id,
+        source: join(projectRoot, record.asset_path),
+        filename: record.asset_path.split('/').pop(),
+      });
+    }
+  }
+
+  // Write metadata.jsonl
+  const metadataPath = join(exportDir, 'metadata.jsonl');
+  await writeFile(metadataPath, metadata.map(r => JSON.stringify(r)).join('\n') + '\n');
+
+  // Images directory
+  if (profile.include_images && imagePaths.length > 0) {
+    const imagesDir = join(exportDir, 'images');
+    await mkdir(imagesDir, { recursive: true });
+
+    let linked = 0;
+    for (const img of imagePaths) {
+      const dest = join(imagesDir, img.filename);
+      if (existsSync(img.source)) {
+        try {
+          if (options.copy) {
+            await copyFile(img.source, dest);
+          } else {
+            const relPath = relative(imagesDir, img.source);
+            await symlink(relPath, dest);
+          }
+          linked++;
+        } catch {
+          // Skip individual failures (e.g. duplicate filenames)
+        }
+      }
+    }
+  }
+
+  // Splits
+  if (profile.include_splits) {
+    const splitsDir = join(exportDir, 'splits');
+    await mkdir(splitsDir, { recursive: true });
+
+    for (const partition of ['train', 'val', 'test']) {
+      const entries = await loadSplitPartition(projectRoot, splitId, partition);
+      await writeFile(
+        join(splitsDir, `${partition}.jsonl`),
+        entries.map(e => JSON.stringify(e)).join('\n') + '\n'
+      );
+    }
+  }
+
+  // Dataset card
+  if (profile.include_card) {
+    const { markdown, json } = await generateCard(projectRoot, snapshotId, splitId);
+    await writeFile(join(exportDir, 'dataset-card.md'), markdown);
+    await writeFile(join(exportDir, 'dataset-card.json'), JSON.stringify(json, null, 2) + '\n');
+  }
+
+  // Checksums
+  if (profile.include_checksums) {
+    // Checksum all files in the export dir (except checksums.txt itself)
+    const allFiles = await collectFiles(exportDir);
+    for (const filePath of allFiles.sort()) {
+      const relPath = relative(exportDir, filePath);
+      if (relPath === 'checksums.txt') continue;
+      const hash = await sha256File(filePath);
+      checksumEntries.push(`SHA256 (${relPath}) = ${hash}`);
+    }
+    await writeFile(join(exportDir, 'checksums.txt'), checksumEntries.join('\n') + '\n');
+  }
+
+  // Manifest (written last — references everything)
+  const manifest = {
+    export_id: exportId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-export-v1',
+    project: config.meta.name,
+    snapshot_id: snapshotId,
+    split_id: splitId,
+    config_fingerprint: computeConfigFingerprint(projectRoot),
+    profile,
+    counts: {
+      records: metadata.length,
+      images: imagePaths.filter(i => existsSync(i.source)).length,
+      checksums: checksumEntries.length,
+    },
+  };
+  await writeFile(join(exportDir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+
+  // Summary
+  const summary = {
+    export_id: exportId,
+    snapshot_id: snapshotId,
+    split_id: splitId,
+    record_count: metadata.length,
+    image_count: manifest.counts.images,
+    partitions: {
+      train: split.counts.train,
+      val: split.counts.val,
+      test: split.counts.test,
+    },
+    config_fingerprint: manifest.config_fingerprint,
+  };
+  await writeFile(join(exportDir, 'summary.json'), JSON.stringify(summary, null, 2) + '\n');
+
+  return {
+    exportId,
+    recordCount: metadata.length,
+    imageCount: manifest.counts.images,
+  };
+}
+
+/**
+ * Recursively collect all files in a directory.
+ */
+async function collectFiles(dir, base = dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await collectFiles(full, base));
+    } else {
+      files.push(full);
+    }
+  }
+  return files;
+}
+
+/**
+ * List all exports in a project.
+ */
+export async function listExports(projectRoot) {
+  const exportsDir = join(projectRoot, 'exports');
+  if (!existsSync(exportsDir)) return [];
+
+  const entries = await readdir(exportsDir, { withFileTypes: true });
+  const exports = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('export-')) continue;
+    const manifestPath = join(exportsDir, entry.name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      exports.push({
+        id: manifest.export_id,
+        created_at: manifest.created_at,
+        snapshot_id: manifest.snapshot_id,
+        split_id: manifest.split_id,
+        records: manifest.counts.records,
+        images: manifest.counts.images,
+      });
+    } catch {
+      // Skip malformed
+    }
+  }
+
+  return exports.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}
+
+/**
+ * Load an export manifest.
+ */
+export async function loadExport(projectRoot, exportId) {
+  const path = join(projectRoot, 'exports', exportId, 'manifest.json');
+  if (!existsSync(path)) {
+    throw new Error(`Export "${exportId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}