@mcptoolshop/style-dataset-lab 2.0.0 → 2.2.0

package/lib/card.js

@@ -0,0 +1,145 @@
+/**
+ * Dataset card generation — markdown + JSON twin.
+ *
+ * Produces a human-readable dataset card summarizing provenance,
+ * selection criteria, split strategy, and distribution statistics.
+ */
+
+import { loadSnapshot, loadSnapshotIncluded } from './snapshot.js';
+import { loadSplit, loadSplitAudit } from './split.js';
+import { loadProjectConfig } from './config.js';
+
+/**
+ * Generate a dataset card from snapshot + split + project config.
+ *
+ * @param {string} projectRoot
+ * @param {string} snapshotId
+ * @param {string} splitId
+ * @returns {Promise<{markdown: string, json: Object}>}
+ */
+export async function generateCard(projectRoot, snapshotId, splitId) {
+  const config = loadProjectConfig(projectRoot);
+  const snapshot = await loadSnapshot(projectRoot, snapshotId);
+  const split = await loadSplit(projectRoot, splitId);
+  const audit = await loadSplitAudit(projectRoot, splitId);
+  const included = await loadSnapshotIncluded(projectRoot, snapshotId);
+
+  const json = {
+    dataset_name: `${config.meta.name}-visual`,
+    version: config.meta.version || '1.0.0',
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-card-v1',
+    project: {
+      name: config.meta.name,
+      display_name: config.meta.display_name,
+      domain: config.meta.domain,
+    },
+    snapshot: {
+      id: snapshot.snapshot_id,
+      created_at: snapshot.created_at,
+      config_fingerprint: snapshot.config_fingerprint,
+      total_records: snapshot.counts.total_records,
+      included: snapshot.counts.included,
+      excluded: snapshot.counts.excluded,
+    },
+    selection: snapshot.selection_profile,
+    split: {
+      id: split.split_id,
+      strategy: split.profile.strategy,
+      seed: split.profile.seed,
+      ratios: {
+        train: split.profile.train_ratio,
+        val: split.profile.val_ratio,
+        test: split.profile.test_ratio,
+      },
+      counts: split.counts,
+      leakage_free: audit.leakage_check.passed,
+    },
+    lane_balance: audit.lane_balance,
+    constitution: {
+      rule_count: config.constitution.rules?.length || 0,
+    },
+    rubric: {
+      dimension_count: config.rubric.dimensions?.length || 0,
+      thresholds: config.rubric.thresholds,
+    },
+  };
+
+  const markdown = renderMarkdown(json, config);
+
+  return { markdown, json };
+}
+
+function renderMarkdown(card, config) {
+  const lines = [];
+  const add = (s = '') => lines.push(s);
+
+  add(`# ${card.dataset_name}`);
+  add();
+  add(`**Project:** ${card.project.display_name}  `);
+  add(`**Domain:** ${card.project.domain}  `);
+  add(`**Version:** ${card.version}  `);
+  add(`**Generated:** ${card.created_at}  `);
+  add();
+
+  add('## Selection');
+  add();
+  add(`- **Snapshot:** \`${card.snapshot.id}\``);
+  add(`- **Config fingerprint:** \`${card.snapshot.config_fingerprint.slice(0, 16)}...\``);
+  add(`- **Total records evaluated:** ${card.snapshot.total_records}`);
+  add(`- **Included:** ${card.snapshot.included} (${(card.snapshot.included / card.snapshot.total_records * 100).toFixed(1)}%)`);
+  add(`- **Excluded:** ${card.snapshot.excluded}`);
+  add();
+
+  add('### Selection criteria');
+  add();
+  const sel = card.selection;
+  if (sel.require_judgment) add('- Requires human judgment');
+  if (sel.require_status?.length) add(`- Status: ${sel.require_status.join(', ')}`);
+  if (sel.require_canon_bound) add('- Must be canon-bound (constitution assertions)');
+  if (sel.minimum_pass_ratio != null) add(`- Minimum pass ratio: ${(sel.minimum_pass_ratio * 100).toFixed(0)}%`);
+  add();
+
+  add('## Split');
+  add();
+  add(`- **Split ID:** \`${card.split.id}\``);
+  add(`- **Strategy:** ${card.split.strategy}`);
+  add(`- **Seed:** ${card.split.seed}`);
+  add(`- **Subject leakage:** ${card.split.leakage_free ? 'None (verified)' : 'DETECTED'}`);
+  add();
+
+  add('| Partition | Records | % |');
+  add('|-----------|---------|---|');
+  const total = card.split.counts.total_records;
+  add(`| Train | ${card.split.counts.train} | ${(card.split.counts.train / total * 100).toFixed(1)}% |`);
+  add(`| Val | ${card.split.counts.val} | ${(card.split.counts.val / total * 100).toFixed(1)}% |`);
+  add(`| Test | ${card.split.counts.test} | ${(card.split.counts.test / total * 100).toFixed(1)}% |`);
+  add();
+
+  add('## Lane Balance');
+  add();
+  add('| Lane | Total | Train% | Val% | Test% |');
+  add('|------|-------|--------|------|-------|');
+  for (const [lane, b] of Object.entries(card.lane_balance).sort((a, b) => b[1].total - a[1].total)) {
+    add(`| ${lane} | ${b.total} | ${b.train_pct}% | ${b.val_pct}% | ${b.test_pct}% |`);
+  }
+  add();
+
+  add('## Quality Gates');
+  add();
+  add(`- **Constitution rules:** ${card.constitution.rule_count}`);
+  add(`- **Rubric dimensions:** ${card.rubric.dimension_count}`);
+  if (card.rubric.thresholds) {
+    add(`- **Pass threshold:** ${(card.rubric.thresholds.pass * 100).toFixed(0)}%`);
+  }
+  add();
+
+  add('## Provenance');
+  add();
+  add('This dataset was produced by [Style Dataset Lab](https://github.com/mcp-tool-shop-org/style-dataset-lab).');
+  add('Every included record has been human-reviewed, canon-bound, and verified against project constitution rules.');
+  add('Split strategy ensures no subject appears in multiple partitions.');
+  add();
+
+  return lines.join('\n');
+}

package/lib/config.js

@@ -102,3 +102,147 @@ export function loadProjectConfig(projectRoot) {
     terminology: loadTerminology(projectRoot),
   };
 }
+
+// ─── Profile Loaders ─────────────────────────────────────────────────
+
+const DEFAULT_SELECTION_PROFILE = {
+  require_judgment: true,
+  require_status: ['approved'],
+  require_canon_bound: true,
+  minimum_pass_ratio: 0.5,
+  rights_filter: null,
+  exclude_lanes: [],
+  max_per_subject: null,
+  max_per_lane: null,
+};
+
+const DEFAULT_SPLIT_PROFILE = {
+  profile_id: 'balanced-default',
+  strategy: 'subject-isolated',
+  train_ratio: 0.8,
+  val_ratio: 0.1,
+  test_ratio: 0.1,
+  lane_balance: true,
+  subject_isolation: true,
+  seed: 42,
+};
+
+const DEFAULT_EXPORT_PROFILE = {
+  profile_id: 'canonical-package',
+  include_images: true,
+  image_strategy: 'symlink',
+  include_splits: true,
+  include_card: true,
+  include_checksums: true,
+  metadata_fields: ['id', 'asset_path', 'provenance', 'judgment', 'canon', 'identity', 'lineage'],
+};
+
+/**
+ * Load a selection profile from project or return built-in default.
+ */
+export function loadSelectionProfile(projectRoot, profileId) {
+  if (!profileId) return { ...DEFAULT_SELECTION_PROFILE };
+  const data = loadJsonFile(join(projectRoot, 'selection-profiles', `${profileId}.json`));
+  if (!data) return { ...DEFAULT_SELECTION_PROFILE };
+  return { ...DEFAULT_SELECTION_PROFILE, ...data };
+}
+
+/**
+ * Load a split profile from project or return built-in default.
+ */
+export function loadSplitProfile(projectRoot, profileId) {
+  if (!profileId) return { ...DEFAULT_SPLIT_PROFILE };
+  const data = loadJsonFile(join(projectRoot, 'split-profiles', `${profileId}.json`));
+  if (!data) return { ...DEFAULT_SPLIT_PROFILE };
+  return { ...DEFAULT_SPLIT_PROFILE, ...data };
+}
+
+/**
+ * Load an export profile from project or return built-in default.
+ */
+export function loadExportProfile(projectRoot, profileId) {
+  if (!profileId) return { ...DEFAULT_EXPORT_PROFILE };
+  const data = loadJsonFile(join(projectRoot, 'export-profiles', `${profileId}.json`));
+  if (!data) return { ...DEFAULT_EXPORT_PROFILE };
+  return { ...DEFAULT_EXPORT_PROFILE, ...data };
+}
+
+// ─── Detection Functions ─────────────────────────────────────────────
+// Extracted from scripts/canon-bind.js for reuse in eligibility,
+// snapshot, and split modules.
+
+/**
+ * Detect lane from asset ID using project lane config.
+ * Tests each lane's id_patterns as regex against the lowercase ID.
+ * Returns the first match, or the default_lane.
+ */
+export function detectLane(id, prompt, lanesConfig) {
+  const lower = id.toLowerCase();
+  for (const lane of lanesConfig.lanes) {
+    if (!lane.id_patterns || lane.id_patterns.length === 0) continue;
+    for (const pattern of lane.id_patterns) {
+      if (new RegExp(pattern).test(lower)) return lane.id;
+    }
+  }
+  return lanesConfig.default_lane;
+}
+
+/**
+ * Detect group (faction) from asset ID and prompt using terminology config.
+ * Uses explicit id_detection_order and prompt_detection_order when provided
+ * to preserve the exact detection priority of the original implementation.
+ */
+export function detectGroup(id, prompt, termConfig) {
+  const idOrder = termConfig.id_detection_order || Object.keys(termConfig.groups);
+  const promptOrder = termConfig.prompt_detection_order || Object.keys(termConfig.groups);
+
+  // 1. Primary ID patterns per group (id_detection_order)
+  for (const groupName of idOrder) {
+    const group = termConfig.groups[groupName];
+    if (!group?.id_patterns) continue;
+    for (const pattern of group.id_patterns) {
+      if (new RegExp(pattern).test(id)) return groupName;
+    }
+  }
+
+  // 2. Cross-group patterns
+  for (const [pattern, behavior] of Object.entries(termConfig.cross_group_patterns || {})) {
+    if (new RegExp(pattern).test(id)) {
+      if (behavior === 'detect_from_id_suffix') {
+        for (const groupName of idOrder) {
+          if (id.includes(groupName)) return groupName;
+        }
+      }
+      return null;
+    }
+  }
+
+  // 3. Edge defaults
+  for (const [pattern, groupName] of Object.entries(termConfig.edge_defaults || {})) {
+    if (new RegExp(pattern).test(id)) return groupName;
+  }
+
+  // 4. Prompt-based fallback (prompt_detection_order)
+  if (prompt) {
+    const p = prompt.toLowerCase();
+    for (const groupName of promptOrder) {
+      const group = termConfig.groups[groupName];
+      if (!group?.prompt_patterns || group.prompt_patterns.length === 0) continue;
+      for (const keyword of group.prompt_patterns) {
+        if (new RegExp(keyword).test(p)) return groupName;
+      }
+    }
+  }
+
+  // 5. Null-faction patterns (generic rejects — no faction)
+  for (const pattern of termConfig.null_faction_patterns || []) {
+    if (new RegExp(pattern).test(id)) return null;
+  }
+
+  // 6. ID fallback patterns
+  for (const [pattern, groupName] of Object.entries(termConfig.id_fallbacks || {})) {
+    if (new RegExp(pattern).test(id)) return groupName;
+  }
+
+  return null;
+}

package/lib/eligibility.js

@@ -0,0 +1,110 @@
+/**
+ * Eligibility evaluation engine.
+ *
+ * Determines whether each record qualifies for inclusion in a dataset
+ * snapshot, with explicit reason traces for every decision.
+ */
+
+import { detectLane, detectGroup } from './config.js';
+
+/**
+ * Evaluate a single record against a selection profile.
+ * @param {Object} record — full record object
+ * @param {Object} profile — selection profile
+ * @param {Object} lanesConfig — from lanes.json
+ * @param {Object} termConfig — from terminology.json
+ * @returns {{ eligible: boolean, reasons: string[] }}
+ */
+export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
+  const reasons = [];
+
+  // 1. Judgment required
+  if (profile.require_judgment) {
+    if (!record.judgment) {
+      reasons.push('no judgment (record has not been reviewed)');
+    }
+  }
+
+  // 2. Status filter
+  if (profile.require_status?.length > 0 && record.judgment) {
+    if (!profile.require_status.includes(record.judgment.status)) {
+      reasons.push(`status "${record.judgment.status}" not in [${profile.require_status.join(', ')}]`);
+    }
+  }
+
+  // 3. Canon bound
+  if (profile.require_canon_bound) {
+    if (!record.canon?.assertions || record.canon.assertions.length === 0) {
+      reasons.push('not canon-bound (no assertions)');
+    }
+  }
+
+  // 4. Minimum pass ratio
+  if (profile.minimum_pass_ratio != null && record.canon?.assertion_count > 0) {
+    const ratio = record.canon.pass_count / record.canon.assertion_count;
+    if (ratio < profile.minimum_pass_ratio) {
+      reasons.push(`pass ratio ${(ratio * 100).toFixed(0)}% below minimum ${(profile.minimum_pass_ratio * 100).toFixed(0)}%`);
+    }
+  }
+
+  // 5. Lane exclusion
+  if (profile.exclude_lanes?.length > 0) {
+    const prompt = record.provenance?.prompt || '';
+    const lane = detectLane(record.id, prompt, lanesConfig);
+    if (profile.exclude_lanes.includes(lane)) {
+      reasons.push(`lane "${lane}" is excluded`);
+    }
+  }
+
+  // 6. Rights filter
+  if (profile.rights_filter != null) {
+    const rights = record.rights_status || record.training_rights_status || null;
+    if (rights !== profile.rights_filter) {
+      reasons.push(`rights "${rights || 'unknown'}" does not match required "${profile.rights_filter}"`);
+    }
+  }
+
+  // 7. Tag exclusion
+  if (profile.exclude_tags?.length > 0 && record.dataset_tags) {
+    const excluded = record.dataset_tags.filter(t => profile.exclude_tags.includes(t));
+    if (excluded.length > 0) {
+      reasons.push(`has excluded tags: ${excluded.join(', ')}`);
+    }
+  }
+
+  return {
+    eligible: reasons.length === 0,
+    reasons,
+  };
+}
+
+/**
+ * Categorize exclusion reasons for a set of records.
+ * @param {Array<{record_id: string, reasons: string[]}>} excluded
+ * @returns {Object} — category → count mapping
+ */
+export function categorizeExclusions(excluded) {
+  const categories = {
+    no_judgment: 0,
+    wrong_status: 0,
+    not_canon_bound: 0,
+    low_pass_ratio: 0,
+    excluded_lane: 0,
+    rights_filter: 0,
+    excluded_tags: 0,
+  };
+
+  for (const { reasons } of excluded) {
+    for (const r of reasons) {
+      if (r.includes('no judgment')) categories.no_judgment++;
+      else if (r.includes('status')) categories.wrong_status++;
+      else if (r.includes('canon-bound')) categories.not_canon_bound++;
+      else if (r.includes('pass ratio')) categories.low_pass_ratio++;
+      else if (r.includes('lane')) categories.excluded_lane++;
+      else if (r.includes('rights')) categories.rights_filter++;
+      else if (r.includes('tags')) categories.excluded_tags++;
+    }
+  }
+
+  return categories;
+}

package/lib/eval-pack.js

@@ -0,0 +1,283 @@
+/**
+ * Canon-aware eval pack builder.
+ *
+ * Eval packs are curated test instruments that verify a model's
+ * relationship with project canon. Four task types:
+ *
+ * 1. Lane coverage — representative approved records per lane
+ * 2. Forbidden drift — rejected/borderline records paired with violated rules
+ * 3. Anchor/gold — highest pass-ratio records per faction
+ * 4. Subject continuity — same-subject records for identity testing
+ */
+
+import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { loadProjectConfig, detectLane, detectGroup } from './config.js';
+import { loadRecord } from './records.js';
+
+/**
+ * Generate an eval-pack ID: eval-YYYYMMDD-HHMMSS-XXXX
+ */
+function generateEvalId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `eval-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Build an eval pack from project records.
+ *
+ * @param {string} projectRoot
+ * @param {Object} options — { maxPerLane?, maxPerFaction?, maxDrift?, maxContinuity? }
+ * @returns {Promise<{evalId: string, tasks: Object}>}
+ */
+export async function buildEvalPack(projectRoot, options = {}) {
+  const config = loadProjectConfig(projectRoot);
+  const recordsDir = join(projectRoot, 'records');
+  const files = (await readdir(recordsDir)).filter(f => f.endsWith('.json')).sort();
+
+  const maxPerLane = options.maxPerLane || 5;
+  const maxPerFaction = options.maxPerFaction || 3;
+  const maxDrift = options.maxDrift || 20;
+  const maxContinuity = options.maxContinuity || 10;
+
+  // Load all records
+  const records = [];
+  for (const file of files) {
+    const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+    records.push(record);
+  }
+
+  // Task 1: Lane coverage
+  const laneCoverage = buildLaneCoverage(records, config, maxPerLane);
+
+  // Task 2: Forbidden drift
+  const forbiddenDrift = buildForbiddenDrift(records, config, maxDrift);
+
+  // Task 3: Anchor/gold
+  const anchorGold = buildAnchorGold(records, config, maxPerFaction);
+
+  // Task 4: Subject continuity
+  const subjectContinuity = buildSubjectContinuity(records, maxContinuity);
+
+  // Write eval pack
+  const evalId = generateEvalId();
+  const evalDir = join(projectRoot, 'eval-packs', evalId);
+  await mkdir(evalDir, { recursive: true });
+
+  const manifest = {
+    eval_id: evalId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-eval-v1',
+    project: config.meta.name,
+    task_counts: {
+      lane_coverage: laneCoverage.length,
+      forbidden_drift: forbiddenDrift.length,
+      anchor_gold: anchorGold.length,
+      subject_continuity: subjectContinuity.length,
+    },
+    total_records: laneCoverage.length + forbiddenDrift.length +
+      anchorGold.length + subjectContinuity.length,
+  };
+
+  await writeFile(join(evalDir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
+  await writeFile(join(evalDir, 'lane-coverage.jsonl'), laneCoverage.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(evalDir, 'forbidden-drift.jsonl'), forbiddenDrift.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(evalDir, 'anchor-gold.jsonl'), anchorGold.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(evalDir, 'subject-continuity.jsonl'), subjectContinuity.map(r => JSON.stringify(r)).join('\n') + '\n');
+
+  return {
+    evalId,
+    tasks: manifest.task_counts,
+  };
+}
+
+/**
+ * Task 1: Lane coverage — best approved record per lane.
+ */
+function buildLaneCoverage(records, config, maxPerLane) {
+  const byLane = new Map();
+
+  for (const record of records) {
+    if (record.judgment?.status !== 'approved') continue;
+    if (!record.canon?.assertions?.length) continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const lane = detectLane(record.id, prompt, config.lanes);
+    if (!byLane.has(lane)) byLane.set(lane, []);
+    byLane.get(lane).push(record);
+  }
+
+  const result = [];
+  for (const [lane, recs] of [...byLane].sort((a, b) => a[0].localeCompare(b[0]))) {
+    // Sort by pass ratio descending
+    const sorted = recs.sort((a, b) => {
+      const ratioA = a.canon.assertion_count > 0 ? a.canon.pass_count / a.canon.assertion_count : 0;
+      const ratioB = b.canon.assertion_count > 0 ? b.canon.pass_count / b.canon.assertion_count : 0;
+      return ratioB - ratioA;
+    });
+
+    for (const record of sorted.slice(0, maxPerLane)) {
+      result.push({
+        task: 'lane_coverage',
+        record_id: record.id,
+        lane,
+        pass_ratio: record.canon.assertion_count > 0
+          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : 0,
+        asset_path: record.asset_path,
+      });
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Task 2: Forbidden drift — rejected/borderline records with violated rules.
+ */
+function buildForbiddenDrift(records, config, maxDrift) {
+  const candidates = [];
+
+  for (const record of records) {
+    const status = record.judgment?.status;
+    if (status !== 'rejected' && status !== 'borderline') continue;
+
+    // Collect failure modes from judgment
+    const failureModes = record.judgment?.failure_modes || [];
+    // Collect failed canon assertions
+    const failedAssertions = (record.canon?.assertions || [])
+      .filter(a => a.verdict === 'fail')
+      .map(a => a.rule_id || a.rule);
+
+    const violations = [...new Set([...failureModes, ...failedAssertions])];
+    if (violations.length === 0 && !record.judgment?.explanation) continue;
+
+    candidates.push({
+      task: 'forbidden_drift',
+      record_id: record.id,
+      status,
+      violations,
+      explanation: record.judgment?.explanation || null,
+      asset_path: record.asset_path,
+    });
+  }
+
+  // Sort by number of violations descending (most instructive first)
+  candidates.sort((a, b) => b.violations.length - a.violations.length);
+  return candidates.slice(0, maxDrift);
+}
+
+/**
+ * Task 3: Anchor/gold — highest pass-ratio records per faction.
+ */
+function buildAnchorGold(records, config, maxPerFaction) {
+  const byFaction = new Map();
+
+  for (const record of records) {
+    if (record.judgment?.status !== 'approved') continue;
+    if (!record.canon?.assertions?.length) continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const faction = detectGroup(record.id, prompt, config.terminology) || 'unknown';
+    if (!byFaction.has(faction)) byFaction.set(faction, []);
+    byFaction.get(faction).push(record);
+  }
+
+  const result = [];
+  for (const [faction, recs] of [...byFaction].sort((a, b) => a[0].localeCompare(b[0]))) {
+    const sorted = recs.sort((a, b) => {
+      const ratioA = a.canon.assertion_count > 0 ? a.canon.pass_count / a.canon.assertion_count : 0;
+      const ratioB = b.canon.assertion_count > 0 ? b.canon.pass_count / b.canon.assertion_count : 0;
+      return ratioB - ratioA;
+    });
+
+    for (const record of sorted.slice(0, maxPerFaction)) {
+      result.push({
+        task: 'anchor_gold',
+        record_id: record.id,
+        faction,
+        pass_ratio: record.canon.assertion_count > 0
+          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : 0,
+        assertion_count: record.canon.assertion_count,
+        asset_path: record.asset_path,
+      });
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Task 4: Subject continuity — groups of same-subject records for identity testing.
+ */
+function buildSubjectContinuity(records, maxGroups) {
+  const bySubject = new Map();
+
+  for (const record of records) {
+    const subject = record.identity?.subject_name;
+    if (!subject) continue;
+    if (!bySubject.has(subject)) bySubject.set(subject, []);
+    bySubject.get(subject).push(record);
+  }
+
+  const result = [];
+  // Sort subjects by record count descending (richest clusters first)
+  const sorted = [...bySubject].sort((a, b) => b[1].length - a[1].length);
+
+  for (const [subject, recs] of sorted.slice(0, maxGroups)) {
+    result.push({
+      task: 'subject_continuity',
+      subject_name: subject,
+      record_count: recs.length,
+      record_ids: recs.map(r => r.id).sort(),
+      subject_type: recs[0].identity?.subject_type || 'unknown',
+      faction: recs[0].identity?.faction || null,
+    });
+  }
+
+  return result;
+}
+
+/**
+ * List all eval packs in a project.
+ */
+export async function listEvalPacks(projectRoot) {
+  const evalDir = join(projectRoot, 'eval-packs');
+  if (!existsSync(evalDir)) return [];
+
+  const entries = await readdir(evalDir, { withFileTypes: true });
+  const packs = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('eval-')) continue;
+    const manifestPath = join(evalDir, entry.name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      packs.push({
+        id: manifest.eval_id,
+        created_at: manifest.created_at,
+        total: manifest.total_records,
+        tasks: manifest.task_counts,
+      });
+    } catch {
+      // Skip malformed
+    }
+  }
+
+  return packs.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}
+
+/**
+ * Load an eval pack manifest.
+ */
+export async function loadEvalPack(projectRoot, evalId) {
+  const path = join(projectRoot, 'eval-packs', evalId, 'manifest.json');
+  if (!existsSync(path)) {
+    throw new Error(`Eval pack "${evalId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}