@mcptoolshop/style-dataset-lab 2.0.0 → 2.2.0

package/lib/implementation-packs.js

@@ -0,0 +1,316 @@
+/**
+ * Implementation example packs.
+ *
+ * Shows how to use the trained asset in practice: prompt examples,
+ * lane-targeted examples, subject continuity examples, known failure
+ * cases, and re-ingest guidance.
+ *
+ * Every implementation pack is tied to a training manifest.
+ */
+
+import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { loadTrainingManifest } from './training-manifests.js';
+import { loadTrainingProfile } from './training-profiles.js';
+import { loadProjectConfig, detectLane, detectGroup } from './config.js';
+import { loadSplitPartition } from './split.js';
+import { loadRecord } from './records.js';
+
+function generateImplId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `impl-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Build an implementation pack from a training manifest.
+ *
+ * @param {string} projectRoot
+ * @param {string} manifestId
+ * @returns {Promise<{implId: string, prompts: number, failures: number, subjects: number}>}
+ */
+export async function buildImplementationPack(projectRoot, manifestId) {
+  const manifest = await loadTrainingManifest(projectRoot, manifestId);
+  const profile = await loadTrainingProfile(projectRoot, manifest.training_profile_id);
+  const config = loadProjectConfig(projectRoot);
+  const recordsDir = join(projectRoot, 'records');
+
+  // Load test partition records (best for implementation examples)
+  const testEntries = await loadSplitPartition(projectRoot, manifest.source_split_id, 'test');
+
+  // Also load some train records for broader coverage
+  const trainEntries = await loadSplitPartition(projectRoot, manifest.source_split_id, 'train');
+
+  // Build prompt examples — per-lane best records
+  const promptExamples = [];
+  const laneExamples = new Map();
+
+  for (const entry of [...testEntries, ...trainEntries.slice(0, 50)]) {
+    const record = await loadRecord(recordsDir, entry.record_id);
+    if (!record) continue;
+    if (record.judgment?.status !== 'approved') continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const lane = detectLane(record.id, prompt, config.lanes);
+    const group = detectGroup(record.id, prompt, config.terminology);
+
+    // Filter by profile lanes
+    if (profile.eligible_lanes.length > 0 && !profile.eligible_lanes.includes(lane)) continue;
+
+    if (!laneExamples.has(lane)) laneExamples.set(lane, []);
+    if (laneExamples.get(lane).length < 3) {
+      laneExamples.get(lane).push(record);
+
+      const passRatio = record.canon?.assertion_count > 0
+        ? record.canon.pass_count / record.canon.assertion_count : 0;
+
+      promptExamples.push({
+        record_id: record.id,
+        lane,
+        group: group || 'unknown',
+        prompt: prompt.slice(0, 500),
+        pass_ratio: +passRatio.toFixed(3),
+        asset_path: record.asset_path,
+        usage_note: `Use as ${lane} reference for ${group || 'general'} style`,
+      });
+    }
+  }
+
+  // Build known failure cases from rejected records
+  const knownFailures = [];
+  const allFiles = (await readdir(recordsDir)).filter(f => f.endsWith('.json')).sort();
+  let failCount = 0;
+
+  for (const file of allFiles) {
+    if (failCount >= 10) break;
+    const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+    if (record.judgment?.status !== 'rejected') continue;
+
+    const prompt = record.provenance?.prompt || '';
+    const lane = detectLane(record.id, prompt, config.lanes);
+    if (profile.eligible_lanes.length > 0 && !profile.eligible_lanes.includes(lane)) continue;
+
+    const failureModes = record.judgment?.failure_modes || [];
+    if (failureModes.length === 0 && !record.judgment?.explanation) continue;
+
+    knownFailures.push({
+      record_id: record.id,
+      lane,
+      failure_modes: failureModes,
+      explanation: record.judgment?.explanation || null,
+      lesson: `Avoid: ${failureModes.join(', ') || record.judgment?.explanation?.slice(0, 100)}`,
+    });
+    failCount++;
+  }
+
+  // Build subject continuity examples
+  const subjectExamples = [];
+  const bySubject = new Map();
+
+  for (const entry of [...testEntries, ...trainEntries]) {
+    const record = await loadRecord(recordsDir, entry.record_id);
+    if (!record?.identity?.subject_name) continue;
+    const name = record.identity.subject_name;
+    if (!bySubject.has(name)) bySubject.set(name, []);
+    bySubject.get(name).push(record);
+  }
+
+  for (const [name, records] of [...bySubject].sort((a, b) => b[1].length - a[1].length).slice(0, 5)) {
+    subjectExamples.push({
+      subject_name: name,
+      record_count: records.length,
+      record_ids: records.map(r => r.id).sort(),
+      faction: records[0].identity?.faction || null,
+      note: `${records.length} views of ${name} — use to verify identity consistency`,
+    });
+  }
+
+  // Build expected behaviors
+  const expectedBehaviors = {
+    style_markers: buildStyleMarkers(profile, config),
+    forbidden_patterns: buildForbiddenPatterns(config),
+    quality_thresholds: config.rubric.thresholds || {},
+  };
+
+  // Write implementation pack
+  const implId = generateImplId();
+  const implDir = join(projectRoot, 'training', 'implementations', implId);
+  await mkdir(implDir, { recursive: true });
+
+  const implManifest = {
+    implementation_pack_id: implId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-implementation-v1',
+    training_manifest_id: manifestId,
+    training_profile_id: manifest.training_profile_id,
+    counts: {
+      prompt_examples: promptExamples.length,
+      known_failures: knownFailures.length,
+      subject_examples: subjectExamples.length,
+      lanes_covered: laneExamples.size,
+    },
+  };
+
+  await writeFile(join(implDir, 'manifest.json'), JSON.stringify(implManifest, null, 2) + '\n');
+  await writeFile(join(implDir, 'prompts.jsonl'), promptExamples.map(p => JSON.stringify(p)).join('\n') + '\n');
+  await writeFile(join(implDir, 'known-failures.json'), JSON.stringify(knownFailures, null, 2) + '\n');
+  await writeFile(join(implDir, 'expected-behaviors.json'), JSON.stringify(expectedBehaviors, null, 2) + '\n');
+
+  if (subjectExamples.length > 0) {
+    await writeFile(join(implDir, 'subject-continuity.json'), JSON.stringify(subjectExamples, null, 2) + '\n');
+  }
+
+  // Write eval tasks JSONL — tasks that can be run against generated outputs
+  const evalTasks = promptExamples.map(p => ({
+    task: 'generate_and_compare',
+    prompt: p.prompt,
+    lane: p.lane,
+    reference_record: p.record_id,
+    expected: 'style-consistent output matching canon',
+  }));
+  await writeFile(join(implDir, 'eval-tasks.jsonl'), evalTasks.map(t => JSON.stringify(t)).join('\n') + '\n');
+
+  // Write reingest guide
+  const reingestGuide = generateReingestGuide(profile, manifest);
+  await writeFile(join(implDir, 'reingest-guide.md'), reingestGuide);
+
+  // Write README
+  const readme = generateImplReadme(implManifest, profile, manifest);
+  await writeFile(join(implDir, 'README.md'), readme);
+
+  return {
+    implId,
+    prompts: promptExamples.length,
+    failures: knownFailures.length,
+    subjects: subjectExamples.length,
+  };
+}
+
+function buildStyleMarkers(profile, config) {
+  const markers = [];
+  const constitution = config.constitution;
+  if (constitution?.rules) {
+    for (const rule of constitution.rules.slice(0, 10)) {
+      markers.push({ rule_id: rule.id, summary: rule.summary || rule.id });
+    }
+  }
+  return markers;
+}
+
+function buildForbiddenPatterns(config) {
+  const forbidden = [];
+  const rubric = config.rubric;
+  if (rubric?.failure_to_rules) {
+    for (const [mode, rules] of Object.entries(rubric.failure_to_rules)) {
+      forbidden.push({ failure_mode: mode, linked_rules: rules });
+    }
+  }
+  return forbidden;
+}
+
+function generateReingestGuide(profile, manifest) {
+  return `# Re-ingest Guide
+
+## When to re-ingest
+
+After using the trained ${profile.asset_type} (${profile.label}) to generate new images,
+accepted outputs should be re-ingested into the project as new records.
+
+## How to re-ingest
+
+\`\`\`bash
+sdlab reingest generated --project <name> --source <outputs-dir>
+\`\`\`
+
+## Rules
+
+1. Generated outputs enter as new records with \`provenance.source: "generated"\`
+2. They must go through normal curation (\`sdlab curate\`)
+3. They must be canon-bound (\`sdlab bind\`)
+4. No bypass around review — generated work is judged like everything else
+5. Accepted re-ingested records become eligible for future snapshots
+
+## Provenance fields
+
+Re-ingested records carry:
+
+- \`provenance.source\`: \`"generated"\`
+- \`provenance.training_manifest_id\`: \`"${manifest.training_manifest_id}"\`
+- \`provenance.training_profile_id\`: \`"${manifest.training_profile_id}"\`
+- \`provenance.adapter_target\`: \`"${manifest.adapter_target}"\`
+- \`provenance.base_model\`: \`"${manifest.base_model || '(not specified)'}"\`
+`;
+}
+
+function generateImplReadme(impl, profile, manifest) {
+  return `# Implementation Pack: ${impl.implementation_pack_id}
+
+**Profile:** ${profile.label} (${profile.profile_id})
+**Training manifest:** ${manifest.training_manifest_id}
+**Created:** ${impl.created_at}
+
+## Contents
+
+| File | Description |
+|------|-------------|
+| prompts.jsonl | ${impl.counts.prompt_examples} prompt examples across ${impl.counts.lanes_covered} lanes |
+| known-failures.json | ${impl.counts.known_failures} documented failure cases to avoid |
+| expected-behaviors.json | Style markers and forbidden patterns from constitution |
+| subject-continuity.json | ${impl.counts.subject_examples} named-subject groups for identity testing |
+| eval-tasks.jsonl | Generate-and-compare tasks for automated evaluation |
+| reingest-guide.md | How to re-ingest accepted generated outputs |
+
+## Usage
+
+1. Load the trained ${profile.asset_type} with the prompt strategy from the profile
+2. Use prompts.jsonl as starting points for each lane
+3. Check generated outputs against expected-behaviors.json
+4. Compare with known-failures.json to catch common drift
+5. For named subjects, verify against subject-continuity.json
+6. Re-ingest accepted outputs per reingest-guide.md
+`;
+}
+
+/**
+ * List all implementation packs.
+ */
+export async function listImplementationPacks(projectRoot) {
+  const dir = join(projectRoot, 'training', 'implementations');
+  if (!existsSync(dir)) return [];
+
+  const entries = await readdir(dir, { withFileTypes: true });
+  const packs = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('impl-')) continue;
+    const manifestPath = join(dir, entry.name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const data = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      packs.push({
+        id: data.implementation_pack_id,
+        created_at: data.created_at,
+        manifest: data.training_manifest_id,
+        profile: data.training_profile_id,
+        prompts: data.counts.prompt_examples,
+        failures: data.counts.known_failures,
+      });
+    } catch { /* skip */ }
+  }
+
+  return packs.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}
+
+/**
+ * Load an implementation pack manifest.
+ */
+export async function loadImplementationPack(projectRoot, implId) {
+  const path = join(projectRoot, 'training', 'implementations', implId, 'manifest.json');
+  if (!existsSync(path)) {
+    throw new Error(`Implementation pack "${implId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}

package/lib/reingest.js

@@ -0,0 +1,131 @@
+/**
+ * Re-ingest engine for generated outputs.
+ *
+ * Accepted generated outputs re-enter the project as new records
+ * with provenance fields linking back to the training manifest.
+ * No bypass around review — generated work must be curated and
+ * canon-bound like everything else.
+ */
+
+import { readFile, writeFile, readdir, stat } from 'node:fs/promises';
+import { join, basename, extname } from 'node:path';
+import { existsSync } from 'node:fs';
+import { loadTrainingManifest } from './training-manifests.js';
+import { buildBaseRecord } from './records.js';
+
+const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
+
+/**
+ * Re-ingest generated outputs from a directory into the project.
+ *
+ * Each image in the source directory becomes a new record with:
+ * - provenance.source = "generated"
+ * - provenance.training_manifest_id = <manifest-id>
+ * - judgment = null (must be reviewed)
+ * - canon = null (must be bound)
+ *
+ * @param {string} projectRoot
+ * @param {string} sourcePath — directory containing generated images
+ * @param {string} manifestId — training manifest that produced them
+ * @param {Object} options — { dryRun? }
+ * @returns {Promise<{created: number, skipped: number, records: string[]}>}
+ */
+export async function reingestGenerated(projectRoot, sourcePath, manifestId, options = {}) {
+  const manifest = await loadTrainingManifest(projectRoot, manifestId);
+  const recordsDir = join(projectRoot, 'records');
+
+  // Find image files in source directory
+  const entries = await readdir(sourcePath);
+  const imageFiles = entries
+    .filter(f => IMAGE_EXTENSIONS.has(extname(f).toLowerCase()))
+    .sort();
+
+  const created = [];
+  const skipped = [];
+
+  for (const file of imageFiles) {
+    const stem = basename(file, extname(file));
+    const recordId = `gen_${stem}`;
+    const recordPath = join(recordsDir, `${recordId}.json`);
+
+    // Skip if record already exists
+    if (existsSync(recordPath)) {
+      skipped.push(recordId);
+      continue;
+    }
+
+    const filePath = join(sourcePath, file);
+    const fileStat = await stat(filePath);
+
+    // Relative asset path from project root
+    const assetPath = `outputs/candidates/${file}`;
+
+    const provenance = {
+      source: 'generated',
+      training_manifest_id: manifestId,
+      training_profile_id: manifest.training_profile_id,
+      adapter_target: manifest.adapter_target,
+      base_model: manifest.base_model || null,
+      source_export_id: manifest.source_export_id,
+      generated_at: new Date().toISOString(),
+      original_path: filePath,
+    };
+
+    const record = buildBaseRecord(recordId, assetPath, {
+      width: null,
+      height: null,
+      bytes: fileStat.size,
+    }, provenance);
+
+    record.schema_version = '2.1.0';
+
+    if (!options.dryRun) {
+      await writeFile(recordPath, JSON.stringify(record, null, 2) + '\n');
+    }
+
+    created.push(recordId);
+  }
+
+  return { created: created.length, skipped: skipped.length, records: created };
+}
+
+/**
+ * Audit re-ingested records — check that they've been properly reviewed.
+ *
+ * @param {string} projectRoot
+ * @returns {Promise<{total: number, reviewed: number, unreviewed: number, bound: number, unbound: number}>}
+ */
+export async function auditReingest(projectRoot) {
+  const recordsDir = join(projectRoot, 'records');
+  const files = (await readdir(recordsDir))
+    .filter(f => f.startsWith('gen_') && f.endsWith('.json'))
+    .sort();
+
+  let reviewed = 0;
+  let unreviewed = 0;
+  let bound = 0;
+  let unbound = 0;
+
+  for (const file of files) {
+    const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+
+    if (record.judgment) {
+      reviewed++;
+      if (record.canon?.assertions?.length > 0) {
+        bound++;
+      } else {
+        unbound++;
+      }
+    } else {
+      unreviewed++;
+    }
+  }
+
+  return {
+    total: files.length,
+    reviewed,
+    unreviewed,
+    bound,
+    unbound,
+  };
+}

package/lib/snapshot.js

@@ -0,0 +1,211 @@
+/**
+ * Snapshot creation and management.
+ *
+ * A snapshot is a frozen, versioned selection of records from a project
+ * at a point in time. Once created, it never silently changes.
+ */
+
+import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
+import { readFileSync, existsSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { createHash, randomBytes } from 'node:crypto';
+import { loadProjectConfig, loadSelectionProfile, detectLane, detectGroup } from './config.js';
+import { evaluateEligibility } from './eligibility.js';
+
+/**
+ * Generate a snapshot ID: snap-YYYYMMDD-HHMMSS-XXXX
+ */
+function generateSnapshotId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14); // YYYYMMDDHHMMSS
+  const rand = randomBytes(2).toString('hex');
+  return `snap-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Compute config fingerprint — SHA-256 of all 5 config files concatenated.
+ * Deterministic: files read in alphabetical order.
+ */
+export function computeConfigFingerprint(projectRoot) {
+  const configFiles = [
+    'constitution.json',
+    'lanes.json',
+    'project.json',
+    'rubric.json',
+    'terminology.json',
+  ];
+
+  const hash = createHash('sha256');
+  for (const file of configFiles) {
+    const filePath = join(projectRoot, file);
+    if (existsSync(filePath)) {
+      hash.update(`--- ${file} ---\n`);
+      hash.update(readFileSync(filePath, 'utf-8'));
+      hash.update('\n');
+    }
+  }
+  return hash.digest('hex');
+}
+
+/**
+ * Create a snapshot — frozen selection of eligible records.
+ * @param {string} projectRoot — absolute path to project
+ * @param {Object} profile — selection profile
+ * @returns {Promise<{snapshotId: string, included: number, excluded: number}>}
+ */
+export async function createSnapshot(projectRoot, profile) {
+  const config = loadProjectConfig(projectRoot);
+  const recordsDir = join(projectRoot, 'records');
+  const snapshotId = generateSnapshotId();
+  const snapshotDir = join(projectRoot, 'snapshots', snapshotId);
+
+  // Load all records (sorted for deterministic output)
+  const files = (await readdir(recordsDir))
+    .filter(f => f.endsWith('.json'))
+    .sort();
+
+  const included = [];
+  const excluded = [];
+  const laneDistribution = {};
+  const factionDistribution = {};
+  const verdictDistribution = { pass: 0, fail: 0, partial: 0 };
+
+  for (const file of files) {
+    const raw = await readFile(join(recordsDir, file), 'utf-8');
+    const record = JSON.parse(raw);
+
+    const { eligible, reasons } = evaluateEligibility(
+      record, profile, config.lanes, config.terminology
+    );
+
+    if (eligible) {
+      const prompt = record.provenance?.prompt || '';
+      const lane = detectLane(record.id, prompt, config.lanes);
+      const faction = detectGroup(record.id, prompt, config.terminology);
+
+      // Build reason trace
+      const passRatio = record.canon?.assertion_count > 0
+        ? `${record.canon.pass_count}/${record.canon.assertion_count}`
+        : 'n/a';
+      const reason = `${record.judgment.status}, canon-bound, pass_ratio=${passRatio}, lane=${lane}`;
+
+      included.push({ record_id: record.id, reason });
+
+      // Track distributions
+      laneDistribution[lane] = (laneDistribution[lane] || 0) + 1;
+      factionDistribution[faction || 'unknown'] = (factionDistribution[faction || 'unknown'] || 0) + 1;
+      if (record.canon?.assertions) {
+        for (const a of record.canon.assertions) {
+          verdictDistribution[a.verdict] = (verdictDistribution[a.verdict] || 0) + 1;
+        }
+      }
+    } else {
+      excluded.push({ record_id: record.id, reasons });
+    }
+  }
+
+  // Write snapshot files
+  await mkdir(snapshotDir, { recursive: true });
+
+  const manifest = {
+    snapshot_id: snapshotId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-snapshot-v1',
+    project_name: config.meta.name,
+    frozen: true,
+    selection_profile: profile,
+    config_fingerprint: computeConfigFingerprint(projectRoot),
+    config_versions: {
+      constitution: config.constitution.version || '1.0.0',
+      project: config.meta.version || '0.0.0',
+    },
+    counts: {
+      total_records: files.length,
+      evaluated: files.length,
+      included: included.length,
+      excluded: excluded.length,
+    },
+  };
+
+  const summary = {
+    snapshot_id: snapshotId,
+    lane_distribution: laneDistribution,
+    faction_distribution: factionDistribution,
+    verdict_distribution: verdictDistribution,
+  };
+
+  await writeFile(join(snapshotDir, 'snapshot.json'), JSON.stringify(manifest, null, 2) + '\n');
+  await writeFile(join(snapshotDir, 'included.jsonl'), included.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(snapshotDir, 'excluded.jsonl'), excluded.map(r => JSON.stringify(r)).join('\n') + '\n');
+  await writeFile(join(snapshotDir, 'summary.json'), JSON.stringify(summary, null, 2) + '\n');
+
+  return { snapshotId, included: included.length, excluded: excluded.length };
+}
+
+/**
+ * Load a snapshot manifest.
+ */
+export async function loadSnapshot(projectRoot, snapshotId) {
+  const path = join(projectRoot, 'snapshots', snapshotId, 'snapshot.json');
+  if (!existsSync(path)) {
+    throw new Error(`Snapshot "${snapshotId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}
+
+/**
+ * Load included record IDs from a snapshot.
+ */
+export async function loadSnapshotIncluded(projectRoot, snapshotId) {
+  const path = join(projectRoot, 'snapshots', snapshotId, 'included.jsonl');
+  const raw = await readFile(path, 'utf-8');
+  return raw.trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
+}
+
+/**
+ * List all snapshots in a project.
+ */
+export async function listSnapshots(projectRoot) {
+  const snapshotsDir = join(projectRoot, 'snapshots');
+  if (!existsSync(snapshotsDir)) return [];
+
+  const entries = await readdir(snapshotsDir, { withFileTypes: true });
+  const snapshots = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('snap-')) continue;
+    try {
+      const manifest = JSON.parse(
+        await readFile(join(snapshotsDir, entry.name, 'snapshot.json'), 'utf-8')
+      );
+      snapshots.push({
+        id: manifest.snapshot_id,
+        created_at: manifest.created_at,
+        included: manifest.counts.included,
+        excluded: manifest.counts.excluded,
+        fingerprint: manifest.config_fingerprint.slice(0, 12),
+      });
+    } catch {
+      // Skip malformed snapshots
+    }
+  }
+
+  return snapshots.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}
+
+/**
+ * Diff two snapshots — records added, removed, and shared.
+ */
+export async function diffSnapshots(projectRoot, idA, idB) {
+  const includedA = await loadSnapshotIncluded(projectRoot, idA);
+  const includedB = await loadSnapshotIncluded(projectRoot, idB);
+
+  const setA = new Set(includedA.map(r => r.record_id));
+  const setB = new Set(includedB.map(r => r.record_id));
+
+  const added = [...setB].filter(id => !setA.has(id));
+  const removed = [...setA].filter(id => !setB.has(id));
+  const shared = [...setA].filter(id => setB.has(id));
+
+  return { added, removed, shared, countA: setA.size, countB: setB.size };
+}