@mcptoolshop/style-dataset-lab 2.1.0 → 2.2.0

package/lib/reingest.js

@@ -0,0 +1,131 @@
+/**
+ * Re-ingest engine for generated outputs.
+ *
+ * Accepted generated outputs re-enter the project as new records
+ * with provenance fields linking back to the training manifest.
+ * No bypass around review — generated work must be curated and
+ * canon-bound like everything else.
+ */
+
+import { readFile, writeFile, readdir, stat } from 'node:fs/promises';
+import { join, basename, extname } from 'node:path';
+import { existsSync } from 'node:fs';
+import { loadTrainingManifest } from './training-manifests.js';
+import { buildBaseRecord } from './records.js';
+
+const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
+
+/**
+ * Re-ingest generated outputs from a directory into the project.
+ *
+ * Each image in the source directory becomes a new record with:
+ * - provenance.source = "generated"
+ * - provenance.training_manifest_id = <manifest-id>
+ * - judgment = null (must be reviewed)
+ * - canon = null (must be bound)
+ *
+ * @param {string} projectRoot
+ * @param {string} sourcePath — directory containing generated images
+ * @param {string} manifestId — training manifest that produced them
+ * @param {Object} options — { dryRun? }
+ * @returns {Promise<{created: number, skipped: number, records: string[]}>}
+ */
+export async function reingestGenerated(projectRoot, sourcePath, manifestId, options = {}) {
+  const manifest = await loadTrainingManifest(projectRoot, manifestId);
+  const recordsDir = join(projectRoot, 'records');
+
+  // Find image files in source directory
+  const entries = await readdir(sourcePath);
+  const imageFiles = entries
+    .filter(f => IMAGE_EXTENSIONS.has(extname(f).toLowerCase()))
+    .sort();
+
+  const created = [];
+  const skipped = [];
+
+  for (const file of imageFiles) {
+    const stem = basename(file, extname(file));
+    const recordId = `gen_${stem}`;
+    const recordPath = join(recordsDir, `${recordId}.json`);
+
+    // Skip if record already exists
+    if (existsSync(recordPath)) {
+      skipped.push(recordId);
+      continue;
+    }
+
+    const filePath = join(sourcePath, file);
+    const fileStat = await stat(filePath);
+
+    // Relative asset path from project root
+    const assetPath = `outputs/candidates/${file}`;
+
+    const provenance = {
+      source: 'generated',
+      training_manifest_id: manifestId,
+      training_profile_id: manifest.training_profile_id,
+      adapter_target: manifest.adapter_target,
+      base_model: manifest.base_model || null,
+      source_export_id: manifest.source_export_id,
+      generated_at: new Date().toISOString(),
+      original_path: filePath,
+    };
+
+    const record = buildBaseRecord(recordId, assetPath, {
+      width: null,
+      height: null,
+      bytes: fileStat.size,
+    }, provenance);
+
+    record.schema_version = '2.1.0';
+
+    if (!options.dryRun) {
+      await writeFile(recordPath, JSON.stringify(record, null, 2) + '\n');
+    }
+
+    created.push(recordId);
+  }
+
+  return { created: created.length, skipped: skipped.length, records: created };
+}
+
+/**
+ * Audit re-ingested records — check that they've been properly reviewed.
+ *
+ * @param {string} projectRoot
+ * @returns {Promise<{total: number, reviewed: number, unreviewed: number, bound: number, unbound: number}>}
+ */
+export async function auditReingest(projectRoot) {
+  const recordsDir = join(projectRoot, 'records');
+  const files = (await readdir(recordsDir))
+    .filter(f => f.startsWith('gen_') && f.endsWith('.json'))
+    .sort();
+
+  let reviewed = 0;
+  let unreviewed = 0;
+  let bound = 0;
+  let unbound = 0;
+
+  for (const file of files) {
+    const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+
+    if (record.judgment) {
+      reviewed++;
+      if (record.canon?.assertions?.length > 0) {
+        bound++;
+      } else {
+        unbound++;
+      }
+    } else {
+      unreviewed++;
+    }
+  }
+
+  return {
+    total: files.length,
+    reviewed,
+    unreviewed,
+    bound,
+    unbound,
+  };
+}

package/lib/training-manifests.js

@@ -0,0 +1,172 @@
+/**
+ * Training manifest management.
+ *
+ * A training manifest is a frozen, versioned contract for one training attempt.
+ * It captures the exact export package, split, profile, and config fingerprint
+ * so the training run is fully reproducible.
+ */
+
+import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { loadExport } from './export.js';
+import { loadTrainingProfile } from './training-profiles.js';
+import { computeConfigFingerprint } from './snapshot.js';
+import { listEvalPacks } from './eval-pack.js';
+
+/**
+ * Generate a training manifest ID: tm-YYYYMMDD-HHMMSS-XXXX
+ */
+function generateManifestId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `tm-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+/**
+ * Compute SHA-256 of a file's contents.
+ */
+async function hashFile(filePath) {
+  const data = await readFile(filePath);
+  return createHash('sha256').update(data).digest('hex');
+}
+
+/**
+ * Create a training manifest from an export package and training profile.
+ *
+ * @param {string} projectRoot
+ * @param {string} exportId
+ * @param {string} profileId
+ * @param {Object} options — { baseModel?, adapterTarget?, hyperparameters?, captionMode? }
+ * @returns {Promise<{manifestId: string, exportId: string, profileId: string}>}
+ */
+export async function createTrainingManifest(projectRoot, exportId, profileId, options = {}) {
+  const profile = await loadTrainingProfile(projectRoot, profileId);
+  const exportManifest = await loadExport(projectRoot, exportId);
+
+  // Hash the export manifest for integrity
+  const exportManifestPath = join(projectRoot, 'exports', exportId, 'manifest.json');
+  const exportManifestHash = await hashFile(exportManifestPath);
+
+  // Determine adapter target
+  const adapterTarget = options.adapterTarget || profile.adapter_targets[0];
+  if (!profile.adapter_targets.includes(adapterTarget)) {
+    throw new Error(
+      `Adapter target "${adapterTarget}" not in profile's adapter_targets: [${profile.adapter_targets.join(', ')}]`
+    );
+  }
+
+  // Find linked eval packs
+  const evalPacks = await listEvalPacks(projectRoot);
+  const linkedEvalPackIds = evalPacks.map(p => p.id);
+
+  const manifestId = generateManifestId();
+  const manifest = {
+    training_manifest_id: manifestId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-training-manifest-v1',
+    project_id: exportManifest.project,
+    training_profile_id: profileId,
+    source_export_id: exportId,
+    source_snapshot_id: exportManifest.snapshot_id,
+    source_split_id: exportManifest.split_id,
+    config_fingerprint: computeConfigFingerprint(projectRoot),
+    export_manifest_hash: exportManifestHash,
+    dataset_counts: exportManifest.counts,
+    base_model: options.baseModel || profile.base_model_recommendations?.[0] || null,
+    adapter_target: adapterTarget,
+    hyperparameters: options.hyperparameters || {},
+    caption_mode: options.captionMode || profile.caption_strategy || 'filename',
+    expected_outputs: {
+      asset_type: profile.asset_type,
+      target_family: profile.target_family,
+    },
+    linked_eval_pack_ids: linkedEvalPackIds,
+  };
+
+  // Write manifest
+  const dir = join(projectRoot, 'training', 'manifests');
+  await mkdir(dir, { recursive: true });
+  await writeFile(join(dir, `${manifestId}.json`), JSON.stringify(manifest, null, 2) + '\n');
+
+  return { manifestId, exportId, profileId };
+}
+
+/**
+ * Validate a training manifest against current project state.
+ * Checks that referenced artifacts still exist and fingerprints match.
+ *
+ * @returns {{ valid: boolean, issues: string[] }}
+ */
+export async function validateTrainingManifest(projectRoot, manifestId) {
+  const manifest = await loadTrainingManifest(projectRoot, manifestId);
+  const issues = [];
+
+  // Check export exists
+  const exportPath = join(projectRoot, 'exports', manifest.source_export_id, 'manifest.json');
+  if (!existsSync(exportPath)) {
+    issues.push(`source export "${manifest.source_export_id}" not found`);
+  } else {
+    // Check export hash
+    const currentHash = await hashFile(exportPath);
+    if (currentHash !== manifest.export_manifest_hash) {
+      issues.push(`export manifest hash mismatch — export may have been modified`);
+    }
+  }
+
+  // Check config fingerprint
+  const currentFingerprint = computeConfigFingerprint(projectRoot);
+  if (currentFingerprint !== manifest.config_fingerprint) {
+    issues.push(`config fingerprint mismatch — project config has changed since manifest creation`);
+  }
+
+  // Check training profile exists
+  const profilePath = join(projectRoot, 'training', 'profiles', `${manifest.training_profile_id}.json`);
+  if (!existsSync(profilePath)) {
+    issues.push(`training profile "${manifest.training_profile_id}" not found`);
+  }
+
+  return { valid: issues.length === 0, issues };
+}
+
+/**
+ * Load a training manifest.
+ */
+export async function loadTrainingManifest(projectRoot, manifestId) {
+  const path = join(projectRoot, 'training', 'manifests', `${manifestId}.json`);
+  if (!existsSync(path)) {
+    throw new Error(`Training manifest "${manifestId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}
+
+/**
+ * List all training manifests in a project.
+ */
+export async function listTrainingManifests(projectRoot) {
+  const dir = join(projectRoot, 'training', 'manifests');
+  if (!existsSync(dir)) return [];
+
+  const files = (await readdir(dir)).filter(f => f.endsWith('.json')).sort();
+  const manifests = [];
+
+  for (const file of files) {
+    try {
+      const data = JSON.parse(await readFile(join(dir, file), 'utf-8'));
+      manifests.push({
+        id: data.training_manifest_id,
+        created_at: data.created_at,
+        profile: data.training_profile_id,
+        export_id: data.source_export_id,
+        adapter: data.adapter_target,
+        records: data.dataset_counts?.records || 0,
+      });
+    } catch {
+      // Skip malformed
+    }
+  }
+
+  return manifests.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}

package/lib/training-packages.js

@@ -0,0 +1,241 @@
+/**
+ * Training package builder.
+ *
+ * Builds trainer-ready dataset packages from training manifests.
+ * Adapters transform the canonical layout for specific training targets
+ * but never mutate inclusion or split truth.
+ */
+
+import { readFile, writeFile, mkdir, readdir, symlink, copyFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { existsSync } from 'node:fs';
+import { createHash, randomBytes } from 'node:crypto';
+import { loadTrainingManifest } from './training-manifests.js';
+import { loadTrainingProfile } from './training-profiles.js';
+import { loadSplitPartition } from './split.js';
+import { loadRecord } from './records.js';
+import { loadProjectConfig, detectLane, detectGroup } from './config.js';
+
+/**
+ * Generate a training package ID: tp-YYYYMMDD-HHMMSS-XXXX
+ */
+function generatePackageId() {
+  const now = new Date();
+  const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
+  const rand = randomBytes(2).toString('hex');
+  return `tp-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
+}
+
+async function sha256File(filePath) {
+  const data = await readFile(filePath);
+  return createHash('sha256').update(data).digest('hex');
+}
+
+/**
+ * Load an adapter module by name.
+ */
+async function loadAdapter(adapterTarget) {
+  const adapterPath = `./adapters/${adapterTarget}.js`;
+  try {
+    return await import(adapterPath);
+  } catch {
+    throw new Error(
+      `Adapter "${adapterTarget}" not found. Available: generic-image-caption, diffusers-lora`
+    );
+  }
+}
+
+/**
+ * Build a training package from a manifest.
+ *
+ * @param {string} projectRoot
+ * @param {string} manifestId
+ * @param {Object} options — { copy?, adapterOverride? }
+ * @returns {Promise<{packageId: string, records: number, images: number}>}
+ */
+export async function buildTrainingPackage(projectRoot, manifestId, options = {}) {
+  const manifest = await loadTrainingManifest(projectRoot, manifestId);
+  const profile = await loadTrainingProfile(projectRoot, manifest.training_profile_id);
+  const config = loadProjectConfig(projectRoot);
+  const adapterTarget = options.adapterOverride || manifest.adapter_target;
+  const adapter = await loadAdapter(adapterTarget);
+
+  const recordsDir = join(projectRoot, 'records');
+  const packageId = generatePackageId();
+  const packageDir = join(projectRoot, 'training', 'packages', packageId);
+
+  // Load split partitions
+  const partitions = {};
+  let totalRecords = 0;
+  let totalImages = 0;
+
+  for (const partition of ['train', 'val', 'test']) {
+    const entries = await loadSplitPartition(projectRoot, manifest.source_split_id, partition);
+    const records = [];
+
+    for (const entry of entries) {
+      const record = await loadRecord(recordsDir, entry.record_id);
+      if (!record) continue;
+
+      // Filter by profile's eligible lanes
+      const prompt = record.provenance?.prompt || '';
+      const lane = detectLane(record.id, prompt, config.lanes);
+      if (profile.eligible_lanes.length > 0 && !profile.eligible_lanes.includes(lane)) {
+        continue;
+      }
+
+      const group = detectGroup(record.id, prompt, config.terminology);
+      records.push({ record, lane, group });
+    }
+
+    partitions[partition] = records;
+    totalRecords += records.length;
+  }
+
+  // Let the adapter build the package layout
+  const adapterResult = await adapter.buildPackage({
+    packageDir,
+    partitions,
+    profile,
+    manifest,
+    config,
+    projectRoot,
+    copy: options.copy || false,
+  });
+
+  totalImages = adapterResult.imageCount || 0;
+
+  // Generate checksums for all files in the package
+  const checksumEntries = [];
+  const allFiles = await collectFiles(packageDir);
+  for (const filePath of allFiles.sort()) {
+    const relPath = relative(packageDir, filePath);
+    if (relPath === 'checksums.txt') continue;
+    const hash = await sha256File(filePath);
+    checksumEntries.push(`SHA256 (${relPath}) = ${hash}`);
+  }
+  await writeFile(join(packageDir, 'checksums.txt'), checksumEntries.join('\n') + '\n');
+
+  // Write package manifest (last — references everything)
+  const packageManifest = {
+    training_package_id: packageId,
+    created_at: new Date().toISOString(),
+    created_by: 'sdlab-training-package-v1',
+    training_manifest_id: manifestId,
+    training_profile_id: manifest.training_profile_id,
+    adapter_target: adapterTarget,
+    source_export_id: manifest.source_export_id,
+    source_snapshot_id: manifest.source_snapshot_id,
+    source_split_id: manifest.source_split_id,
+    config_fingerprint: manifest.config_fingerprint,
+    counts: {
+      total_records: totalRecords,
+      train: partitions.train.length,
+      val: partitions.val.length,
+      test: partitions.test.length,
+      images: totalImages,
+      checksums: checksumEntries.length,
+    },
+  };
+  await writeFile(join(packageDir, 'manifest.json'), JSON.stringify(packageManifest, null, 2) + '\n');
+
+  // Write README
+  const readme = generatePackageReadme(packageManifest, profile, manifest, adapterTarget);
+  await writeFile(join(packageDir, 'README.md'), readme);
+
+  return { packageId, records: totalRecords, images: totalImages };
+}
+
+function generatePackageReadme(pkg, profile, manifest, adapter) {
+  return `# Training Package: ${pkg.training_package_id}
+
+**Profile:** ${profile.label} (${profile.profile_id})
+**Adapter:** ${adapter}
+**Target:** ${profile.target_family} / ${profile.asset_type}
+**Created:** ${pkg.created_at}
+
+## Source chain
+
+- Training manifest: \`${manifest.training_manifest_id}\`
+- Export: \`${manifest.source_export_id}\`
+- Snapshot: \`${manifest.source_snapshot_id}\`
+- Split: \`${manifest.source_split_id}\`
+- Config fingerprint: \`${manifest.config_fingerprint.slice(0, 16)}...\`
+
+## Counts
+
+| Partition | Records |
+|-----------|---------|
+| Train | ${pkg.counts.train} |
+| Val | ${pkg.counts.val} |
+| Test | ${pkg.counts.test} |
+| **Total** | **${pkg.counts.total_records}** |
+
+Images: ${pkg.counts.images}
+
+## Checksums
+
+All files in this package are checksummed in \`checksums.txt\` (BSD format).
+
+## Provenance
+
+This package was produced by [Style Dataset Lab](https://github.com/mcp-tool-shop-org/style-dataset-lab).
+Every included record has been human-reviewed, canon-bound, and verified against project constitution rules.
+The dataset split ensures no subject family appears in multiple partitions.
+`;
+}
+
+async function collectFiles(dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await collectFiles(full));
+    } else {
+      files.push(full);
+    }
+  }
+  return files;
+}
+
+/**
+ * List all training packages.
+ */
+export async function listTrainingPackages(projectRoot) {
+  const dir = join(projectRoot, 'training', 'packages');
+  if (!existsSync(dir)) return [];
+
+  const entries = await readdir(dir, { withFileTypes: true });
+  const packages = [];
+
+  for (const entry of entries) {
+    if (!entry.isDirectory() || !entry.name.startsWith('tp-')) continue;
+    const manifestPath = join(dir, entry.name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const data = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      packages.push({
+        id: data.training_package_id,
+        created_at: data.created_at,
+        profile: data.training_profile_id,
+        adapter: data.adapter_target,
+        records: data.counts.total_records,
+        images: data.counts.images,
+      });
+    } catch { /* skip */ }
+  }
+
+  return packages.sort((a, b) => a.created_at.localeCompare(b.created_at));
+}
+
+/**
+ * Load a training package manifest.
+ */
+export async function loadTrainingPackage(projectRoot, packageId) {
+  const path = join(projectRoot, 'training', 'packages', packageId, 'manifest.json');
+  if (!existsSync(path)) {
+    throw new Error(`Training package "${packageId}" not found at ${path}`);
+  }
+  return JSON.parse(await readFile(path, 'utf-8'));
+}

package/lib/training-profiles.js

@@ -0,0 +1,97 @@
+/**
+ * Training profile management.
+ *
+ * A training profile defines how a project wants to turn an export package
+ * into a trainable model asset. Profiles never select data directly — they
+ * define eligibility rules and packaging expectations.
+ */
+
+import { readFile, readdir, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+
+const REQUIRED_FIELDS = ['profile_id', 'label', 'target_family', 'asset_type', 'eligible_lanes', 'adapter_targets'];
+
+/**
+ * Validate a training profile object.
+ * @returns {{ valid: boolean, errors: string[] }}
+ */
+export function validateProfile(profile) {
+  const errors = [];
+  for (const field of REQUIRED_FIELDS) {
+    if (profile[field] === undefined || profile[field] === null) {
+      errors.push(`missing required field: ${field}`);
+    }
+  }
+  if (profile.eligible_lanes && !Array.isArray(profile.eligible_lanes)) {
+    errors.push('eligible_lanes must be an array');
+  }
+  if (profile.adapter_targets && !Array.isArray(profile.adapter_targets)) {
+    errors.push('adapter_targets must be an array');
+  }
+  if (profile.adapter_targets?.length === 0) {
+    errors.push('adapter_targets must have at least one target');
+  }
+  return { valid: errors.length === 0, errors };
+}
+
+/**
+ * Load a training profile from a project.
+ */
+export async function loadTrainingProfile(projectRoot, profileId) {
+  const path = join(projectRoot, 'training', 'profiles', `${profileId}.json`);
+  if (!existsSync(path)) {
+    throw new Error(`Training profile "${profileId}" not found at ${path}`);
+  }
+  const data = JSON.parse(await readFile(path, 'utf-8'));
+  const { valid, errors } = validateProfile(data);
+  if (!valid) {
+    throw new Error(`Training profile "${profileId}" is invalid:\n  ${errors.join('\n  ')}`);
+  }
+  return data;
+}
+
+/**
+ * List all training profiles in a project.
+ */
+export async function listTrainingProfiles(projectRoot) {
+  const dir = join(projectRoot, 'training', 'profiles');
+  if (!existsSync(dir)) return [];
+
+  const files = (await readdir(dir)).filter(f => f.endsWith('.json')).sort();
+  const profiles = [];
+
+  for (const file of files) {
+    try {
+      const data = JSON.parse(await readFile(join(dir, file), 'utf-8'));
+      profiles.push({
+        profile_id: data.profile_id,
+        label: data.label,
+        target_family: data.target_family,
+        asset_type: data.asset_type,
+        lanes: data.eligible_lanes?.length || 0,
+        adapters: data.adapter_targets || [],
+      });
+    } catch {
+      // Skip malformed
+    }
+  }
+
+  return profiles;
+}
+
+/**
+ * Save a training profile to a project.
+ */
+export async function saveTrainingProfile(projectRoot, profile) {
+  const { valid, errors } = validateProfile(profile);
+  if (!valid) {
+    throw new Error(`Invalid training profile:\n  ${errors.join('\n  ')}`);
+  }
+  const dir = join(projectRoot, 'training', 'profiles');
+  await mkdir(dir, { recursive: true });
+  await writeFile(
+    join(dir, `${profile.profile_id}.json`),
+    JSON.stringify(profile, null, 2) + '\n'
+  );
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mcptoolshop/style-dataset-lab",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Visual canon and dataset production pipeline — snapshots, splits, versioned exports, eval packs, and canon-bound curation for any image production domain",
   "type": "module",
   "bin": {

package/scripts/doctor.js

@@ -36,6 +36,11 @@ const REQUIRED_DIRS = [
   'snapshots',
   'splits',
   'eval-packs',
+  'training/profiles',
+  'training/manifests',
+  'training/packages',
+  'training/eval-runs',
+  'training/implementations',
 ];
 
 function pass(msg) { console.log(`  \x1b[32m✓\x1b[0m ${msg}`); }