@mcptoolshop/style-dataset-lab 2.0.0 → 2.2.0

package/scripts/split.js

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+
+/**
+ * split.js — Build, show, and audit dataset splits.
+ *
+ * Usage:
+ *   sdlab split build [--snapshot <id>] [--profile <name>] [--project <name>]
+ *   sdlab split list [--project <name>]
+ *   sdlab split show <split-id> [--project <name>]
+ *   sdlab split audit <split-id> [--project <name>]
+ */
+
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { getProjectName } from '../lib/args.js';
+import { REPO_ROOT } from '../lib/paths.js';
+import { loadSplitProfile } from '../lib/config.js';
+import { listSnapshots } from '../lib/snapshot.js';
+import { createSplit, loadSplit, loadSplitAudit, listSplits } from '../lib/split.js';
+
+export async function run(argv = process.argv.slice(2)) {
+  const projectName = getProjectName(argv);
+  const projectRoot = join(REPO_ROOT, 'projects', projectName);
+  const subcommand = argv.find(a => !a.startsWith('--')) || 'list';
+  const args = argv.filter(a => a !== subcommand);
+
+  if (subcommand === 'build') {
+    // Find snapshot ID
+    let snapshotId;
+    const snapIdx = args.indexOf('--snapshot');
+    if (snapIdx >= 0) {
+      snapshotId = args[snapIdx + 1];
+    } else {
+      // Use latest snapshot
+      const snapshots = await listSnapshots(projectRoot);
+      if (snapshots.length === 0) {
+        throw new Error('No snapshots found. Run: sdlab snapshot create');
+      }
+      snapshotId = snapshots[snapshots.length - 1].id;
+    }
+
+    const profileIdx = args.indexOf('--profile');
+    const profileId = profileIdx >= 0 ? args[profileIdx + 1] : null;
+    const profile = loadSplitProfile(projectRoot, profileId);
+
+    console.log(`\x1b[1msdlab split build\x1b[0m — ${projectName}`);
+    console.log(`  Snapshot: ${snapshotId}`);
+    console.log(`  Profile:  ${profileId || 'balanced-default (built-in)'}`);
+    console.log(`  Strategy: ${profile.strategy}`);
+    console.log(`  Ratios:   ${profile.train_ratio}/${profile.val_ratio}/${profile.test_ratio}`);
+    console.log(`  Seed:     ${profile.seed}`);
+    console.log('');
+
+    const result = await createSplit(projectRoot, snapshotId, profile);
+
+    console.log(`  \x1b[32m✓\x1b[0m Split: ${result.splitId}`);
+    console.log(`  Train: ${result.train}`);
+    console.log(`  Val:   ${result.val}`);
+    console.log(`  Test:  ${result.test}`);
+
+    // Show audit summary
+    const audit = await loadSplitAudit(projectRoot, result.splitId);
+    console.log('');
+    if (audit.leakage_check.passed) {
+      console.log(`  \x1b[32m✓\x1b[0m No subject leakage detected`);
+    } else {
+      console.log(`  \x1b[31m✗\x1b[0m Subject leakage detected:`);
+      for (const issue of audit.leakage_check.issues) {
+        console.log(`    family="${issue.family}" (${issue.record_count} records) → ${issue.leaked_to.join(', ')}`);
+      }
+    }
+
+    console.log(`  Families: ${audit.family_count} (train=${audit.families_per_split.train}, val=${audit.families_per_split.val}, test=${audit.families_per_split.test})`);
+
+  } else if (subcommand === 'list') {
+    const splits = await listSplits(projectRoot);
+    if (splits.length === 0) {
+      console.log('No splits found. Run: sdlab split build');
+      return;
+    }
+    console.log(`\x1b[1mSplits\x1b[0m — ${projectName}\n`);
+    for (const s of splits) {
+      console.log(`  ${s.id}  snap=${s.snapshot_id}  train=${s.train} val=${s.val} test=${s.test}  families=${s.families}  ${s.created_at}`);
+    }
+
+  } else if (subcommand === 'show') {
+    const splitId = args.find(a => a.startsWith('split-'));
+    if (!splitId) throw new Error('Usage: sdlab split show <split-id>');
+
+    const manifest = await loadSplit(projectRoot, splitId);
+    console.log(`\x1b[1mSplit\x1b[0m — ${splitId}\n`);
+    console.log(`  Created:  ${manifest.created_at}`);
+    console.log(`  Snapshot: ${manifest.snapshot_id}`);
+    console.log(`  Records:  ${manifest.counts.total_records}`);
+    console.log(`  Families: ${manifest.counts.total_families}`);
+    console.log(`  Train:    ${manifest.counts.train}`);
+    console.log(`  Val:      ${manifest.counts.val}`);
+    console.log(`  Test:     ${manifest.counts.test}`);
+    console.log('');
+    console.log('  Profile:');
+    for (const [k, v] of Object.entries(manifest.profile)) {
+      console.log(`    ${k}: ${JSON.stringify(v)}`);
+    }
+    if (manifest.warnings.length > 0) {
+      console.log('');
+      console.log('  Warnings:');
+      for (const w of manifest.warnings) {
+        console.log(`    ⚠ ${w}`);
+      }
+    }
+
+  } else if (subcommand === 'audit') {
+    const splitId = args.find(a => a.startsWith('split-'));
+    if (!splitId) throw new Error('Usage: sdlab split audit <split-id>');
+
+    const audit = await loadSplitAudit(projectRoot, splitId);
+    console.log(`\x1b[1mSplit audit\x1b[0m — ${splitId}\n`);
+
+    // Leakage check
+    if (audit.leakage_check.passed) {
+      console.log(`  \x1b[32m✓\x1b[0m Leakage check: PASSED`);
+    } else {
+      console.log(`  \x1b[31m✗\x1b[0m Leakage check: FAILED`);
+      for (const issue of audit.leakage_check.issues) {
+        console.log(`    family="${issue.family}" (${issue.record_count} records) leaked to: ${issue.leaked_to.join(', ')}`);
+      }
+    }
+
+    // Family distribution
+    console.log(`\n  Families: ${audit.family_count}`);
+    console.log(`    Train: ${audit.families_per_split.train}`);
+    console.log(`    Val:   ${audit.families_per_split.val}`);
+    console.log(`    Test:  ${audit.families_per_split.test}`);
+
+    // Lane balance
+    console.log('\n  Lane balance:');
+    console.log(`    ${'Lane'.padEnd(20)} ${'Total'.padStart(6)} ${'Train%'.padStart(8)} ${'Val%'.padStart(8)} ${'Test%'.padStart(8)}`);
+    console.log(`    ${'─'.repeat(52)}`);
+    for (const [lane, b] of Object.entries(audit.lane_balance).sort((a, b) => b[1].total - a[1].total)) {
+      console.log(`    ${lane.padEnd(20)} ${String(b.total).padStart(6)} ${String(b.train_pct + '%').padStart(8)} ${String(b.val_pct + '%').padStart(8)} ${String(b.test_pct + '%').padStart(8)}`);
+    }
+
+  } else {
+    throw new Error(`Unknown subcommand: ${subcommand}. Use: build, list, show, audit`);
+  }
+}
+
+// Direct execution guard
+if (process.argv[1] && (process.argv[1].endsWith('split.js') || process.argv[1].endsWith('split'))) {
+  run().catch((err) => {
+    console.error(err.message || err);
+    process.exit(1);
+  });
+}

package/scripts/training-manifest.js

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+
+/**
+ * training-manifest.js — Create, validate, list, and show training manifests.
+ *
+ * Usage:
+ *   sdlab training-manifest create --export <id> --profile <id> [--adapter <target>] [--base-model <name>] [--project <name>]
+ *   sdlab training-manifest validate <manifest-id> [--project <name>]
+ *   sdlab training-manifest show <manifest-id> [--project <name>]
+ *   sdlab training-manifest list [--project <name>]
+ */
+
+import { join } from 'node:path';
+import { getProjectName } from '../lib/args.js';
+import { REPO_ROOT } from '../lib/paths.js';
+import { listExports } from '../lib/export.js';
+import {
+  createTrainingManifest,
+  validateTrainingManifest,
+  loadTrainingManifest,
+  listTrainingManifests,
+} from '../lib/training-manifests.js';
+
+export async function run(argv = process.argv.slice(2)) {
+  const projectName = getProjectName(argv);
+  const projectRoot = join(REPO_ROOT, 'projects', projectName);
+  const subcommand = argv.find(a => !a.startsWith('--') && !a.startsWith('tm-')) || 'list';
+  const args = argv.filter(a => a !== subcommand);
+
+  if (subcommand === 'create') {
+    // Required: --export and --profile
+    const exportIdx = args.indexOf('--export');
+    if (exportIdx < 0) {
+      // Try latest export
+      const exports = await listExports(projectRoot);
+      if (exports.length === 0) throw new Error('No exports found. Run: sdlab export build');
+    }
+    const exportId = exportIdx >= 0 ? args[exportIdx + 1] : (await listExports(projectRoot)).pop().id;
+
+    const profileIdx = args.indexOf('--profile');
+    if (profileIdx < 0) throw new Error('--profile is required. Use: sdlab training-profile list');
+    const profileId = args[profileIdx + 1];
+
+    const adapterIdx = args.indexOf('--adapter');
+    const adapterTarget = adapterIdx >= 0 ? args[adapterIdx + 1] : undefined;
+
+    const baseModelIdx = args.indexOf('--base-model');
+    const baseModel = baseModelIdx >= 0 ? args[baseModelIdx + 1] : undefined;
+
+    console.log(`\x1b[1msdlab training-manifest create\x1b[0m — ${projectName}`);
+    console.log(`  Export:  ${exportId}`);
+    console.log(`  Profile: ${profileId}`);
+    if (adapterTarget) console.log(`  Adapter: ${adapterTarget}`);
+    if (baseModel) console.log(`  Base:    ${baseModel}`);
+    console.log('');
+
+    const result = await createTrainingManifest(projectRoot, exportId, profileId, {
+      adapterTarget,
+      baseModel,
+    });
+
+    console.log(`  \x1b[32m✓\x1b[0m Manifest: ${result.manifestId}`);
+
+  } else if (subcommand === 'validate') {
+    const manifestId = args.find(a => a.startsWith('tm-'));
+    if (!manifestId) throw new Error('Usage: sdlab training-manifest validate <manifest-id>');
+
+    console.log(`\x1b[1msdlab training-manifest validate\x1b[0m — ${manifestId}\n`);
+    const { valid, issues } = await validateTrainingManifest(projectRoot, manifestId);
+
+    if (valid) {
+      console.log(`  \x1b[32m✓\x1b[0m Manifest is valid — all references intact, fingerprints match`);
+    } else {
+      console.log(`  \x1b[31m✗\x1b[0m Manifest has issues:`);
+      for (const issue of issues) {
+        console.log(`    - ${issue}`);
+      }
+    }
+
+  } else if (subcommand === 'show') {
+    const manifestId = args.find(a => a.startsWith('tm-'));
+    if (!manifestId) throw new Error('Usage: sdlab training-manifest show <manifest-id>');
+
+    const manifest = await loadTrainingManifest(projectRoot, manifestId);
+    console.log(`\x1b[1mTraining manifest\x1b[0m — ${manifestId}\n`);
+    console.log(`  Created:     ${manifest.created_at}`);
+    console.log(`  Profile:     ${manifest.training_profile_id}`);
+    console.log(`  Export:      ${manifest.source_export_id}`);
+    console.log(`  Snapshot:    ${manifest.source_snapshot_id}`);
+    console.log(`  Split:       ${manifest.source_split_id}`);
+    console.log(`  Adapter:     ${manifest.adapter_target}`);
+    console.log(`  Base model:  ${manifest.base_model || '(not specified)'}`);
+    console.log(`  Caption:     ${manifest.caption_mode}`);
+    console.log(`  Records:     ${manifest.dataset_counts?.records || 0}`);
+    console.log(`  Fingerprint: ${manifest.config_fingerprint.slice(0, 16)}...`);
+    console.log(`  Export hash: ${manifest.export_manifest_hash.slice(0, 16)}...`);
+    if (manifest.linked_eval_pack_ids?.length) {
+      console.log(`  Eval packs:  ${manifest.linked_eval_pack_ids.length}`);
+    }
+
+  } else if (subcommand === 'list') {
+    const manifests = await listTrainingManifests(projectRoot);
+    if (manifests.length === 0) {
+      console.log('No training manifests found. Run: sdlab training-manifest create');
+      return;
+    }
+    console.log(`\x1b[1mTraining manifests\x1b[0m — ${projectName}\n`);
+    for (const m of manifests) {
+      console.log(`  ${m.id}  profile=${m.profile}  export=${m.export_id}  adapter=${m.adapter}  records=${m.records}  ${m.created_at}`);
+    }
+
+  } else {
+    throw new Error(`Unknown subcommand: ${subcommand}. Use: create, validate, show, list`);
+  }
+}
+
+if (process.argv[1] && (process.argv[1].endsWith('training-manifest.js') || process.argv[1].endsWith('training-manifest'))) {
+  run().catch((err) => { console.error(err.message || err); process.exit(1); });
+}

package/scripts/training-package.js

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+
+/**
+ * training-package.js — Build and inspect training packages.
+ *
+ * Usage:
+ *   sdlab training-package build --manifest <id> [--adapter <target>] [--copy] [--project <name>]
+ *   sdlab training-package show <package-id> [--project <name>]
+ *   sdlab training-package list [--project <name>]
+ */
+
+import { join } from 'node:path';
+import { getProjectName } from '../lib/args.js';
+import { REPO_ROOT } from '../lib/paths.js';
+import { listTrainingManifests } from '../lib/training-manifests.js';
+import { buildTrainingPackage, listTrainingPackages, loadTrainingPackage } from '../lib/training-packages.js';
+
+export async function run(argv = process.argv.slice(2)) {
+  const projectName = getProjectName(argv);
+  const projectRoot = join(REPO_ROOT, 'projects', projectName);
+  const subcommand = argv.find(a => !a.startsWith('--') && !a.startsWith('tp-') && !a.startsWith('tm-')) || 'list';
+  const args = argv.filter(a => a !== subcommand);
+
+  if (subcommand === 'build') {
+    // Find manifest
+    let manifestId;
+    const manifestIdx = args.indexOf('--manifest');
+    if (manifestIdx >= 0) {
+      manifestId = args[manifestIdx + 1];
+    } else {
+      const manifests = await listTrainingManifests(projectRoot);
+      if (manifests.length === 0) throw new Error('No training manifests found. Run: sdlab training-manifest create');
+      manifestId = manifests[manifests.length - 1].id;
+    }
+
+    const adapterIdx = args.indexOf('--adapter');
+    const adapterOverride = adapterIdx >= 0 ? args[adapterIdx + 1] : undefined;
+    const copy = args.includes('--copy');
+
+    console.log(`\x1b[1msdlab training-package build\x1b[0m — ${projectName}`);
+    console.log(`  Manifest: ${manifestId}`);
+    if (adapterOverride) console.log(`  Adapter:  ${adapterOverride}`);
+    console.log(`  Images:   ${copy ? 'copy' : 'symlink'}`);
+    console.log('');
+
+    const result = await buildTrainingPackage(projectRoot, manifestId, { copy, adapterOverride });
+
+    console.log(`  \x1b[32m✓\x1b[0m Package: ${result.packageId}`);
+    console.log(`  Records: ${result.records}`);
+    console.log(`  Images:  ${result.images}`);
+
+  } else if (subcommand === 'show') {
+    const packageId = args.find(a => a.startsWith('tp-'));
+    if (!packageId) throw new Error('Usage: sdlab training-package show <package-id>');
+
+    const pkg = await loadTrainingPackage(projectRoot, packageId);
+    console.log(`\x1b[1mTraining package\x1b[0m — ${packageId}\n`);
+    console.log(`  Created:  ${pkg.created_at}`);
+    console.log(`  Profile:  ${pkg.training_profile_id}`);
+    console.log(`  Adapter:  ${pkg.adapter_target}`);
+    console.log(`  Manifest: ${pkg.training_manifest_id}`);
+    console.log(`  Export:   ${pkg.source_export_id}`);
+    console.log(`  Records:  ${pkg.counts.total_records} (train=${pkg.counts.train} val=${pkg.counts.val} test=${pkg.counts.test})`);
+    console.log(`  Images:   ${pkg.counts.images}`);
+
+  } else if (subcommand === 'list') {
+    const packages = await listTrainingPackages(projectRoot);
+    if (packages.length === 0) {
+      console.log('No training packages found. Run: sdlab training-package build');
+      return;
+    }
+    console.log(`\x1b[1mTraining packages\x1b[0m — ${projectName}\n`);
+    for (const p of packages) {
+      console.log(`  ${p.id}  profile=${p.profile}  adapter=${p.adapter}  records=${p.records}  images=${p.images}  ${p.created_at}`);
+    }
+
+  } else {
+    throw new Error(`Unknown subcommand: ${subcommand}. Use: build, show, list`);
+  }
+}
+
+if (process.argv[1] && (process.argv[1].endsWith('training-package.js') || process.argv[1].endsWith('training-package'))) {
+  run().catch((err) => { console.error(err.message || err); process.exit(1); });
+}

package/scripts/training-profile.js

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+
+/**
+ * training-profile.js — List and show training profiles.
+ *
+ * Usage:
+ *   sdlab training-profile list [--project <name>]
+ *   sdlab training-profile show <profile-id> [--project <name>]
+ */
+
+import { join } from 'node:path';
+import { getProjectName } from '../lib/args.js';
+import { REPO_ROOT } from '../lib/paths.js';
+import { listTrainingProfiles, loadTrainingProfile } from '../lib/training-profiles.js';
+
+export async function run(argv = process.argv.slice(2)) {
+  const projectName = getProjectName(argv);
+  const projectRoot = join(REPO_ROOT, 'projects', projectName);
+  const subcommand = argv.find(a => !a.startsWith('--')) || 'list';
+  const args = argv.filter(a => a !== subcommand);
+
+  if (subcommand === 'list') {
+    const profiles = await listTrainingProfiles(projectRoot);
+    if (profiles.length === 0) {
+      console.log('No training profiles found. Add JSON files to training/profiles/');
+      return;
+    }
+    console.log(`\x1b[1mTraining profiles\x1b[0m — ${projectName}\n`);
+    for (const p of profiles) {
+      console.log(`  ${p.profile_id.padEnd(30)} ${p.target_family.padEnd(15)} ${p.asset_type.padEnd(10)} lanes=${p.lanes}  adapters=${p.adapters.join(',')}`);
+    }
+
+  } else if (subcommand === 'show') {
+    const profileId = args.find(a => !a.startsWith('--'));
+    if (!profileId) throw new Error('Usage: sdlab training-profile show <profile-id>');
+
+    const profile = await loadTrainingProfile(projectRoot, profileId);
+    console.log(`\x1b[1mTraining profile\x1b[0m — ${profileId}\n`);
+    console.log(`  Label:           ${profile.label}`);
+    console.log(`  Target family:   ${profile.target_family}`);
+    console.log(`  Asset type:      ${profile.asset_type}`);
+    console.log(`  Intended use:    ${profile.intended_use || '(not specified)'}`);
+    console.log(`  Eligible lanes:  ${profile.eligible_lanes.join(', ')}`);
+    console.log(`  Adapter targets: ${profile.adapter_targets.join(', ')}`);
+    if (profile.caption_strategy) console.log(`  Caption:         ${profile.caption_strategy}`);
+    if (profile.base_model_recommendations?.length) {
+      console.log(`  Base models:     ${profile.base_model_recommendations.join(', ')}`);
+    }
+    if (profile.excluded_tags?.length) {
+      console.log(`  Excluded tags:   ${profile.excluded_tags.join(', ')}`);
+    }
+    if (profile.required_eval_pack_profiles?.length) {
+      console.log(`  Required evals:  ${profile.required_eval_pack_profiles.join(', ')}`);
+    }
+
+  } else {
+    throw new Error(`Unknown subcommand: ${subcommand}. Use: list, show`);
+  }
+}
+
+if (process.argv[1] && (process.argv[1].endsWith('training-profile.js') || process.argv[1].endsWith('training-profile'))) {
+  run().catch((err) => { console.error(err.message || err); process.exit(1); });
+}