@mcptoolshop/style-dataset-lab 3.0.1 → 3.2.0

package/lib/adapters/diffusers-lora.js

@@ -1,60 +1,40 @@
 /**
  * Diffusers LoRA adapter.
  *
- * Produces a layout compatible with diffusers fine-tuning scripts:
- * - dataset/<partition>/ with image files
- * - dataset/<partition>/<image>.txt caption sidecar files
- * - metadata/<partition>.jsonl with structured rows
+ * Consumes canonical rows (per D6) — either produced in memory by
+ * training-packages.js from records, or read from a canon-build
+ * dataset.jsonl. The row shape is defined in lib/rows.js and pinned by
+ * ROW_SCHEMA_VERSION.
  *
- * Does not mutate inclusion or split truth.
+ * Output layout (unchanged from pre-refactor):
+ *   dataset/<partition>/<entity_id>.<ext>        — image (copied or symlinked)
+ *   dataset/<partition>/<entity_id>.txt          — caption sidecar
+ *   metadata/<partition>.jsonl                   — per-row audit
  */
 
 import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
-import { join, relative, basename } from 'node:path';
+import { join, relative } from 'node:path';
 import { existsSync } from 'node:fs';
+import { groupRowsByPartition } from '../rows.js';
 
 /**
- * Build a diffusers-style caption from record metadata.
+ * Build training package in diffusers LoRA format from canonical rows.
+ *
+ * @param {Object} args
+ * @param {string} args.packageDir
+ * @param {Array<Object>} args.rows — canonical rows; partition field groups them
+ * @param {Object} args.profile
+ * @param {Object} args.manifest
+ * @param {Object} args.config
+ * @param {string} args.projectRoot
+ * @param {boolean} [args.copy]
  */
-function buildCaption(record, lane, group, profile) {
-  const parts = [];
-
-  // Trigger word for LoRA activation
-  if (profile.prompt_strategy === 'trigger-word' && profile.profile_id) {
-    parts.push(profile.profile_id.replace(/-/g, '_'));
-  }
-
-  // Subject identity if available
-  if (record.identity?.subject_name) {
-    parts.push(record.identity.subject_name.replace(/_/g, ' '));
-  }
-
-  // Faction / group
-  if (group && group !== 'unknown') {
-    parts.push(`${group} faction`);
-  }
-
-  // Lane context
-  if (lane) parts.push(lane);
+export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
+  const partitioned = groupRowsByPartition(rows || []);
 
-  // Use original prompt if available, otherwise build from ID
-  const prompt = record.provenance?.prompt;
-  if (prompt) {
-    parts.push(prompt.slice(0, 300));
-  } else {
-    parts.push(record.id.replace(/_/g, ' '));
-  }
-
-  return parts.join(', ');
-}
-
-/**
- * Build training package in diffusers LoRA format.
- */
-export async function buildPackage({ packageDir, partitions, profile, manifest, config, projectRoot, copy }) {
   let imageCount = 0;
 
-  for (const [partition, records] of Object.entries(partitions)) {
+  for (const [partition, partitionRows] of Object.entries(partitioned)) {
     const datasetDir = join(packageDir, 'dataset', partition);
     const metadataDir = join(packageDir, 'metadata');
     await mkdir(datasetDir, { recursive: true });
@@ -62,19 +42,17 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
 
     const metadataRows = [];
 
-    for (const { record, lane, group } of records) {
-      if (!record.asset_path) continue;
+    for (const row of partitionRows) {
+      if (!row.asset_path) continue;
 
-      const sourcePath = join(projectRoot, record.asset_path);
+      const sourcePath = join(projectRoot, row.asset_path);
       if (!existsSync(sourcePath)) continue;
 
-      // D-009: Name output files by record.id (not source basename) to
-      // avoid silent collisions when two records share a filename stem.
-      const sourceBasename = record.asset_path.split(/[\\/]/).pop();
+      const sourceBasename = row.asset_path.split(/[\\/]/).pop();
       const extMatch = sourceBasename.match(/\.[^.]+$/);
       const ext = extMatch ? extMatch[0] : '';
-      const filename = `${record.id}${ext}`;
-      const stem = record.id;
+      const filename = `${row.entity_id}${ext}`;
+      const stem = row.entity_id;
       const destPath = join(datasetDir, filename);
 
       try {
@@ -88,28 +66,30 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
         continue;
       }
 
-      // Write caption sidecar (.txt next to image)
-      const caption = buildCaption(record, lane, group, profile);
-      await writeFile(join(datasetDir, `${stem}.txt`), caption + '\n');
+      await writeFile(join(datasetDir, `${stem}.txt`), row.caption + '\n');
 
-      metadataRows.push({
-        file: `dataset/${partition}/${filename}`,
-        caption_file: `dataset/${partition}/${stem}.txt`,
-        record_id: record.id,
-        caption,
-        lane,
-        group,
-        subject: record.identity?.subject_name || null,
-        pass_ratio: record.canon?.assertion_count > 0
-          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : null,
-      });
+      metadataRows.push(buildMetadataRow(row, partition, filename, stem));
     }
 
     await writeFile(
       join(metadataDir, `${partition}.jsonl`),
-      metadataRows.map(r => JSON.stringify(r)).join('\n') + '\n'
+      metadataRows.map(r => JSON.stringify(r)).join('\n') + (metadataRows.length ? '\n' : '')
     );
   }
 
   return { imageCount };
 }
+
+function buildMetadataRow(row, partition, filename, stem) {
+  const out = {
+    file: `dataset/${partition}/${filename}`,
+    caption_file: `dataset/${partition}/${stem}.txt`,
+    record_id: row.entity_id,
+    caption: row.caption,
+    lane: row.lane,
+    group: row.group ?? null,
+    subject: row.subject_filter_key ?? null,
+    pass_ratio: row.pass_ratio ?? null,
+  };
+  return out;
+}

package/lib/adapters/generic-image-caption.js

@@ -1,49 +1,26 @@
 /**
  * Generic image+caption adapter.
  *
- * Simple, inspectable, durable baseline:
- * - dataset/<partition>/ with image files
- * - metadata/<partition>.jsonl with one row per image
+ * Consumes canonical rows (per D6) and emits:
+ *   dataset/<partition>/<entity_id>.<ext>  — image (copied or symlinked)
+ *   metadata/<partition>.jsonl             — per-row audit (caption inline)
  *
- * Does not mutate inclusion or split truth.
+ * No caption sidecar files — this adapter keeps metadata flat and
+ * machine-readable. Use diffusers-lora or ai-toolkit for trainer-specific
+ * caption-file conventions.
  */
 
 import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
 import { join, relative } from 'node:path';
 import { existsSync } from 'node:fs';
+import { groupRowsByPartition } from '../rows.js';
 
-/**
- * Build a caption string from a record and profile.
- */
-function buildCaption(record, lane, group, profile) {
-  const parts = [];
-
-  if (profile.prompt_strategy === 'trigger-word' && profile.profile_id) {
-    parts.push(profile.profile_id.replace(/-/g, '_'));
-  }
-
-  if (group) parts.push(group);
-  if (lane) parts.push(lane);
-
-  // Add key provenance if available
-  const prompt = record.provenance?.prompt;
-  if (prompt) {
-    // Use first 200 chars of prompt as caption body
-    parts.push(prompt.slice(0, 200));
-  } else {
-    parts.push(record.id.replace(/_/g, ' '));
-  }
-
-  return parts.join(', ');
-}
+export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
+  const partitioned = groupRowsByPartition(rows || []);
 
-/**
- * Build training package in generic image+caption format.
- */
-export async function buildPackage({ packageDir, partitions, profile, manifest, config, projectRoot, copy }) {
   let imageCount = 0;
 
-  for (const [partition, records] of Object.entries(partitions)) {
+  for (const [partition, partitionRows] of Object.entries(partitioned)) {
     const datasetDir = join(packageDir, 'dataset', partition);
     const metadataDir = join(packageDir, 'metadata');
     await mkdir(datasetDir, { recursive: true });
@@ -51,18 +28,16 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
 
     const metadataRows = [];
 
-    for (const { record, lane, group } of records) {
-      if (!record.asset_path) continue;
+    for (const row of partitionRows) {
+      if (!row.asset_path) continue;
 
-      const sourcePath = join(projectRoot, record.asset_path);
+      const sourcePath = join(projectRoot, row.asset_path);
       if (!existsSync(sourcePath)) continue;
 
-      // D-009: Name output files by record.id (not source basename) to
-      // avoid silent collisions when two records share a filename stem.
-      const sourceBasename = record.asset_path.split(/[\\/]/).pop();
+      const sourceBasename = row.asset_path.split(/[\\/]/).pop();
       const extMatch = sourceBasename.match(/\.[^.]+$/);
       const ext = extMatch ? extMatch[0] : '';
-      const filename = `${record.id}${ext}`;
+      const filename = `${row.entity_id}${ext}`;
       const destPath = join(datasetDir, filename);
 
       try {
@@ -73,25 +48,22 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
         }
         imageCount++;
       } catch {
-        continue; // skip duplicates or broken links
+        continue;
       }
 
-      const caption = buildCaption(record, lane, group, profile);
-
       metadataRows.push({
         file: `dataset/${partition}/${filename}`,
-        record_id: record.id,
-        caption,
-        lane,
-        group,
-        pass_ratio: record.canon?.assertion_count > 0
-          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : null,
+        record_id: row.entity_id,
+        caption: row.caption,
+        lane: row.lane,
+        group: row.group ?? null,
+        pass_ratio: row.pass_ratio ?? null,
       });
     }
 
     await writeFile(
       join(metadataDir, `${partition}.jsonl`),
-      metadataRows.map(r => JSON.stringify(r)).join('\n') + '\n'
+      metadataRows.map(r => JSON.stringify(r)).join('\n') + (metadataRows.length ? '\n' : '')
     );
   }
 

package/lib/batch-runs.js

@@ -85,6 +85,57 @@ export async function createBatchDir(projectRoot) {
   return { batchId, batchDir };
 }
 
+/**
+ * Compute which prior slot_ids are eligible to skip on resume.
+ * Truth source: status === 'ok'. For pre-status manifests we infer success
+ * from a truthy selected_output. Returns a Map<slot_id, prior_result>.
+ */
+export function buildCompletedSlotMap(priorResults) {
+  const completed = new Map();
+  if (!Array.isArray(priorResults)) return completed;
+  for (const r of priorResults) {
+    if (!r || !r.slot_id) continue;
+    const ok = r.status === 'ok' || (r.status === undefined && !!r.selected_output);
+    if (ok) completed.set(r.slot_id, r);
+  }
+  return completed;
+}
+
+/**
+ * Resume an existing batch directory. Loads its prior manifest so the caller
+ * can replay only the incomplete slots while keeping the same batch_id.
+ *
+ * Throws BATCH_NOT_FOUND if the batch dir or manifest is missing.
+ * Throws BATCH_NO_PROGRESS if the manifest predates the incremental-checkpoint
+ * format (no `slots` array) — those batches can't be partially resumed.
+ */
+export function openBatchDirForResume(projectRoot, batchId) {
+  const batchesDir = getBatchesDir(projectRoot);
+  const batchDir = join(batchesDir, batchId);
+  const manifestPath = join(batchDir, 'manifest.json');
+
+  if (!existsSync(manifestPath)) {
+    throw inputError(
+      'BATCH_NOT_FOUND',
+      `Batch "${batchId}" not found at ${manifestPath}`,
+      'Run `sdlab batch list` to see available batch IDs.'
+    );
+  }
+
+  const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+  const priorResults = Array.isArray(manifest.slots) ? manifest.slots : null;
+
+  if (!priorResults) {
+    throw inputError(
+      'BATCH_NO_PROGRESS',
+      `Batch "${batchId}" has no slot-level progress to resume from`,
+      'This batch was created before incremental checkpointing — re-run from scratch instead.'
+    );
+  }
+
+  return { batchId, batchDir, priorResults, priorManifest: manifest };
+}
+
 /**
  * Execute all slot briefs as coordinated runs.
  *
@@ -95,6 +146,7 @@ export async function createBatchDir(projectRoot) {
  * @param {Object} opts.mode — batch mode definition
  * @param {Object[]} opts.slotBriefs — one brief per slot
  * @param {boolean} [opts.dryRun]
+ * @param {Object[]} [opts.priorResults] — previously-completed slot results to carry forward (resume)
  * @returns {Promise<Object[]>} — array of { slot, brief, manifest }
  */
 export async function executeBatchRuns({
@@ -104,6 +156,7 @@ export async function executeBatchRuns({
   mode,
   slotBriefs,
   dryRun = false,
+  priorResults = [],
 }) {
   // PB-005: fail fast on length mismatch instead of crashing deep in the loop.
   if (!Array.isArray(mode?.variant_plan) || mode.variant_plan.length !== slotBriefs.length) {
@@ -115,6 +168,8 @@ export async function executeBatchRuns({
     );
   }
 
+  const completed = buildCompletedSlotMap(priorResults);
+
   const results = [];
   // PB-005: incremental batch-progress manifest — written after every slot
   // so a crash never orphans the batch dir without a record of what ran.
@@ -125,6 +180,13 @@ export async function executeBatchRuns({
     const brief = slotBriefs[i];
     const slotLabel = slot.label || slot.slot_id;
 
+    if (completed.has(slot.slot_id)) {
+      const prior = completed.get(slot.slot_id);
+      info(`[${i + 1}/${slotBriefs.length}] Slot: ${slotLabel} \x1b[2m(resumed — already complete)\x1b[0m`);
+      results.push(prior);
+      continue;
+    }
+
     info(`[${i + 1}/${slotBriefs.length}] Slot: ${slotLabel}`);
 
     // Save brief into batch/briefs/