@mcptoolshop/style-dataset-lab 3.1.0 → 3.3.0

package/lib/adapters/diffusers-lora.js

@@ -1,60 +1,40 @@
 /**
  * Diffusers LoRA adapter.
  *
- * Produces a layout compatible with diffusers fine-tuning scripts:
- * - dataset/<partition>/ with image files
- * - dataset/<partition>/<image>.txt caption sidecar files
- * - metadata/<partition>.jsonl with structured rows
+ * Consumes canonical rows (per D6) — either produced in memory by
+ * training-packages.js from records, or read from a canon-build
+ * dataset.jsonl. The row shape is defined in lib/rows.js and pinned by
+ * ROW_SCHEMA_VERSION.
  *
- * Does not mutate inclusion or split truth.
+ * Output layout (unchanged from pre-refactor):
+ *   dataset/<partition>/<entity_id>.<ext>        — image (copied or symlinked)
+ *   dataset/<partition>/<entity_id>.txt          — caption sidecar
+ *   metadata/<partition>.jsonl                   — per-row audit
  */
 
 import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
-import { join, relative, basename } from 'node:path';
+import { join, relative } from 'node:path';
 import { existsSync } from 'node:fs';
+import { groupRowsByPartition } from '../rows.js';
 
 /**
- * Build a diffusers-style caption from record metadata.
+ * Build training package in diffusers LoRA format from canonical rows.
+ *
+ * @param {Object} args
+ * @param {string} args.packageDir
+ * @param {Array<Object>} args.rows — canonical rows; partition field groups them
+ * @param {Object} args.profile
+ * @param {Object} args.manifest
+ * @param {Object} args.config
+ * @param {string} args.projectRoot
+ * @param {boolean} [args.copy]
  */
-function buildCaption(record, lane, group, profile) {
-  const parts = [];
-
-  // Trigger word for LoRA activation
-  if (profile.prompt_strategy === 'trigger-word' && profile.profile_id) {
-    parts.push(profile.profile_id.replace(/-/g, '_'));
-  }
-
-  // Subject identity if available
-  if (record.identity?.subject_name) {
-    parts.push(record.identity.subject_name.replace(/_/g, ' '));
-  }
-
-  // Faction / group
-  if (group && group !== 'unknown') {
-    parts.push(`${group} faction`);
-  }
-
-  // Lane context
-  if (lane) parts.push(lane);
+export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
+  const partitioned = groupRowsByPartition(rows || []);
 
-  // Use original prompt if available, otherwise build from ID
-  const prompt = record.provenance?.prompt;
-  if (prompt) {
-    parts.push(prompt.slice(0, 300));
-  } else {
-    parts.push(record.id.replace(/_/g, ' '));
-  }
-
-  return parts.join(', ');
-}
-
-/**
- * Build training package in diffusers LoRA format.
- */
-export async function buildPackage({ packageDir, partitions, profile, manifest, config, projectRoot, copy }) {
   let imageCount = 0;
 
-  for (const [partition, records] of Object.entries(partitions)) {
+  for (const [partition, partitionRows] of Object.entries(partitioned)) {
     const datasetDir = join(packageDir, 'dataset', partition);
     const metadataDir = join(packageDir, 'metadata');
     await mkdir(datasetDir, { recursive: true });
@@ -62,19 +42,17 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
 
     const metadataRows = [];
 
-    for (const { record, lane, group } of records) {
-      if (!record.asset_path) continue;
+    for (const row of partitionRows) {
+      if (!row.asset_path) continue;
 
-      const sourcePath = join(projectRoot, record.asset_path);
+      const sourcePath = join(projectRoot, row.asset_path);
       if (!existsSync(sourcePath)) continue;
 
-      // D-009: Name output files by record.id (not source basename) to
-      // avoid silent collisions when two records share a filename stem.
-      const sourceBasename = record.asset_path.split(/[\\/]/).pop();
+      const sourceBasename = row.asset_path.split(/[\\/]/).pop();
       const extMatch = sourceBasename.match(/\.[^.]+$/);
       const ext = extMatch ? extMatch[0] : '';
-      const filename = `${record.id}${ext}`;
-      const stem = record.id;
+      const filename = `${row.entity_id}${ext}`;
+      const stem = row.entity_id;
       const destPath = join(datasetDir, filename);
 
       try {
@@ -88,28 +66,30 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
         continue;
       }
 
-      // Write caption sidecar (.txt next to image)
-      const caption = buildCaption(record, lane, group, profile);
-      await writeFile(join(datasetDir, `${stem}.txt`), caption + '\n');
+      await writeFile(join(datasetDir, `${stem}.txt`), row.caption + '\n');
 
-      metadataRows.push({
-        file: `dataset/${partition}/${filename}`,
-        caption_file: `dataset/${partition}/${stem}.txt`,
-        record_id: record.id,
-        caption,
-        lane,
-        group,
-        subject: record.identity?.subject_name || null,
-        pass_ratio: record.canon?.assertion_count > 0
-          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : null,
-      });
+      metadataRows.push(buildMetadataRow(row, partition, filename, stem));
     }
 
     await writeFile(
       join(metadataDir, `${partition}.jsonl`),
-      metadataRows.map(r => JSON.stringify(r)).join('\n') + '\n'
+      metadataRows.map(r => JSON.stringify(r)).join('\n') + (metadataRows.length ? '\n' : '')
     );
   }
 
   return { imageCount };
 }
+
+function buildMetadataRow(row, partition, filename, stem) {
+  const out = {
+    file: `dataset/${partition}/${filename}`,
+    caption_file: `dataset/${partition}/${stem}.txt`,
+    record_id: row.entity_id,
+    caption: row.caption,
+    lane: row.lane,
+    group: row.group ?? null,
+    subject: row.subject_filter_key ?? null,
+    pass_ratio: row.pass_ratio ?? null,
+  };
+  return out;
+}

package/lib/adapters/generic-image-caption.js

@@ -1,49 +1,26 @@
 /**
  * Generic image+caption adapter.
  *
- * Simple, inspectable, durable baseline:
- * - dataset/<partition>/ with image files
- * - metadata/<partition>.jsonl with one row per image
+ * Consumes canonical rows (per D6) and emits:
+ *   dataset/<partition>/<entity_id>.<ext>  — image (copied or symlinked)
+ *   metadata/<partition>.jsonl             — per-row audit (caption inline)
  *
- * Does not mutate inclusion or split truth.
+ * No caption sidecar files — this adapter keeps metadata flat and
+ * machine-readable. Use diffusers-lora or ai-toolkit for trainer-specific
+ * caption-file conventions.
  */
 
 import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
 import { join, relative } from 'node:path';
 import { existsSync } from 'node:fs';
+import { groupRowsByPartition } from '../rows.js';
 
-/**
- * Build a caption string from a record and profile.
- */
-function buildCaption(record, lane, group, profile) {
-  const parts = [];
-
-  if (profile.prompt_strategy === 'trigger-word' && profile.profile_id) {
-    parts.push(profile.profile_id.replace(/-/g, '_'));
-  }
-
-  if (group) parts.push(group);
-  if (lane) parts.push(lane);
-
-  // Add key provenance if available
-  const prompt = record.provenance?.prompt;
-  if (prompt) {
-    // Use first 200 chars of prompt as caption body
-    parts.push(prompt.slice(0, 200));
-  } else {
-    parts.push(record.id.replace(/_/g, ' '));
-  }
-
-  return parts.join(', ');
-}
+export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
+  const partitioned = groupRowsByPartition(rows || []);
 
-/**
- * Build training package in generic image+caption format.
- */
-export async function buildPackage({ packageDir, partitions, profile, manifest, config, projectRoot, copy }) {
   let imageCount = 0;
 
-  for (const [partition, records] of Object.entries(partitions)) {
+  for (const [partition, partitionRows] of Object.entries(partitioned)) {
     const datasetDir = join(packageDir, 'dataset', partition);
     const metadataDir = join(packageDir, 'metadata');
     await mkdir(datasetDir, { recursive: true });
@@ -51,18 +28,16 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
 
     const metadataRows = [];
 
-    for (const { record, lane, group } of records) {
-      if (!record.asset_path) continue;
+    for (const row of partitionRows) {
+      if (!row.asset_path) continue;
 
-      const sourcePath = join(projectRoot, record.asset_path);
+      const sourcePath = join(projectRoot, row.asset_path);
       if (!existsSync(sourcePath)) continue;
 
-      // D-009: Name output files by record.id (not source basename) to
-      // avoid silent collisions when two records share a filename stem.
-      const sourceBasename = record.asset_path.split(/[\\/]/).pop();
+      const sourceBasename = row.asset_path.split(/[\\/]/).pop();
       const extMatch = sourceBasename.match(/\.[^.]+$/);
       const ext = extMatch ? extMatch[0] : '';
-      const filename = `${record.id}${ext}`;
+      const filename = `${row.entity_id}${ext}`;
       const destPath = join(datasetDir, filename);
 
       try {
@@ -73,25 +48,22 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
         }
         imageCount++;
       } catch {
-        continue; // skip duplicates or broken links
+        continue;
       }
 
-      const caption = buildCaption(record, lane, group, profile);
-
       metadataRows.push({
         file: `dataset/${partition}/${filename}`,
-        record_id: record.id,
-        caption,
-        lane,
-        group,
-        pass_ratio: record.canon?.assertion_count > 0
-          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : null,
+        record_id: row.entity_id,
+        caption: row.caption,
+        lane: row.lane,
+        group: row.group ?? null,
+        pass_ratio: row.pass_ratio ?? null,
       });
     }
 
     await writeFile(
       join(metadataDir, `${partition}.jsonl`),
-      metadataRows.map(r => JSON.stringify(r)).join('\n') + '\n'
+      metadataRows.map(r => JSON.stringify(r)).join('\n') + (metadataRows.length ? '\n' : '')
     );
   }