@mcptoolshop/style-dataset-lab 3.1.0 → 3.2.0

package/bin/sdlab.js

@@ -80,6 +80,15 @@ const SELECTION_COMMANDS = {
   'show': '../scripts/selection-show.js',
 };
 
+// Two-word commands under "canon" namespace
+const CANON_COMMANDS = {
+  'build':         '../scripts/canon-build.js',
+  'freeze':        '../scripts/canon-freeze.js',
+  'unfreeze':      '../scripts/canon-unfreeze.js',
+  'freeze-status': '../scripts/canon-freeze-status.js',
+  'drift':         '../scripts/canon-drift.js',
+};
+
 // ─── Per-command help ─────────────────────────────────────────────────
 //
 // Keyed by the user-facing command (including two-word forms joined by
@@ -264,6 +273,60 @@ Subcommands:
 Flags:
   --project <name>     Project to operate on`,
 
+  'canon freeze': `sdlab canon freeze <entity_id> --project <name> --reason "<text>" [--status frozen|soft-advisory] [--watch <fields>] [--build <sha>] [--by <name>] [--json]
+
+Stamp a freeze block on a canon entry. Requires a canon-build output to
+witness against (the "locked_at_build" field). Writes both the entry
+frontmatter AND canon-build/freeze-events.jsonl.
+
+--reason is REQUIRED — the audit record depends on it.
+
+Statuses:
+  frozen         Regen refused outright; unfreeze ceremony required.
+  soft-advisory  Refused by default; bypassable with --i-know + --reason.`,
+
+  'canon unfreeze': `sdlab canon unfreeze <entity_id> --project <name> --reason "<text>" [--by <name>] [--json]
+
+Lift a freeze back to status=auto. Preserves the freeze.overrides[] history
+on the entry (append-only) and writes an "unfreeze" event.
+
+--reason is REQUIRED — shipping studios (Sabotage, Supergiant, Pentiment)
+most often regretted not capturing why an unfreeze happened.`,
+
+  'canon freeze-status': `sdlab canon freeze-status <entity_id> --project <name> [--json]
+
+Read-only glance at an entry's freeze state. Shows status, locked_at_build,
+frozen_by, frozen_reason, watch_fields, overrides count, and event count.`,
+
+  'canon drift': `sdlab canon drift --project <name> [--since <build_hash>] [--json]
+
+Drift report. For every frozen or on-canon-change entry, computes the
+current watch-field hash and compares against the hash stamped in the
+latest canon-build manifest. Reports drifted entries and overrides since
+a given build (default: since the latest clean build).`,
+
+  'canon build': `sdlab canon build --project <name> [--full] [--no-cache] [--dry-run] [--only <id[,id...]>] [--json]
+
+Build the three canonical projections from a project's canon entity store:
+  - dataset.jsonl       (for training adapters)
+  - prompts/<id>.j2     (Jinja2 templates for ComfyUI workflow invocation)
+  - context/<id>.md     (narrative blocks for Role OS dispatch)
+
+Reads canon-build/config.json under the project directory. Produces
+<project>/canon-build/<canon_sha>/ and a content-addressable cache under
+<project>/canon-build/.cache/.
+
+Flags:
+  --project <name>   Project to operate on (required)
+  --full             Ignore cache hits; rebuild every entity
+  --no-cache         Neither read nor write the cache this run
+  --dry-run          Walk + resolve + plan; write nothing
+  --only <ids>       Limit to specific entity ids (comma-separated)
+  --json             Emit the result summary as JSON on stdout
+  --quiet            Suppress the human-readable summary
+
+Exit codes: 1 = user / config error; 2 = runtime error (context length cap, etc.).`,
+
   'export': `sdlab export <build|list> [args] --project <name>
 
 Build and list self-contained export packages.
@@ -306,6 +369,13 @@ function printHelp() {
   console.log('  painterly            Post-processing painterly style pass');
   console.log('  painterly:test       Calibrate painterly denoise level on sample images');
   console.log('');
+  console.log('Canon:');
+  console.log('  canon build            Build dataset.jsonl + prompts/*.j2 + context/*.md from canon entries');
+  console.log('  canon freeze <id>      Stamp a freeze block on an entry (--reason required)');
+  console.log('  canon unfreeze <id>    Lift a freeze back to status=auto (--reason required)');
+  console.log('  canon freeze-status <id> Glance at an entry\'s freeze state');
+  console.log('  canon drift            Report drift on frozen entries + overrides since last build');
+  console.log('');
   console.log('Dataset:');
   console.log('  snapshot create      Create a frozen dataset snapshot');
   console.log('  snapshot list        List all snapshots');
@@ -444,6 +514,7 @@ async function main() {
     twoWord('run', RUN_COMMANDS),
     twoWord('batch', BATCH_COMMANDS),
     twoWord('selection', SELECTION_COMMANDS),
+    twoWord('canon', CANON_COMMANDS),
   ];
 
   // "project", "workflow", "brief", "run", "batch", "selection" two-word forms

package/lib/adapters/ai-toolkit.js

@@ -0,0 +1,209 @@
+/**
+ * Ostris ai-toolkit adapter.
+ *
+ * Consumes canonical rows (per D6) and emits a training package ai-toolkit
+ * can consume directly:
+ *
+ *   <package_dir>/
+ *     dataset/<partition>/<entity_id>.png
+ *     dataset/<partition>/<entity_id>.txt          (caption sidecar)
+ *     metadata/<partition>.jsonl                   (audit rows)
+ *     ai-toolkit-config.yaml                       (training config)
+ *
+ * Precondition: profile.target_family must be 'flux'. ai-toolkit does support
+ * SDXL, but this adapter only emits Flux-shaped configs — callers using SDXL
+ * should use diffusers-lora instead. Rejecting here keeps config output honest.
+ */
+
+import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { existsSync } from 'node:fs';
+import YAML from 'yaml';
+import { deriveStyleTrigger } from '../captions.js';
+import { groupRowsByPartition } from '../rows.js';
+import { inputError } from '../errors.js';
+
+/**
+ * Build training package in ai-toolkit format from canonical rows.
+ */
+export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
+  if (profile?.target_family !== 'flux') {
+    throw inputError(
+      'ADAPTER_TARGET_FAMILY_MISMATCH',
+      `ai-toolkit adapter requires profile.target_family === 'flux' (got "${profile?.target_family}")`,
+      'Use diffusers-lora for SDXL targets, or set target_family: "flux" on the profile.',
+    );
+  }
+
+  const partitioned = groupRowsByPartition(rows || []);
+
+  let imageCount = 0;
+
+  for (const [partition, partitionRows] of Object.entries(partitioned)) {
+    const datasetDir = join(packageDir, 'dataset', partition);
+    const metadataDir = join(packageDir, 'metadata');
+    await mkdir(datasetDir, { recursive: true });
+    await mkdir(metadataDir, { recursive: true });
+
+    const metadataRows = [];
+
+    for (const row of partitionRows) {
+      if (!row.asset_path) continue;
+
+      const sourcePath = join(projectRoot, row.asset_path);
+      if (!existsSync(sourcePath)) continue;
+
+      const sourceBasename = row.asset_path.split(/[\\/]/).pop();
+      const extMatch = sourceBasename.match(/\.[^.]+$/);
+      const ext = extMatch ? extMatch[0] : '';
+      const filename = `${row.entity_id}${ext}`;
+      const stem = row.entity_id;
+      const destPath = join(datasetDir, filename);
+
+      try {
+        if (copy) {
+          await copyFile(sourcePath, destPath);
+        } else {
+          await symlink(relative(datasetDir, sourcePath), destPath);
+        }
+        imageCount++;
+      } catch {
+        continue;
+      }
+
+      await writeFile(join(datasetDir, `${stem}.txt`), row.caption + '\n');
+
+      metadataRows.push({
+        file: `dataset/${partition}/${filename}`,
+        caption_file: `dataset/${partition}/${stem}.txt`,
+        record_id: row.entity_id,
+        caption: row.caption,
+        lane: row.lane,
+        group: row.group ?? null,
+        subject: row.subject_filter_key ?? null,
+        pass_ratio: row.pass_ratio ?? null,
+      });
+    }
+
+    await writeFile(
+      join(metadataDir, `${partition}.jsonl`),
+      metadataRows.map(r => JSON.stringify(r)).join('\n') + (metadataRows.length ? '\n' : '')
+    );
+  }
+
+  const aiToolkitConfig = buildAiToolkitConfig({ profile, manifest });
+  await writeFile(
+    join(packageDir, 'ai-toolkit-config.yaml'),
+    YAML.stringify(aiToolkitConfig)
+  );
+
+  return { imageCount };
+}
+
+/**
+ * Build the ai-toolkit YAML config object from a profile + manifest.
+ *
+ * Defaults follow ai-toolkit's published Flux examples and the 16GB-VRAM
+ * ceiling documented in the slice-3 handoff. Users can override any field
+ * by editing the emitted YAML before running ai-toolkit.
+ */
+export function buildAiToolkitConfig({ profile, manifest }) {
+  const trigger = deriveStyleTrigger(profile) || profile.profile_id;
+  const baseModel = profile.base_model_recommendations?.[0] || 'black-forest-labs/FLUX.1-dev';
+  const sourceExportId = manifest?.source_export_id || 'unknown';
+
+  // Per-profile training_hyperparameters override the World-LoRA-tuned defaults.
+  // World LoRAs: rank 32, steps 2500 (style absorption across a domain).
+  // Per-character LoRAs: rank 16, alpha 8, steps 2000 (identity on a single
+  // subject — higher rank just overfits face details). See the two-LoRA stack
+  // contract in the handbook + memory/two-lora-stack-research-2026-04-23.md.
+  //
+  // TODO (normalization): the implicit World-LoRA fallback is a transition-era
+  // convenience for existing profiles that pre-date training_hyperparameters.
+  // Once every shipping profile declares training_hyperparameters explicitly,
+  // the ?? fallbacks can be removed in favor of a required block — preventing
+  // the "is this World-tuned by omission or by intent?" ambiguity that grows
+  // once a second World project exists.
+  const hp = profile.training_hyperparameters || {};
+  const linear = hp.rank ?? 32;
+  const linearAlpha = hp.alpha ?? linear;
+  const trainSteps = hp.steps ?? 2500;
+
+  return {
+    job: 'extension',
+    config: {
+      name: `${profile.profile_id}-${sourceExportId}`,
+      process: [
+        {
+          type: 'sd_trainer',
+          training_folder: 'output',
+          device: 'cuda:0',
+          network: {
+            type: 'lora',
+            linear: linear,
+            linear_alpha: linearAlpha,
+          },
+          save: {
+            dtype: 'float16',
+            save_every: 250,
+            max_step_saves_to_keep: 4,
+          },
+          datasets: [
+            {
+              folder_path: './dataset/train',
+              caption_ext: 'txt',
+              caption_dropout_rate: 0.05,
+              shuffle_tokens: false,
+              cache_latents_to_disk: true,
+              resolution: [512, 768, 1024],
+            },
+          ],
+          train: {
+            batch_size: 1,
+            steps: trainSteps,
+            gradient_accumulation_steps: 1,
+            train_unet: true,
+            train_text_encoder: false,
+            gradient_checkpointing: true,
+            noise_scheduler: 'flowmatch',
+            optimizer: 'adamw8bit',
+            lr: 4e-4,
+            ema_config: {
+              use_ema: true,
+              ema_decay: 0.99,
+            },
+            dtype: 'bf16',
+          },
+          model: {
+            name_or_path: baseModel,
+            is_flux: true,
+            quantize: true,
+          },
+          // ai-toolkit's is_style flag: true for world/style LoRAs that teach
+          // aesthetics broadly; false (default) for subject LoRAs that teach a
+          // specific character or object. Drives ai-toolkit's training
+          // regularization — wrong value silently mistrains. See the two-LoRA
+          // stack research (memory/two-lora-stack-research-2026-04-23.md) for
+          // the World LoRA vs per-character LoRA convention.
+          is_style: profile.is_style_lora === true,
+          sample: {
+            sampler: 'flowmatch',
+            sample_every: 250,
+            width: 1024,
+            height: 1024,
+            prompts: [`${trigger} style sample prompt`],
+            neg: '',
+            seed: 42,
+            walk_seed: true,
+            guidance_scale: 4,
+            sample_steps: 20,
+          },
+        },
+      ],
+    },
+    meta: {
+      name: profile.profile_id,
+      version: '1.0',
+    },
+  };
+}

package/lib/adapters/comfyui-workflows.js

@@ -44,11 +44,118 @@ export function resolveTemplateId(outputMode) {
   return 'txt2img-standard';
 }
 
+// ─── Qwen-Image native defaults ──────────────────────────────────
+// The studio's verified NON-ANIME base (Apache-2.0, on rig). See
+// memory/studio-sprite-pipeline-2026-06.md + memory/feedback_no_anime.md.
+// These mirror scripts/qwen_generate.py exactly so both runners emit an
+// identical graph for a given (prompt, seed) — PIN_PER_STEP reproducibility.
+export const QWEN_DEFAULTS = Object.freeze({
+  unet: 'qwen_image_fp8_e4m3fn.safetensors',
+  clip: 'qwen_2.5_vl_7b_fp8_scaled.safetensors',
+  vae: 'qwen_image_vae.safetensors',
+  clip_type: 'qwen_image',
+  weight_dtype: 'default',
+  shift: 3.1,
+  sampler: 'euler',
+  scheduler: 'simple',
+  steps: 22,
+  cfg: 3.5,
+  width: 1024,
+  height: 1024,
+});
+
+/**
+ * Build the Qwen-Image ComfyUI prompt graph.
+ *
+ * Diffusion-transformer base (UNETLoader + qwen_image CLIP + SD3 latent +
+ * AuraFlow model-sampling shift) — NOT an SDXL checkpoint. This is the studio's
+ * non-anime path; never wire an anime checkpoint here. Node ids and wiring
+ * match scripts/qwen_generate.py so output is reproducible across runners.
+ *
+ * `loras` chains DiT-only adapters (LoraLoaderModelOnly) between the
+ * UNETLoader and ModelSamplingAuraFlow. Model-only on purpose: studio Qwen
+ * style LoRAs train the transformer with the text encoder frozen, so there
+ * is no clip strength to wire (unlike the SDXL LoraLoader path). LoRA node
+ * ids start at '40' — also mirrored in qwen_generate.py.
+ *
+ * @returns {{ nodes: Object, saveNodeId: string }}
+ */
+export function buildQwenGraph({
+  prompt,
+  negativePrompt,
+  seed,
+  width = QWEN_DEFAULTS.width,
+  height = QWEN_DEFAULTS.height,
+  steps = QWEN_DEFAULTS.steps,
+  cfg = QWEN_DEFAULTS.cfg,
+  sampler = QWEN_DEFAULTS.sampler,
+  scheduler = QWEN_DEFAULTS.scheduler,
+  shift = QWEN_DEFAULTS.shift,
+  unet = QWEN_DEFAULTS.unet,
+  clip = QWEN_DEFAULTS.clip,
+  vae = QWEN_DEFAULTS.vae,
+  clipType = QWEN_DEFAULTS.clip_type,
+  weightDtype = QWEN_DEFAULTS.weight_dtype,
+  loras = [],
+  filenamePrefix = 'sdl_run',
+}) {
+  const nodes = {
+    '37': { class_type: 'UNETLoader', inputs: { unet_name: unet, weight_dtype: weightDtype } },
+    '38': { class_type: 'CLIPLoader', inputs: { clip_name: clip, type: clipType } },
+    '39': { class_type: 'VAELoader', inputs: { vae_name: vae } },
+    '6': { class_type: 'CLIPTextEncode', inputs: { text: prompt, clip: ['38', 0] } },
+    '7': { class_type: 'CLIPTextEncode', inputs: { text: negativePrompt, clip: ['38', 0] } },
+  };
+
+  // DiT LoRA chain: 37 → 40 → 41 → … → ModelSamplingAuraFlow.
+  let modelOut = ['37', 0];
+  let loraId = 40;
+  for (const lora of loras) {
+    const id = String(loraId++);
+    nodes[id] = {
+      class_type: 'LoraLoaderModelOnly',
+      inputs: {
+        lora_name: lora.name,
+        strength_model: lora.weight ?? 1.0,
+        model: modelOut,
+      },
+    };
+    modelOut = [id, 0];
+  }
+
+  Object.assign(nodes, {
+    '66': { class_type: 'ModelSamplingAuraFlow', inputs: { shift, model: modelOut } },
+    '5': { class_type: 'EmptySD3LatentImage', inputs: { width, height, batch_size: 1 } },
+    '3': {
+      class_type: 'KSampler',
+      inputs: {
+        seed,
+        steps,
+        cfg,
+        sampler_name: sampler,
+        scheduler,
+        denoise: 1.0,
+        model: ['66', 0],
+        positive: ['6', 0],
+        negative: ['7', 0],
+        latent_image: ['5', 0],
+      },
+    },
+    '8': { class_type: 'VAEDecode', inputs: { samples: ['3', 0], vae: ['39', 0] } },
+    '9': { class_type: 'SaveImage', inputs: { filename_prefix: filenamePrefix, images: ['8', 0] } },
+  });
+  return { nodes, saveNodeId: '9' };
+}
+
 // ─── Graph builder ───────────────────────────────────────────────
 
 /**
  * Build a ComfyUI prompt graph from a compiled brief + project config.
  *
+ * Dispatches on `projectDefaults.base`: `"qwen-image"` emits the Qwen graph
+ * (non-anime studio base); anything else (default) emits the SDXL checkpoint →
+ * LoRA chain → KSampler graph. The SDXL path is unchanged.
+ *
  * @param {Object} opts
  * @param {Object} opts.brief — compiled brief from brief-compiler
  * @param {Object} opts.projectDefaults — project.json defaults block
@@ -59,10 +166,34 @@ export function buildWorkflowGraph({ brief, projectDefaults, seed }) {
   const rp = brief.runtime_plan;
   const defaults = projectDefaults;
 
-  const checkpoint = defaults.checkpoint;
-  const loras = defaults.loras || [];
   const prompt = brief.prompt;
   const negativePrompt = brief.negative_prompt;
+
+  // ── Qwen-Image branch (non-anime studio base) ──
+  if (String(defaults.base || '').toLowerCase() === 'qwen-image') {
+    return buildQwenGraph({
+      prompt,
+      negativePrompt,
+      seed,
+      width: rp.width || defaults.width || QWEN_DEFAULTS.width,
+      height: rp.height || defaults.height || QWEN_DEFAULTS.height,
+      steps: rp.steps || defaults.steps || QWEN_DEFAULTS.steps,
+      cfg: rp.cfg || defaults.cfg || QWEN_DEFAULTS.cfg,
+      sampler: rp.sampler || defaults.sampler || QWEN_DEFAULTS.sampler,
+      scheduler: rp.scheduler || defaults.scheduler || QWEN_DEFAULTS.scheduler,
+      shift: defaults.shift ?? QWEN_DEFAULTS.shift,
+      unet: defaults.unet || QWEN_DEFAULTS.unet,
+      clip: defaults.clip || QWEN_DEFAULTS.clip,
+      vae: defaults.vae || QWEN_DEFAULTS.vae,
+      clipType: defaults.clip_type || QWEN_DEFAULTS.clip_type,
+      weightDtype: defaults.weight_dtype || QWEN_DEFAULTS.weight_dtype,
+      loras: defaults.loras || [],
+    });
+  }
+
+  // ── SDXL branch (default) ──
+  const checkpoint = defaults.checkpoint;
+  const loras = defaults.loras || [];
   const width = rp.width || defaults.width || 1024;
   const height = rp.height || defaults.height || 1024;
   const steps = rp.steps || defaults.steps || 30;

package/lib/adapters/diffusers-lora.js

@@ -1,60 +1,40 @@
 /**
  * Diffusers LoRA adapter.
  *
- * Produces a layout compatible with diffusers fine-tuning scripts:
- * - dataset/<partition>/ with image files
- * - dataset/<partition>/<image>.txt caption sidecar files
- * - metadata/<partition>.jsonl with structured rows
+ * Consumes canonical rows (per D6) — either produced in memory by
+ * training-packages.js from records, or read from a canon-build
+ * dataset.jsonl. The row shape is defined in lib/rows.js and pinned by
+ * ROW_SCHEMA_VERSION.
  *
- * Does not mutate inclusion or split truth.
+ * Output layout (unchanged from pre-refactor):
+ *   dataset/<partition>/<entity_id>.<ext>        — image (copied or symlinked)
+ *   dataset/<partition>/<entity_id>.txt          — caption sidecar
+ *   metadata/<partition>.jsonl                   — per-row audit
  */
 
 import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
-import { join, relative, basename } from 'node:path';
+import { join, relative } from 'node:path';
 import { existsSync } from 'node:fs';
+import { groupRowsByPartition } from '../rows.js';
 
 /**
- * Build a diffusers-style caption from record metadata.
+ * Build training package in diffusers LoRA format from canonical rows.
+ *
+ * @param {Object} args
+ * @param {string} args.packageDir
+ * @param {Array<Object>} args.rows — canonical rows; partition field groups them
+ * @param {Object} args.profile
+ * @param {Object} args.manifest
+ * @param {Object} args.config
+ * @param {string} args.projectRoot
+ * @param {boolean} [args.copy]
  */
-function buildCaption(record, lane, group, profile) {
-  const parts = [];
-
-  // Trigger word for LoRA activation
-  if (profile.prompt_strategy === 'trigger-word' && profile.profile_id) {
-    parts.push(profile.profile_id.replace(/-/g, '_'));
-  }
-
-  // Subject identity if available
-  if (record.identity?.subject_name) {
-    parts.push(record.identity.subject_name.replace(/_/g, ' '));
-  }
-
-  // Faction / group
-  if (group && group !== 'unknown') {
-    parts.push(`${group} faction`);
-  }
-
-  // Lane context
-  if (lane) parts.push(lane);
+export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
+  const partitioned = groupRowsByPartition(rows || []);
 
-  // Use original prompt if available, otherwise build from ID
-  const prompt = record.provenance?.prompt;
-  if (prompt) {
-    parts.push(prompt.slice(0, 300));
-  } else {
-    parts.push(record.id.replace(/_/g, ' '));
-  }
-
-  return parts.join(', ');
-}
-
-/**
- * Build training package in diffusers LoRA format.
- */
-export async function buildPackage({ packageDir, partitions, profile, manifest, config, projectRoot, copy }) {
   let imageCount = 0;
 
-  for (const [partition, records] of Object.entries(partitions)) {
+  for (const [partition, partitionRows] of Object.entries(partitioned)) {
     const datasetDir = join(packageDir, 'dataset', partition);
     const metadataDir = join(packageDir, 'metadata');
     await mkdir(datasetDir, { recursive: true });
@@ -62,19 +42,17 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
 
     const metadataRows = [];
 
-    for (const { record, lane, group } of records) {
-      if (!record.asset_path) continue;
+    for (const row of partitionRows) {
+      if (!row.asset_path) continue;
 
-      const sourcePath = join(projectRoot, record.asset_path);
+      const sourcePath = join(projectRoot, row.asset_path);
       if (!existsSync(sourcePath)) continue;
 
-      // D-009: Name output files by record.id (not source basename) to
-      // avoid silent collisions when two records share a filename stem.
-      const sourceBasename = record.asset_path.split(/[\\/]/).pop();
+      const sourceBasename = row.asset_path.split(/[\\/]/).pop();
       const extMatch = sourceBasename.match(/\.[^.]+$/);
       const ext = extMatch ? extMatch[0] : '';
-      const filename = `${record.id}${ext}`;
-      const stem = record.id;
+      const filename = `${row.entity_id}${ext}`;
+      const stem = row.entity_id;
       const destPath = join(datasetDir, filename);
 
       try {
@@ -88,28 +66,30 @@ export async function buildPackage({ packageDir, partitions, profile, manifest,
         continue;
       }
 
-      // Write caption sidecar (.txt next to image)
-      const caption = buildCaption(record, lane, group, profile);
-      await writeFile(join(datasetDir, `${stem}.txt`), caption + '\n');
+      await writeFile(join(datasetDir, `${stem}.txt`), row.caption + '\n');
 
-      metadataRows.push({
-        file: `dataset/${partition}/${filename}`,
-        caption_file: `dataset/${partition}/${stem}.txt`,
-        record_id: record.id,
-        caption,
-        lane,
-        group,
-        subject: record.identity?.subject_name || null,
-        pass_ratio: record.canon?.assertion_count > 0
-          ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3) : null,
-      });
+      metadataRows.push(buildMetadataRow(row, partition, filename, stem));
     }
 
     await writeFile(
       join(metadataDir, `${partition}.jsonl`),
-      metadataRows.map(r => JSON.stringify(r)).join('\n') + '\n'
+      metadataRows.map(r => JSON.stringify(r)).join('\n') + (metadataRows.length ? '\n' : '')
     );
   }
 
   return { imageCount };
 }
+
+function buildMetadataRow(row, partition, filename, stem) {
+  const out = {
+    file: `dataset/${partition}/${filename}`,
+    caption_file: `dataset/${partition}/${stem}.txt`,
+    record_id: row.entity_id,
+    caption: row.caption,
+    lane: row.lane,
+    group: row.group ?? null,
+    subject: row.subject_filter_key ?? null,
+    pass_ratio: row.pass_ratio ?? null,
+  };
+  return out;
+}