@mcptoolshop/style-dataset-lab 3.0.1 → 3.2.0

package/bin/sdlab.js

@@ -80,6 +80,15 @@ const SELECTION_COMMANDS = {
   'show': '../scripts/selection-show.js',
 };
 
+// Two-word commands under "canon" namespace
+const CANON_COMMANDS = {
+  'build':         '../scripts/canon-build.js',
+  'freeze':        '../scripts/canon-freeze.js',
+  'unfreeze':      '../scripts/canon-unfreeze.js',
+  'freeze-status': '../scripts/canon-freeze-status.js',
+  'drift':         '../scripts/canon-drift.js',
+};
+
 // ─── Per-command help ─────────────────────────────────────────────────
 //
 // Keyed by the user-facing command (including two-word forms joined by
@@ -113,7 +122,7 @@ Flags:
   --verbose            Show per-check detail
   --quiet              Only print PASS / FAIL summary`,
 
-  'generate': `sdlab generate [<pack-path>] --project <name> [--dry-run]
+  'generate': `sdlab generate [<pack-path>] --project <name> [--dry-run] [--resume]
 
 Generate candidates from a prompt pack (JSON file under
 projects/<name>/inputs/prompts/).
@@ -124,6 +133,10 @@ Positional:
 Flags:
   --project <name>     Project to operate on
   --dry-run            Preview work without contacting ComfyUI
+  --resume             Skip subjects whose record + image are already on disk.
+                       Seeds remain stable — skipped slots still advance the
+                       seed counter so downstream outputs are bit-identical to
+                       a fresh run.
 
 Env:
   COMFY_URL            ComfyUI HTTP endpoint (default: http://127.0.0.1:8188)
@@ -131,15 +144,19 @@ Env:
 Precedence: CLI flag > env COMFY_URL > project defaults > built-in default.`,
 
   'batch generate': `sdlab batch generate --mode <id> [--subject <id>] [--theme <label>] --project <name>
+       sdlab batch generate --resume <batch_id> --project <name>
 
 Execute a batch production mode (expression sheet, silhouette pack,
 environment board, etc).
 
 Flags:
-  --mode <id>          Batch mode id (required)
+  --mode <id>          Batch mode id (required unless --resume is given)
   --subject <id>       Subject for subject-driven modes
   --theme <label>      Theme label for environment modes
   --asset <id>         Training asset reference
+  --resume <id>        Resume an existing batch — re-runs only the failed or
+                       missing slots, keeps the same batch_id, and inherits
+                       mode/subject/theme/asset from the prior manifest.
   --project <name>     Project to operate on
   --dry-run            Prepare batch without submitting to ComfyUI
   --json               Output manifest as JSON`,
@@ -256,6 +273,60 @@ Subcommands:
 Flags:
   --project <name>     Project to operate on`,
 
+  'canon freeze': `sdlab canon freeze <entity_id> --project <name> --reason "<text>" [--status frozen|soft-advisory] [--watch <fields>] [--build <sha>] [--by <name>] [--json]
+
+Stamp a freeze block on a canon entry. Requires a canon-build output to
+witness against (the "locked_at_build" field). Writes both the entry
+frontmatter AND canon-build/freeze-events.jsonl.
+
+--reason is REQUIRED — the audit record depends on it.
+
+Statuses:
+  frozen         Regen refused outright; unfreeze ceremony required.
+  soft-advisory  Refused by default; bypassable with --i-know + --reason.`,
+
+  'canon unfreeze': `sdlab canon unfreeze <entity_id> --project <name> --reason "<text>" [--by <name>] [--json]
+
+Lift a freeze back to status=auto. Preserves the freeze.overrides[] history
+on the entry (append-only) and writes an "unfreeze" event.
+
+--reason is REQUIRED — shipping studios (Sabotage, Supergiant, Pentiment)
+most often regretted not capturing why an unfreeze happened.`,
+
+  'canon freeze-status': `sdlab canon freeze-status <entity_id> --project <name> [--json]
+
+Read-only glance at an entry's freeze state. Shows status, locked_at_build,
+frozen_by, frozen_reason, watch_fields, overrides count, and event count.`,
+
+  'canon drift': `sdlab canon drift --project <name> [--since <build_hash>] [--json]
+
+Drift report. For every frozen or on-canon-change entry, computes the
+current watch-field hash and compares against the hash stamped in the
+latest canon-build manifest. Reports drifted entries and overrides since
+a given build (default: since the latest clean build).`,
+
+  'canon build': `sdlab canon build --project <name> [--full] [--no-cache] [--dry-run] [--only <id[,id...]>] [--json]
+
+Build the three canonical projections from a project's canon entity store:
+  - dataset.jsonl       (for training adapters)
+  - prompts/<id>.j2     (Jinja2 templates for ComfyUI workflow invocation)
+  - context/<id>.md     (narrative blocks for Role OS dispatch)
+
+Reads canon-build/config.json under the project directory. Produces
+<project>/canon-build/<canon_sha>/ and a content-addressable cache under
+<project>/canon-build/.cache/.
+
+Flags:
+  --project <name>   Project to operate on (required)
+  --full             Ignore cache hits; rebuild every entity
+  --no-cache         Neither read nor write the cache this run
+  --dry-run          Walk + resolve + plan; write nothing
+  --only <ids>       Limit to specific entity ids (comma-separated)
+  --json             Emit the result summary as JSON on stdout
+  --quiet            Suppress the human-readable summary
+
+Exit codes: 1 = user / config error; 2 = runtime error (context length cap, etc.).`,
+
   'export': `sdlab export <build|list> [args] --project <name>
 
 Build and list self-contained export packages.
@@ -298,6 +369,13 @@ function printHelp() {
   console.log('  painterly            Post-processing painterly style pass');
   console.log('  painterly:test       Calibrate painterly denoise level on sample images');
   console.log('');
+  console.log('Canon:');
+  console.log('  canon build            Build dataset.jsonl + prompts/*.j2 + context/*.md from canon entries');
+  console.log('  canon freeze <id>      Stamp a freeze block on an entry (--reason required)');
+  console.log('  canon unfreeze <id>    Lift a freeze back to status=auto (--reason required)');
+  console.log('  canon freeze-status <id> Glance at an entry\'s freeze state');
+  console.log('  canon drift            Report drift on frozen entries + overrides since last build');
+  console.log('');
   console.log('Dataset:');
   console.log('  snapshot create      Create a frozen dataset snapshot');
   console.log('  snapshot list        List all snapshots');
@@ -436,6 +514,7 @@ async function main() {
     twoWord('run', RUN_COMMANDS),
     twoWord('batch', BATCH_COMMANDS),
     twoWord('selection', SELECTION_COMMANDS),
+    twoWord('canon', CANON_COMMANDS),
   ];
 
   // "project", "workflow", "brief", "run", "batch", "selection" two-word forms

package/lib/adapters/ai-toolkit.js

@@ -0,0 +1,209 @@
+/**
+ * Ostris ai-toolkit adapter.
+ *
+ * Consumes canonical rows (per D6) and emits a training package ai-toolkit
+ * can consume directly:
+ *
+ *   <package_dir>/
+ *     dataset/<partition>/<entity_id>.png
+ *     dataset/<partition>/<entity_id>.txt          (caption sidecar)
+ *     metadata/<partition>.jsonl                   (audit rows)
+ *     ai-toolkit-config.yaml                       (training config)
+ *
+ * Precondition: profile.target_family must be 'flux'. ai-toolkit does support
+ * SDXL, but this adapter only emits Flux-shaped configs — callers using SDXL
+ * should use diffusers-lora instead. Rejecting here keeps config output honest.
+ */
+
+import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { existsSync } from 'node:fs';
+import YAML from 'yaml';
+import { deriveStyleTrigger } from '../captions.js';
+import { groupRowsByPartition } from '../rows.js';
+import { inputError } from '../errors.js';
+
+/**
+ * Build training package in ai-toolkit format from canonical rows.
+ */
+export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
+  if (profile?.target_family !== 'flux') {
+    throw inputError(
+      'ADAPTER_TARGET_FAMILY_MISMATCH',
+      `ai-toolkit adapter requires profile.target_family === 'flux' (got "${profile?.target_family}")`,
+      'Use diffusers-lora for SDXL targets, or set target_family: "flux" on the profile.',
+    );
+  }
+
+  const partitioned = groupRowsByPartition(rows || []);
+
+  let imageCount = 0;
+
+  for (const [partition, partitionRows] of Object.entries(partitioned)) {
+    const datasetDir = join(packageDir, 'dataset', partition);
+    const metadataDir = join(packageDir, 'metadata');
+    await mkdir(datasetDir, { recursive: true });
+    await mkdir(metadataDir, { recursive: true });
+
+    const metadataRows = [];
+
+    for (const row of partitionRows) {
+      if (!row.asset_path) continue;
+
+      const sourcePath = join(projectRoot, row.asset_path);
+      if (!existsSync(sourcePath)) continue;
+
+      const sourceBasename = row.asset_path.split(/[\\/]/).pop();
+      const extMatch = sourceBasename.match(/\.[^.]+$/);
+      const ext = extMatch ? extMatch[0] : '';
+      const filename = `${row.entity_id}${ext}`;
+      const stem = row.entity_id;
+      const destPath = join(datasetDir, filename);
+
+      try {
+        if (copy) {
+          await copyFile(sourcePath, destPath);
+        } else {
+          await symlink(relative(datasetDir, sourcePath), destPath);
+        }
+        imageCount++;
+      } catch {
+        continue;
+      }
+
+      await writeFile(join(datasetDir, `${stem}.txt`), row.caption + '\n');
+
+      metadataRows.push({
+        file: `dataset/${partition}/${filename}`,
+        caption_file: `dataset/${partition}/${stem}.txt`,
+        record_id: row.entity_id,
+        caption: row.caption,
+        lane: row.lane,
+        group: row.group ?? null,
+        subject: row.subject_filter_key ?? null,
+        pass_ratio: row.pass_ratio ?? null,
+      });
+    }
+
+    await writeFile(
+      join(metadataDir, `${partition}.jsonl`),
+      metadataRows.map(r => JSON.stringify(r)).join('\n') + (metadataRows.length ? '\n' : '')
+    );
+  }
+
+  const aiToolkitConfig = buildAiToolkitConfig({ profile, manifest });
+  await writeFile(
+    join(packageDir, 'ai-toolkit-config.yaml'),
+    YAML.stringify(aiToolkitConfig)
+  );
+
+  return { imageCount };
+}
+
+/**
+ * Build the ai-toolkit YAML config object from a profile + manifest.
+ *
+ * Defaults follow ai-toolkit's published Flux examples and the 16GB-VRAM
+ * ceiling documented in the slice-3 handoff. Users can override any field
+ * by editing the emitted YAML before running ai-toolkit.
+ */
+export function buildAiToolkitConfig({ profile, manifest }) {
+  const trigger = deriveStyleTrigger(profile) || profile.profile_id;
+  const baseModel = profile.base_model_recommendations?.[0] || 'black-forest-labs/FLUX.1-dev';
+  const sourceExportId = manifest?.source_export_id || 'unknown';
+
+  // Per-profile training_hyperparameters override the World-LoRA-tuned defaults.
+  // World LoRAs: rank 32, steps 2500 (style absorption across a domain).
+  // Per-character LoRAs: rank 16, alpha 8, steps 2000 (identity on a single
+  // subject — higher rank just overfits face details). See the two-LoRA stack
+  // contract in the handbook + memory/two-lora-stack-research-2026-04-23.md.
+  //
+  // TODO (normalization): the implicit World-LoRA fallback is a transition-era
+  // convenience for existing profiles that pre-date training_hyperparameters.
+  // Once every shipping profile declares training_hyperparameters explicitly,
+  // the ?? fallbacks can be removed in favor of a required block — preventing
+  // the "is this World-tuned by omission or by intent?" ambiguity that grows
+  // once a second World project exists.
+  const hp = profile.training_hyperparameters || {};
+  const linear = hp.rank ?? 32;
+  const linearAlpha = hp.alpha ?? linear;
+  const trainSteps = hp.steps ?? 2500;
+
+  return {
+    job: 'extension',
+    config: {
+      name: `${profile.profile_id}-${sourceExportId}`,
+      process: [
+        {
+          type: 'sd_trainer',
+          training_folder: 'output',
+          device: 'cuda:0',
+          network: {
+            type: 'lora',
+            linear: linear,
+            linear_alpha: linearAlpha,
+          },
+          save: {
+            dtype: 'float16',
+            save_every: 250,
+            max_step_saves_to_keep: 4,
+          },
+          datasets: [
+            {
+              folder_path: './dataset/train',
+              caption_ext: 'txt',
+              caption_dropout_rate: 0.05,
+              shuffle_tokens: false,
+              cache_latents_to_disk: true,
+              resolution: [512, 768, 1024],
+            },
+          ],
+          train: {
+            batch_size: 1,
+            steps: trainSteps,
+            gradient_accumulation_steps: 1,
+            train_unet: true,
+            train_text_encoder: false,
+            gradient_checkpointing: true,
+            noise_scheduler: 'flowmatch',
+            optimizer: 'adamw8bit',
+            lr: 4e-4,
+            ema_config: {
+              use_ema: true,
+              ema_decay: 0.99,
+            },
+            dtype: 'bf16',
+          },
+          model: {
+            name_or_path: baseModel,
+            is_flux: true,
+            quantize: true,
+          },
+          // ai-toolkit's is_style flag: true for world/style LoRAs that teach
+          // aesthetics broadly; false (default) for subject LoRAs that teach a
+          // specific character or object. Drives ai-toolkit's training
+          // regularization — wrong value silently mistrains. See the two-LoRA
+          // stack research (memory/two-lora-stack-research-2026-04-23.md) for
+          // the World LoRA vs per-character LoRA convention.
+          is_style: profile.is_style_lora === true,
+          sample: {
+            sampler: 'flowmatch',
+            sample_every: 250,
+            width: 1024,
+            height: 1024,
+            prompts: [`${trigger} style sample prompt`],
+            neg: '',
+            seed: 42,
+            walk_seed: true,
+            guidance_scale: 4,
+            sample_steps: 20,
+          },
+        },
+      ],
+    },
+    meta: {
+      name: profile.profile_id,
+      version: '1.0',
+    },
+  };
+}

package/lib/adapters/comfyui-runner.js

@@ -11,6 +11,7 @@ import { comfyHealth, submitAndWait, downloadImage } from '../comfyui.js';
 import { buildWorkflowGraph, resolveTemplateId } from './comfyui-workflows.js';
 import { buildSeedPlan, prepareRunDir, saveRunManifest } from '../runtime-runs.js';
 import { loadProjectConfig } from '../config.js';
+import { pickOutputImage } from '../comfyui-output.js';
 import { log, info, warn } from '../log.js';
 import { runtimeError } from '../errors.js';
 
@@ -81,7 +82,7 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
 
     try {
       // Build graph
-      const { nodes } = buildWorkflowGraph({
+      const { nodes, saveNodeId } = buildWorkflowGraph({
         brief,
         projectDefaults: defaults,
         seed,
@@ -94,26 +95,20 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
 
       const elapsed = Date.now() - startMs;
 
-      // Extract output image
-      const nodeOutputs = result.outputs || {};
-      let imageFile = null;
-      let imageSubfolder = '';
-      for (const nodeOut of Object.values(nodeOutputs)) {
-        if (nodeOut.images && nodeOut.images.length > 0) {
-          imageFile = nodeOut.images[0].filename;
-          imageSubfolder = nodeOut.images[0].subfolder || '';
-          break;
-        }
-      }
+      // Extract output image — prefer the explicit save node from the
+      // workflow builder (or `expected_outputs.save_node_id` from the
+      // brief/profile), falling back to the highest numeric node id.
+      const preferNodeId = brief.expected_outputs?.save_node_id ?? saveNodeId;
+      const picked = pickOutputImage(result.outputs, { preferNodeId });
 
-      if (!imageFile) {
+      if (!picked) {
         warn(`  No output image for seed ${seed}`);
         outputs.push({ index: i, seed, status: 'no_output', elapsed_ms: elapsed });
         continue;
       }
 
       // Download and save
-      const imgData = await downloadImage(imageFile, imageSubfolder, comfyUrl);
+      const imgData = await downloadImage(picked.filename, picked.subfolder, comfyUrl);
       const destFilename = `${imageIndex}.png`;
       await writeFile(join(runDir, 'outputs', destFilename), imgData);
 
@@ -124,7 +119,8 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
         filename: destFilename,
         bytes: imgData.length,
         elapsed_ms: elapsed,
-        comfy_filename: imageFile,
+        comfy_filename: picked.filename,
+        comfy_node_id: picked.nodeId,
       });
 
       log(`    ✓ ${destFilename} (${imgData.length} bytes, ${(elapsed / 1000).toFixed(1)}s)`);

package/lib/adapters/comfyui-workflows.js

@@ -44,11 +44,118 @@ export function resolveTemplateId(outputMode) {
   return 'txt2img-standard';
 }
 
+// ─── Qwen-Image native defaults ──────────────────────────────────
+// The studio's verified NON-ANIME base (Apache-2.0, on rig). See
+// memory/studio-sprite-pipeline-2026-06.md + memory/feedback_no_anime.md.
+// These mirror scripts/qwen_generate.py exactly so both runners emit an
+// identical graph for a given (prompt, seed) — PIN_PER_STEP reproducibility.
+export const QWEN_DEFAULTS = Object.freeze({
+  unet: 'qwen_image_fp8_e4m3fn.safetensors',
+  clip: 'qwen_2.5_vl_7b_fp8_scaled.safetensors',
+  vae: 'qwen_image_vae.safetensors',
+  clip_type: 'qwen_image',
+  weight_dtype: 'default',
+  shift: 3.1,
+  sampler: 'euler',
+  scheduler: 'simple',
+  steps: 22,
+  cfg: 3.5,
+  width: 1024,
+  height: 1024,
+});
+
+/**
+ * Build the Qwen-Image ComfyUI prompt graph.
+ *
+ * Diffusion-transformer base (UNETLoader + qwen_image CLIP + SD3 latent +
+ * AuraFlow model-sampling shift) — NOT an SDXL checkpoint. This is the studio's
+ * non-anime path; never wire an anime checkpoint here. Node ids and wiring
+ * match scripts/qwen_generate.py so output is reproducible across runners.
+ *
+ * `loras` chains DiT-only adapters (LoraLoaderModelOnly) between the
+ * UNETLoader and ModelSamplingAuraFlow. Model-only on purpose: studio Qwen
+ * style LoRAs train the transformer with the text encoder frozen, so there
+ * is no clip strength to wire (unlike the SDXL LoraLoader path). LoRA node
+ * ids start at '40' — also mirrored in qwen_generate.py.
+ *
+ * @returns {{ nodes: Object, saveNodeId: string }}
+ */
+export function buildQwenGraph({
+  prompt,
+  negativePrompt,
+  seed,
+  width = QWEN_DEFAULTS.width,
+  height = QWEN_DEFAULTS.height,
+  steps = QWEN_DEFAULTS.steps,
+  cfg = QWEN_DEFAULTS.cfg,
+  sampler = QWEN_DEFAULTS.sampler,
+  scheduler = QWEN_DEFAULTS.scheduler,
+  shift = QWEN_DEFAULTS.shift,
+  unet = QWEN_DEFAULTS.unet,
+  clip = QWEN_DEFAULTS.clip,
+  vae = QWEN_DEFAULTS.vae,
+  clipType = QWEN_DEFAULTS.clip_type,
+  weightDtype = QWEN_DEFAULTS.weight_dtype,
+  loras = [],
+  filenamePrefix = 'sdl_run',
+}) {
+  const nodes = {
+    '37': { class_type: 'UNETLoader', inputs: { unet_name: unet, weight_dtype: weightDtype } },
+    '38': { class_type: 'CLIPLoader', inputs: { clip_name: clip, type: clipType } },
+    '39': { class_type: 'VAELoader', inputs: { vae_name: vae } },
+    '6': { class_type: 'CLIPTextEncode', inputs: { text: prompt, clip: ['38', 0] } },
+    '7': { class_type: 'CLIPTextEncode', inputs: { text: negativePrompt, clip: ['38', 0] } },
+  };
+
+  // DiT LoRA chain: 37 → 40 → 41 → … → ModelSamplingAuraFlow.
+  let modelOut = ['37', 0];
+  let loraId = 40;
+  for (const lora of loras) {
+    const id = String(loraId++);
+    nodes[id] = {
+      class_type: 'LoraLoaderModelOnly',
+      inputs: {
+        lora_name: lora.name,
+        strength_model: lora.weight ?? 1.0,
+        model: modelOut,
+      },
+    };
+    modelOut = [id, 0];
+  }
+
+  Object.assign(nodes, {
+    '66': { class_type: 'ModelSamplingAuraFlow', inputs: { shift, model: modelOut } },
+    '5': { class_type: 'EmptySD3LatentImage', inputs: { width, height, batch_size: 1 } },
+    '3': {
+      class_type: 'KSampler',
+      inputs: {
+        seed,
+        steps,
+        cfg,
+        sampler_name: sampler,
+        scheduler,
+        denoise: 1.0,
+        model: ['66', 0],
+        positive: ['6', 0],
+        negative: ['7', 0],
+        latent_image: ['5', 0],
+      },
+    },
+    '8': { class_type: 'VAEDecode', inputs: { samples: ['3', 0], vae: ['39', 0] } },
+    '9': { class_type: 'SaveImage', inputs: { filename_prefix: filenamePrefix, images: ['8', 0] } },
+  });
+  return { nodes, saveNodeId: '9' };
+}
+
 // ─── Graph builder ───────────────────────────────────────────────
 
 /**
  * Build a ComfyUI prompt graph from a compiled brief + project config.
  *
+ * Dispatches on `projectDefaults.base`: `"qwen-image"` emits the Qwen graph
+ * (non-anime studio base); anything else (default) emits the SDXL checkpoint →
+ * LoRA chain → KSampler graph. The SDXL path is unchanged.
+ *
  * @param {Object} opts
  * @param {Object} opts.brief — compiled brief from brief-compiler
  * @param {Object} opts.projectDefaults — project.json defaults block
@@ -59,10 +166,34 @@ export function buildWorkflowGraph({ brief, projectDefaults, seed }) {
   const rp = brief.runtime_plan;
   const defaults = projectDefaults;
 
-  const checkpoint = defaults.checkpoint;
-  const loras = defaults.loras || [];
   const prompt = brief.prompt;
   const negativePrompt = brief.negative_prompt;
+
+  // ── Qwen-Image branch (non-anime studio base) ──
+  if (String(defaults.base || '').toLowerCase() === 'qwen-image') {
+    return buildQwenGraph({
+      prompt,
+      negativePrompt,
+      seed,
+      width: rp.width || defaults.width || QWEN_DEFAULTS.width,
+      height: rp.height || defaults.height || QWEN_DEFAULTS.height,
+      steps: rp.steps || defaults.steps || QWEN_DEFAULTS.steps,
+      cfg: rp.cfg || defaults.cfg || QWEN_DEFAULTS.cfg,
+      sampler: rp.sampler || defaults.sampler || QWEN_DEFAULTS.sampler,
+      scheduler: rp.scheduler || defaults.scheduler || QWEN_DEFAULTS.scheduler,
+      shift: defaults.shift ?? QWEN_DEFAULTS.shift,
+      unet: defaults.unet || QWEN_DEFAULTS.unet,
+      clip: defaults.clip || QWEN_DEFAULTS.clip,
+      vae: defaults.vae || QWEN_DEFAULTS.vae,
+      clipType: defaults.clip_type || QWEN_DEFAULTS.clip_type,
+      weightDtype: defaults.weight_dtype || QWEN_DEFAULTS.weight_dtype,
+      loras: defaults.loras || [],
+    });
+  }
+
+  // ── SDXL branch (default) ──
+  const checkpoint = defaults.checkpoint;
+  const loras = defaults.loras || [];
   const width = rp.width || defaults.width || 1024;
   const height = rp.height || defaults.height || 1024;
   const steps = rp.steps || defaults.steps || 30;