@mcptoolshop/style-dataset-lab 3.0.1 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +77 -0
  2. package/README.es.md +154 -41
  3. package/README.fr.md +153 -40
  4. package/README.hi.md +160 -47
  5. package/README.it.md +155 -42
  6. package/README.ja.md +159 -46
  7. package/README.md +40 -0
  8. package/README.pt-BR.md +154 -41
  9. package/README.zh.md +154 -41
  10. package/bin/sdlab.js +81 -2
  11. package/lib/adapters/ai-toolkit.js +209 -0
  12. package/lib/adapters/comfyui-runner.js +11 -15
  13. package/lib/adapters/comfyui-workflows.js +133 -2
  14. package/lib/adapters/diffusers-lora.js +46 -66
  15. package/lib/adapters/generic-image-caption.js +22 -50
  16. package/lib/batch-runs.js +62 -0
  17. package/lib/canon-build/build.js +450 -0
  18. package/lib/canon-build/emitters/base.js +122 -0
  19. package/lib/canon-build/emitters/character.js +167 -0
  20. package/lib/canon-build/emitters/deity.js +165 -0
  21. package/lib/canon-build/emitters/faction.js +161 -0
  22. package/lib/canon-build/emitters/index.js +38 -0
  23. package/lib/canon-build/emitters/location.js +153 -0
  24. package/lib/canon-build/emitters/monster.js +131 -0
  25. package/lib/canon-build/emitters/relic.js +142 -0
  26. package/lib/canon-build/emitters/ship.js +154 -0
  27. package/lib/canon-build/emitters/species.js +165 -0
  28. package/lib/canon-build/frontmatter.js +76 -0
  29. package/lib/canon-build/lane-resolve.js +64 -0
  30. package/lib/canon-build/load-config.js +117 -0
  31. package/lib/canon-build/load-entry.js +62 -0
  32. package/lib/canon-build/load-schema.js +46 -0
  33. package/lib/captions.js +197 -0
  34. package/lib/comfyui-output.js +70 -0
  35. package/lib/config.js +19 -0
  36. package/lib/freeze-events.js +109 -0
  37. package/lib/freeze-gate.js +165 -0
  38. package/lib/freeze-stamp.js +161 -0
  39. package/lib/rows.js +228 -0
  40. package/lib/training-adapters.js +84 -0
  41. package/lib/training-packages.js +25 -32
  42. package/lib/training-profiles.js +43 -0
  43. package/package.json +29 -2
  44. package/scripts/_batch_cut.py +71 -0
  45. package/scripts/_seam_cutout.py +74 -0
  46. package/scripts/batch-generate.js +56 -13
  47. package/scripts/canon-build.js +88 -0
  48. package/scripts/canon-drift.js +193 -0
  49. package/scripts/canon-freeze-status.js +85 -0
  50. package/scripts/canon-freeze.js +211 -0
  51. package/scripts/canon-unfreeze.js +137 -0
  52. package/scripts/curate.js +12 -0
  53. package/scripts/generate-controlnet.js +12 -0
  54. package/scripts/generate-identity.js +20 -4
  55. package/scripts/generate-ipadapter.js +11 -0
  56. package/scripts/generate.js +250 -102
  57. package/scripts/qwen_generate.py +124 -0
@@ -24,9 +24,10 @@ import { writeFile, mkdir } from "node:fs/promises";
24
24
  import { join } from "node:path";
25
25
  import { getProjectName, parseNumberFlag } from "../lib/args.js";
26
26
  import { REPO_ROOT, resolveSafeProjectPath } from "../lib/paths.js";
27
- import { readJsonFile } from "../lib/config.js";
27
+ import { readJsonFile, loadProjectMeta, resolveGpuModel } from "../lib/config.js";
28
28
  import { inputError, runtimeError, handleCliError } from "../lib/errors.js";
29
29
  import { comfyHealth, submitAndWait, downloadImage } from "../lib/comfyui.js";
30
+ import { assertNotFrozenBySubject } from "../lib/freeze-gate.js";
30
31
 
31
32
  // ── Defaults (match existing lab pipeline) ──
32
33
 
@@ -309,7 +310,7 @@ function buildImg2ImgWorkflow(prompt, negativePrompt, seed, imagePath, denoise)
309
310
 
310
311
  // ── Record Builder ──
311
312
 
312
- function buildRecord(pack, subject, shot, seed, seedIndex, imgData, destPath, elapsedMs, phase, anchorInfo) {
313
+ function buildRecord(pack, subject, shot, seed, seedIndex, imgData, destPath, elapsedMs, phase, anchorInfo, gpuModel) {
313
314
  const assetId = `${shot.id}_s${seedIndex}`;
314
315
  const fullPrompt = `${pack.style_prefix}, ${shot.prompt}`;
315
316
  const negativePrompt = [
@@ -343,7 +344,7 @@ function buildRecord(pack, subject, shot, seed, seedIndex, imgData, destPath, el
343
344
  width: DEFAULTS.width,
344
345
  height: DEFAULTS.height,
345
346
  generation_time_ms: elapsedMs,
346
- gpu_model: "RTX 5080",
347
+ gpu_model: gpuModel,
347
348
  denoise: phase === "follow_on" ? (anchorInfo?.denoise || 0.38) : 1.0,
348
349
  },
349
350
  identity: {
@@ -379,6 +380,7 @@ export async function run(argv = process.argv.slice(2)) {
379
380
  const projectName = getProjectName(argv);
380
381
  const GAME_ROOT = join(REPO_ROOT, 'projects', projectName);
381
382
  const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
383
+ const gpuModel = resolveGpuModel(loadProjectMeta(GAME_ROOT)?.defaults);
382
384
 
383
385
  // Collect positionals, skipping values that belong to known flags
384
386
  const knownFlagsWithValue = new Set(['--subject', '--seeds', '--phase', '--anchor', '--denoise', '--project', '--game']);
@@ -396,6 +398,20 @@ export async function run(argv = process.argv.slice(2)) {
396
398
  const subjectFilter = argv.includes("--subject")
397
399
  ? argv[argv.indexOf("--subject") + 1]
398
400
  : null;
401
+
402
+ // Freeze gate (advisory): when --subject is given, refuse if that subject's
403
+ // canon entry is frozen. No-op when --subject is absent (pack-driven) or
404
+ // when the project has no canon-build config.
405
+ if (subjectFilter) {
406
+ const reasonFlagIdx = argv.indexOf('--reason');
407
+ const bypassReason = reasonFlagIdx >= 0 && argv[reasonFlagIdx + 1] ? argv[reasonFlagIdx + 1] : null;
408
+ await assertNotFrozenBySubject(GAME_ROOT, subjectFilter, {
409
+ action: 'generate:identity',
410
+ allowSoftAdvisoryBypass: argv.includes('--i-know'),
411
+ bypassReason,
412
+ by: 'mike',
413
+ });
414
+ }
399
415
  const seedCount = argv.includes("--seeds")
400
416
  ? parseNumberFlag('seeds', argv[argv.indexOf("--seeds") + 1], { int: true, min: 1 })
401
417
  : 3;
@@ -536,7 +552,7 @@ export async function run(argv = process.argv.slice(2)) {
536
552
 
537
553
  const record = buildRecord(
538
554
  pack, subject, shot, seed, si,
539
- imgData, destPath, elapsed, phase, anchorInfo
555
+ imgData, destPath, elapsed, phase, anchorInfo, gpuModel
540
556
  );
541
557
 
542
558
  await writeFile(
@@ -17,6 +17,7 @@ import { getProjectName, parseNumberFlag } from "../lib/args.js";
17
17
  import { REPO_ROOT, resolveSafeProjectPath } from "../lib/paths.js";
18
18
  import { inputError, runtimeError, handleCliError } from "../lib/errors.js";
19
19
  import { comfyHealth, submitAndWait, downloadImage, uploadImage } from "../lib/comfyui.js";
20
+ import { assertNotFrozenBySubject } from "../lib/freeze-gate.js";
20
21
 
21
22
  const DEFAULTS = {
22
23
  checkpoint: "dreamshaperXL_v21TurboDPMSDE.safetensors",
@@ -186,6 +187,16 @@ export async function run(argv = process.argv.slice(2)) {
186
187
  throw inputError('INPUT_MISSING_FLAG', "--prompt is required");
187
188
  }
188
189
 
190
+ // Freeze gate (advisory): refuse if the subject's canon entry is frozen.
191
+ const reasonFlagIdx = argv.indexOf('--reason');
192
+ const bypassReason = reasonFlagIdx >= 0 && argv[reasonFlagIdx + 1] ? argv[reasonFlagIdx + 1] : null;
193
+ await assertNotFrozenBySubject(GAME_ROOT, opts.subject, {
194
+ action: 'generate:ipadapter',
195
+ allowSoftAdvisoryBypass: argv.includes('--i-know'),
196
+ bypassReason,
197
+ by: 'mike',
198
+ });
199
+
189
200
  // Validate --ref path stays inside project tree.
190
201
  const safeRefPath = resolveSafeProjectPath(GAME_ROOT, opts.refPath, { flagName: 'ref' });
191
202
 
@@ -11,12 +11,15 @@
11
11
  */
12
12
 
13
13
  import { writeFile, mkdir } from "node:fs/promises";
14
+ import { existsSync } from "node:fs";
14
15
  import { join } from "node:path";
15
16
  import { parseArgs, getProjectName } from "../lib/args.js";
16
17
  import { REPO_ROOT, resolveSafeProjectPath } from "../lib/paths.js";
17
- import { readJsonFile } from "../lib/config.js";
18
- import { runtimeError, handleCliError } from "../lib/errors.js";
18
+ import { readJsonFile, loadProjectMeta, resolveGpuModel } from "../lib/config.js";
19
+ import { runtimeError, inputError, handleCliError } from "../lib/errors.js";
19
20
  import { comfyHealth, submitAndWait, downloadImage } from "../lib/comfyui.js";
21
+ import { pickOutputImage } from "../lib/comfyui-output.js";
22
+ import { buildQwenGraph, QWEN_DEFAULTS } from "../lib/adapters/comfyui-workflows.js";
20
23
  import { info, result } from "../lib/log.js";
21
24
 
22
25
  /**
@@ -175,6 +178,89 @@ function buildWorkflow(prompt, negativePrompt, checkpoint, loras, seed, steps, c
175
178
  return { nodes, saveNodeId: saveId };
176
179
  }
177
180
 
181
+ /**
182
+ * Detect the input schema and normalize to a unified shape:
183
+ * { lane, style_prefix, negative, defaults, base, jobs:[{assetId, subjectId, variationId, prompt, seed, loras}] }
184
+ *
185
+ * Two accepted schemas (both produce jobs in subject-major order, so the legacy
186
+ * seed sequence stays byte-identical):
187
+ *
188
+ * • Legacy pack — top-level `lane` + `variations` (array) applied to every
189
+ * subject. seed = base_seed + jobIndex + (variation.seed_offset||0);
190
+ * assetId = `${subject.id}_${variation.id}`; `negative` is top-level.
191
+ *
192
+ * • Wave (what `sdlab init` scaffolds, e.g. example-wave.json) — each subject
193
+ * carries `variations` as an integer count; `negative` lives in `defaults`.
194
+ * seed = base_seed + jobIndex; assetId = `${subject.id}_v${v}`. This matches
195
+ * scripts/qwen_generate.py's seed/filename scheme.
196
+ *
197
+ * `base` resolves: input.base → input.defaults.base → project.json defaults.base
198
+ * → "sdxl". `"qwen-image"` routes to the non-anime Qwen graph.
199
+ */
200
+ function normalizeGenerationInput(raw, projectMeta) {
201
+ if (!Array.isArray(raw.subjects) || raw.subjects.length === 0) {
202
+ throw inputError(
203
+ 'INPUT_MISSING_FIELDS',
204
+ 'Prompt pack/wave must have a non-empty "subjects" array.',
205
+ 'See inputs/prompts/example-wave.json for the expected shape.'
206
+ );
207
+ }
208
+ const defaults = raw.defaults || {};
209
+ const base = String(
210
+ raw.base || defaults.base || projectMeta?.defaults?.base || 'sdxl'
211
+ ).toLowerCase();
212
+ const baseSeed = Number.isFinite(defaults.base_seed) ? defaults.base_seed : 0;
213
+ const stylePrefix = raw.style_prefix || '';
214
+ const compose = (p) => (stylePrefix ? `${stylePrefix}, ${p}` : p);
215
+
216
+ const jobs = [];
217
+ let k = 0;
218
+
219
+ if (Array.isArray(raw.variations)) {
220
+ // Legacy pack schema
221
+ if (!raw.lane) {
222
+ throw inputError(
223
+ 'INPUT_MISSING_FIELDS',
224
+ 'Legacy pack (top-level "variations" array) requires a "lane".',
225
+ 'Add a "lane" string, or use the wave schema (per-subject integer "variations").'
226
+ );
227
+ }
228
+ const negative = raw.negative ?? '';
229
+ for (const subject of raw.subjects) {
230
+ for (const variation of raw.variations) {
231
+ jobs.push({
232
+ assetId: `${subject.id}_${variation.id}`,
233
+ subjectId: subject.id,
234
+ variationId: variation.id,
235
+ prompt: compose(subject.prompt),
236
+ seed: baseSeed + k + (variation.seed_offset || 0),
237
+ loras: variation.loras || [],
238
+ });
239
+ k++;
240
+ }
241
+ }
242
+ return { lane: raw.lane, style_prefix: stylePrefix, negative, defaults, base, jobs, schema: 'pack' };
243
+ }
244
+
245
+ // Wave schema
246
+ const negative = raw.negative ?? defaults.negative ?? '';
247
+ for (const subject of raw.subjects) {
248
+ const nvar = Number.isFinite(subject.variations) ? subject.variations : 1;
249
+ for (let v = 0; v < nvar; v++) {
250
+ jobs.push({
251
+ assetId: `${subject.id}_v${v}`,
252
+ subjectId: subject.id,
253
+ variationId: `v${v}`,
254
+ prompt: compose(subject.prompt),
255
+ seed: baseSeed + k,
256
+ loras: [],
257
+ });
258
+ k++;
259
+ }
260
+ }
261
+ return { lane: raw.wave || raw.lane || 'default', style_prefix: stylePrefix, negative, defaults, base, jobs, schema: 'wave' };
262
+ }
263
+
178
264
  export async function run(argv = process.argv.slice(2)) {
179
265
  const projectName = getProjectName(argv);
180
266
  const GAME_ROOT = join(REPO_ROOT, 'projects', projectName);
@@ -185,20 +271,49 @@ export async function run(argv = process.argv.slice(2)) {
185
271
  project: { type: 'string' },
186
272
  game: { type: 'string' },
187
273
  'dry-run': { type: 'boolean' },
274
+ resume: { type: 'boolean' },
188
275
  },
189
276
  deprecated: { game: 'project' },
190
277
  });
191
278
  const packPath = parsed.positionals[0] || "inputs/prompts/rpg-icons-lane1.json";
192
279
  const dryRun = parsed.flags['dry-run'] === true;
280
+ const resume = parsed.flags['resume'] === true;
193
281
 
194
282
  const fullPackPath = resolveSafeProjectPath(GAME_ROOT, packPath, { flagName: 'pack-path' });
195
- const pack = await readJsonFile(fullPackPath, { requiredFields: ['lane', 'subjects', 'variations', 'defaults'] });
283
+ const raw = await readJsonFile(fullPackPath);
284
+ const projectMeta = loadProjectMeta(GAME_ROOT);
285
+ const gpuModel = resolveGpuModel(projectMeta?.defaults);
286
+ const pack = normalizeGenerationInput(raw, projectMeta);
287
+
288
+ const d = pack.defaults;
289
+ const base = pack.base;
290
+ const isQwen = base === 'qwen-image';
291
+
292
+ // Resolve effective sampler params once. Qwen applies its own defaults when a
293
+ // wave omits them; SDXL uses the pack defaults exactly as authored (unchanged).
294
+ const resolved = isQwen
295
+ ? {
296
+ width: d.width || QWEN_DEFAULTS.width,
297
+ height: d.height || QWEN_DEFAULTS.height,
298
+ steps: d.steps || QWEN_DEFAULTS.steps,
299
+ cfg: d.cfg || QWEN_DEFAULTS.cfg,
300
+ sampler: d.sampler || QWEN_DEFAULTS.sampler,
301
+ scheduler: d.scheduler || QWEN_DEFAULTS.scheduler,
302
+ }
303
+ : {
304
+ width: d.width,
305
+ height: d.height,
306
+ steps: d.steps,
307
+ cfg: d.cfg,
308
+ sampler: d.sampler,
309
+ scheduler: d.scheduler,
310
+ };
196
311
 
197
312
  console.log(`\x1b[1mstyle-dataset-lab\x1b[0m generate`);
198
313
  console.log(` Lane: ${pack.lane}`);
199
- console.log(` Subjects: ${pack.subjects.length}`);
200
- console.log(` Variations: ${pack.variations.length}`);
201
- console.log(` Total candidates: ${pack.subjects.length * pack.variations.length}`);
314
+ console.log(` Base: ${base}${isQwen ? ' \x1b[2m(non-anime)\x1b[0m' : ''}`);
315
+ console.log(` Subjects: ${raw.subjects.length}`);
316
+ console.log(` Total candidates: ${pack.jobs.length}`);
202
317
  console.log("");
203
318
 
204
319
  if (!dryRun) {
@@ -212,122 +327,155 @@ export async function run(argv = process.argv.slice(2)) {
212
327
  await mkdir(join(GAME_ROOT, "outputs/candidates"), { recursive: true });
213
328
  await mkdir(join(GAME_ROOT, "records"), { recursive: true });
214
329
 
215
- const d = pack.defaults;
216
330
  let generated = 0;
217
331
  let errors = 0;
218
- const totalExpected = pack.subjects.length * pack.variations.length;
332
+ let skipped = 0;
333
+ const totalExpected = pack.jobs.length;
219
334
  const runStartMs = Date.now();
220
335
 
221
- for (const subject of pack.subjects) {
222
- for (const variation of pack.variations) {
223
- const assetId = `${subject.id}_${variation.id}`;
224
- const seed = (d.base_seed + generated) + (variation.seed_offset || 0);
225
- const fullPrompt = `${pack.style_prefix}, ${subject.prompt}`;
226
- const loras = variation.loras || [];
227
-
228
- console.log(` [${generated + 1}/${totalExpected}] ${assetId} (seed: ${seed}, loras: ${loras.length})`);
336
+ for (const job of pack.jobs) {
337
+ const { assetId, prompt: fullPrompt, seed, loras } = job;
229
338
 
230
- if (dryRun) {
339
+ // --resume: skip jobs whose record + image are both already on disk.
340
+ // Seeds are precomputed per job index, so skipping never shifts later seeds.
341
+ if (resume && !dryRun) {
342
+ const recordPath = join(GAME_ROOT, `records/${assetId}.json`);
343
+ const imagePath = join(GAME_ROOT, `outputs/candidates/${assetId}.png`);
344
+ if (existsSync(recordPath) && existsSync(imagePath)) {
345
+ console.log(` [${generated + 1}/${totalExpected}] ${assetId} \x1b[2m(resumed — skipped)\x1b[0m`);
231
346
  generated++;
347
+ skipped++;
232
348
  continue;
233
349
  }
350
+ }
234
351
 
235
- const startMs = Date.now();
236
- const { nodes } = buildWorkflow(
237
- fullPrompt, pack.negative,
238
- d.checkpoint, loras, seed,
239
- d.steps, d.cfg, d.sampler, d.scheduler,
240
- d.width, d.height,
241
- );
352
+ console.log(` [${generated + 1}/${totalExpected}] ${assetId} (seed: ${seed}, loras: ${isQwen ? (d.loras || []).length : loras.length})`);
242
353
 
243
- try {
244
- const result = await submitAndWait(nodes, COMFY_URL);
245
- const elapsed = Date.now() - startMs;
246
-
247
- // Find output image
248
- const outputs = result.outputs || {};
249
- let imageFile = null;
250
- let imageSubfolder = "";
251
- for (const nodeOut of Object.values(outputs)) {
252
- if (nodeOut.images && nodeOut.images.length > 0) {
253
- imageFile = nodeOut.images[0].filename;
254
- imageSubfolder = nodeOut.images[0].subfolder || "";
255
- break;
256
- }
257
- }
258
-
259
- if (!imageFile) {
260
- console.log(` \x1b[33m⚠\x1b[0m No output image found`);
261
- generated++;
262
- continue;
263
- }
264
-
265
- // Download and save
266
- const imgData = await downloadImage(imageFile, imageSubfolder, COMFY_URL);
267
- const destPath = `outputs/candidates/${assetId}.png`;
268
- await writeFile(join(GAME_ROOT, destPath), imgData);
269
-
270
- // Write provenance record
271
- const record = {
272
- id: assetId,
273
- schema_version: "1.0.0",
274
- created_at: new Date().toISOString(),
275
- asset_path: destPath,
276
- image: { format: "png", width: d.width, height: d.height, bytes: imgData.length },
277
- provenance: {
278
- workflow_id: pack.lane,
279
- workflow_version: "1.0.0",
280
- checkpoint: d.checkpoint,
281
- loras,
282
- prompt: fullPrompt,
283
- negative_prompt: pack.negative,
284
- seed,
285
- steps: d.steps,
286
- cfg: d.cfg,
287
- sampler: d.sampler,
288
- scheduler: d.scheduler,
289
- width: d.width,
290
- height: d.height,
291
- generation_time_ms: elapsed,
292
- gpu_model: "RTX 5080",
293
- batch_index: generated,
294
- },
295
- judgment: null,
296
- canon: null,
297
- iteration: null,
298
- };
299
-
300
- await writeFile(
301
- join(GAME_ROOT, `records/${assetId}.json`),
302
- JSON.stringify(record, null, 2),
354
+ if (dryRun) {
355
+ generated++;
356
+ continue;
357
+ }
358
+
359
+ const startMs = Date.now();
360
+ const { nodes, saveNodeId } = isQwen
361
+ ? buildQwenGraph({
362
+ prompt: fullPrompt,
363
+ negativePrompt: pack.negative,
364
+ seed,
365
+ width: resolved.width,
366
+ height: resolved.height,
367
+ steps: resolved.steps,
368
+ cfg: resolved.cfg,
369
+ sampler: resolved.sampler,
370
+ scheduler: resolved.scheduler,
371
+ shift: d.shift ?? QWEN_DEFAULTS.shift,
372
+ unet: d.unet || QWEN_DEFAULTS.unet,
373
+ clip: d.clip || QWEN_DEFAULTS.clip,
374
+ vae: d.vae || QWEN_DEFAULTS.vae,
375
+ clipType: d.clip_type || QWEN_DEFAULTS.clip_type,
376
+ weightDtype: d.weight_dtype || QWEN_DEFAULTS.weight_dtype,
377
+ loras: d.loras || [],
378
+ filenamePrefix: 'sdl',
379
+ })
380
+ : buildWorkflow(
381
+ fullPrompt, pack.negative,
382
+ d.checkpoint, loras, seed,
383
+ resolved.steps, resolved.cfg, resolved.sampler, resolved.scheduler,
384
+ resolved.width, resolved.height,
303
385
  );
304
386
 
305
- console.log(` \x1b[32m✓\x1b[0m ${destPath} (${elapsed}ms, ${imgData.length} bytes)`);
306
- } catch (err) {
307
- console.log(` \x1b[31m✗\x1b[0m ${err.message}`);
308
- errors++;
309
- }
387
+ try {
388
+ const submitResult = await submitAndWait(nodes, COMFY_URL);
389
+ const elapsed = Date.now() - startMs;
310
390
 
311
- generated++;
391
+ // Prefer the explicit SaveImage node from the builder; fall back
392
+ // to highest numeric node id (see lib/comfyui-output.js).
393
+ const picked = pickOutputImage(submitResult.outputs, { preferNodeId: saveNodeId });
394
+ if (!picked) {
395
+ console.log(` \x1b[33m⚠\x1b[0m No output image found`);
396
+ generated++;
397
+ continue;
398
+ }
312
399
 
313
- // ETA every 5 items (not every item — too noisy for long runs).
314
- if (!dryRun && generated > 0 && generated % 5 === 0 && generated < totalExpected) {
315
- const elapsedTotal = Date.now() - runStartMs;
316
- const avgMs = elapsedTotal / generated;
317
- const remaining = totalExpected - generated;
318
- const etaMs = avgMs * remaining;
319
- info('generate', `progress ${generated}/${totalExpected} — avg ${formatEta(avgMs)}/item, ETA ~${formatEta(etaMs)}`);
400
+ // Download and save
401
+ const imgData = await downloadImage(picked.filename, picked.subfolder, COMFY_URL);
402
+ const destPath = `outputs/candidates/${assetId}.png`;
403
+ await writeFile(join(GAME_ROOT, destPath), imgData);
404
+
405
+ // Write provenance record (base-aware: Qwen records unet/clip/vae/shift,
406
+ // SDXL records checkpoint/loras).
407
+ const provenance = {
408
+ workflow_id: pack.lane,
409
+ workflow_version: "1.0.0",
410
+ base,
411
+ prompt: fullPrompt,
412
+ negative_prompt: pack.negative,
413
+ seed,
414
+ steps: resolved.steps,
415
+ cfg: resolved.cfg,
416
+ sampler: resolved.sampler,
417
+ scheduler: resolved.scheduler,
418
+ width: resolved.width,
419
+ height: resolved.height,
420
+ generation_time_ms: elapsed,
421
+ gpu_model: gpuModel,
422
+ batch_index: generated,
423
+ };
424
+ if (isQwen) {
425
+ provenance.unet = d.unet || QWEN_DEFAULTS.unet;
426
+ provenance.clip = d.clip || QWEN_DEFAULTS.clip;
427
+ provenance.vae = d.vae || QWEN_DEFAULTS.vae;
428
+ provenance.shift = d.shift ?? QWEN_DEFAULTS.shift;
429
+ provenance.loras = d.loras || [];
430
+ } else {
431
+ provenance.checkpoint = d.checkpoint;
432
+ provenance.loras = loras;
320
433
  }
434
+
435
+ const record = {
436
+ id: assetId,
437
+ schema_version: "1.0.0",
438
+ created_at: new Date().toISOString(),
439
+ asset_path: destPath,
440
+ image: { format: "png", width: resolved.width, height: resolved.height, bytes: imgData.length },
441
+ provenance,
442
+ judgment: null,
443
+ canon: null,
444
+ iteration: null,
445
+ };
446
+
447
+ await writeFile(
448
+ join(GAME_ROOT, `records/${assetId}.json`),
449
+ JSON.stringify(record, null, 2),
450
+ );
451
+
452
+ console.log(` \x1b[32m✓\x1b[0m ${destPath} (${elapsed}ms, ${imgData.length} bytes)`);
453
+ } catch (err) {
454
+ console.log(` \x1b[31m✗\x1b[0m ${err.message}`);
455
+ errors++;
456
+ }
457
+
458
+ generated++;
459
+
460
+ // ETA every 5 items (not every item — too noisy for long runs).
461
+ if (!dryRun && generated > 0 && generated % 5 === 0 && generated < totalExpected) {
462
+ const elapsedTotal = Date.now() - runStartMs;
463
+ const avgMs = elapsedTotal / generated;
464
+ const remaining = totalExpected - generated;
465
+ const etaMs = avgMs * remaining;
466
+ info('generate', `progress ${generated}/${totalExpected} — avg ${formatEta(avgMs)}/item, ETA ~${formatEta(etaMs)}`);
321
467
  }
322
468
  }
323
469
 
324
470
  console.log("");
325
- const succeeded = generated - errors;
471
+ const succeeded = generated - errors - skipped;
326
472
  const totalElapsedMs = Date.now() - runStartMs;
327
- console.log(`\x1b[32m✓\x1b[0m Generated ${succeeded} candidates (${errors} errors)`);
328
- if (!dryRun && generated > 0) {
329
- const avgMs = totalElapsedMs / generated;
330
- console.log(` Total: ${formatEta(totalElapsedMs)} — avg ${formatEta(avgMs)}/item`);
473
+ const skippedSuffix = skipped > 0 ? `, ${skipped} resumed` : '';
474
+ console.log(`\x1b[32m✓\x1b[0m Generated ${succeeded} candidates (${errors} errors${skippedSuffix})`);
475
+ if (!dryRun && generated > skipped) {
476
+ const ran = generated - skipped;
477
+ const avgMs = totalElapsedMs / ran;
478
+ console.log(` Total: ${formatEta(totalElapsedMs)} — avg ${formatEta(avgMs)}/item over ${ran} new`);
331
479
  }
332
480
  if (dryRun) console.log(" (dry run — no images produced)");
333
481
  // Always print the output directory so scripted pipelines can parse it,
@@ -0,0 +1,124 @@
1
+ #!/usr/bin/env python3
2
+ """Studio NON-ANIME generation on Qwen-Image (the verified base) via ComfyUI.
3
+
4
+ Reads a canon-derived WAVE json ({style_prefix, defaults, subjects:[{id,prompt,variations}]}) and
5
+ generates each subject on Qwen-Image, saving images + a provenance receipt (PIN_PER_STEP) to an
6
+ output dir. This is the bridge generator until sdlab's buildWorkflowGraph() gains native Qwen
7
+ support (tracked follow-on). It is the studio's NON-ANIME path — see memory/feedback_no_anime.md.
8
+
9
+ All Qwen-Image components are on the rig (diffusion_models/text_encoders/vae). Run with ComfyUI's
10
+ embedded python (has PIL): start the watchdog (E:/AI/training/_watchdog.ps1) before GPU runs.
11
+
12
+ python qwen_generate.py --wave <wave.json> --out <dir> [--variations N]
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import json
19
+ import time
20
+ import urllib.request
21
+ import uuid
22
+ from pathlib import Path
23
+
24
+ from PIL import Image
25
+
26
+ COMFY = "http://127.0.0.1:8188"
27
+ COMFY_OUT = Path(r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/output")
28
+
29
+ # Qwen-Image native components (on rig). NON-ANIME base — never an anime checkpoint.
30
+ QWEN_UNET = "qwen_image_fp8_e4m3fn.safetensors"
31
+ QWEN_CLIP = "qwen_2.5_vl_7b_fp8_scaled.safetensors"
32
+ QWEN_VAE = "qwen_image_vae.safetensors"
33
+
34
+
35
+ def qwen_graph(pos: str, neg: str, seed: int, w: int, h: int, steps: int, cfg: float, prefix: str,
36
+ loras: list | None = None) -> dict:
37
+ """loras: [{name, weight}] chained as DiT-only LoraLoaderModelOnly nodes (ids from
38
+ "40", between UNETLoader and ModelSamplingAuraFlow) — mirrors sdlab buildQwenGraph.
39
+ Model-only because studio Qwen style LoRAs train the transformer with the text
40
+ encoder frozen."""
41
+ nodes = {
42
+ "37": {"class_type": "UNETLoader", "inputs": {"unet_name": QWEN_UNET, "weight_dtype": "default"}},
43
+ "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": QWEN_CLIP, "type": "qwen_image"}},
44
+ "39": {"class_type": "VAELoader", "inputs": {"vae_name": QWEN_VAE}},
45
+ "6": {"class_type": "CLIPTextEncode", "inputs": {"text": pos, "clip": ["38", 0]}},
46
+ "7": {"class_type": "CLIPTextEncode", "inputs": {"text": neg, "clip": ["38", 0]}},
47
+ }
48
+ model_out = ["37", 0]
49
+ for i, lora in enumerate(loras or []):
50
+ lid = str(40 + i)
51
+ nodes[lid] = {"class_type": "LoraLoaderModelOnly", "inputs": {
52
+ "lora_name": lora["name"], "strength_model": lora.get("weight", 1.0), "model": model_out}}
53
+ model_out = [lid, 0]
54
+ nodes.update({
55
+ "66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift": 3.1, "model": model_out}},
56
+ "5": {"class_type": "EmptySD3LatentImage", "inputs": {"width": w, "height": h, "batch_size": 1}},
57
+ "3": {"class_type": "KSampler", "inputs": {"seed": seed, "steps": steps, "cfg": cfg, "sampler_name": "euler",
58
+ "scheduler": "simple", "denoise": 1.0, "model": ["66", 0], "positive": ["6", 0], "negative": ["7", 0], "latent_image": ["5", 0]}},
59
+ "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
60
+ "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": prefix, "images": ["8", 0]}},
61
+ })
62
+ return nodes
63
+
64
+
65
+ def _post(path: str, payload: dict):
66
+ return json.loads(urllib.request.urlopen(urllib.request.Request(
67
+ f"{COMFY}{path}", json.dumps(payload).encode(), {"Content-Type": "application/json"}), timeout=90).read())
68
+
69
+
70
+ def generate(pos: str, neg: str, seed: int, dst: Path, **kw) -> Path:
71
+ g = qwen_graph(pos, neg, seed, kw["w"], kw["h"], kw["steps"], kw["cfg"], dst.stem, loras=kw.get("loras"))
72
+ pid = _post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
73
+ for _ in range(1200):
74
+ hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
75
+ if pid in hist and hist[pid].get("outputs"):
76
+ img = hist[pid]["outputs"]["9"]["images"][0]
77
+ src = COMFY_OUT / img["subfolder"] / img["filename"]
78
+ for _w in range(120):
79
+ if src.exists():
80
+ Image.open(src).convert("RGB").save(dst)
81
+ return dst
82
+ time.sleep(0.5)
83
+ time.sleep(0.5)
84
+ raise RuntimeError("ComfyUI generation timed out")
85
+
86
+
87
+ def main() -> None:
88
+ ap = argparse.ArgumentParser(description=__doc__)
89
+ ap.add_argument("--wave", required=True, type=Path)
90
+ ap.add_argument("--out", required=True, type=Path)
91
+ ap.add_argument("--variations", type=int, default=None, help="override per-subject variation count")
92
+ args = ap.parse_args()
93
+
94
+ wave = json.loads(args.wave.read_text(encoding="utf-8"))
95
+ prefix = wave.get("style_prefix", "")
96
+ d = wave.get("defaults", {})
97
+ neg = d.get("negative", "")
98
+ w, h = d.get("width", 1024), d.get("height", 1024)
99
+ steps, cfg = d.get("steps", 22), d.get("cfg", 3.5)
100
+ base_seed = d.get("base_seed", 1000)
101
+ loras = d.get("loras", [])
102
+ args.out.mkdir(parents=True, exist_ok=True)
103
+
104
+ receipt = {"wave": wave.get("wave"), "base": "qwen-image", "unet": QWEN_UNET, "clip": QWEN_CLIP,
105
+ "vae": QWEN_VAE, "sampler": "euler", "scheduler": "simple", "shift": 3.1,
106
+ "steps": steps, "cfg": cfg, "size": [w, h], "style_prefix": prefix, "negative": neg,
107
+ "loras": loras, "items": []}
108
+ seed = base_seed
109
+ for subj in wave["subjects"]:
110
+ nvar = args.variations if args.variations is not None else subj.get("variations", 1)
111
+ pos = f"{prefix}, {subj['prompt']}" if prefix else subj["prompt"]
112
+ for v in range(nvar):
113
+ dst = args.out / f"{subj['id']}_v{v}.png"
114
+ generate(pos, neg, seed, dst, w=w, h=h, steps=steps, cfg=cfg, loras=loras)
115
+ print(f" {subj['id']} v{v} (seed {seed}) -> {dst.name}", flush=True)
116
+ receipt["items"].append({"id": subj["id"], "variation": v, "seed": seed, "file": dst.name, "prompt": pos})
117
+ seed += 1
118
+
119
+ (args.out / "generation.json").write_text(json.dumps(receipt, indent=2), encoding="utf-8")
120
+ print(f"\n{len(receipt['items'])} images -> {args.out} (+ generation.json provenance)")
121
+
122
+
123
+ if __name__ == "__main__":
124
+ main()