@mcptoolshop/style-dataset-lab 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.es.md +35 -25
- package/README.fr.md +38 -28
- package/README.hi.md +36 -26
- package/README.it.md +37 -27
- package/README.ja.md +40 -30
- package/README.md +15 -5
- package/README.pt-BR.md +35 -25
- package/README.zh.md +33 -23
- package/bin/sdlab.js +51 -4
- package/lib/adapters/ai-toolkit.js +37 -5
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +10 -2
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +13 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +16 -1
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/measure.js +422 -0
- package/lib/paths.js +196 -7
- package/lib/reingest-selected.js +98 -16
- package/lib/run-manifest.js +102 -16
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/workflow-profiles.js +24 -3
- package/package.json +3 -2
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +27 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +181 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +48 -5
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
|
@@ -14,12 +14,13 @@
|
|
|
14
14
|
|
|
15
15
|
import { writeFile, mkdir } from "node:fs/promises";
|
|
16
16
|
import { join } from "node:path";
|
|
17
|
-
import { getProjectName, parseNumberFlag } from "../lib/args.js";
|
|
18
|
-
import {
|
|
17
|
+
import { getProjectName, parseNumberFlag, takeFlagValue } from "../lib/args.js";
|
|
18
|
+
import { getProjectRoot, resolveSafeProjectPath } from "../lib/paths.js";
|
|
19
19
|
import { readJsonFile } from "../lib/config.js";
|
|
20
20
|
import { inputError, runtimeError, handleCliError } from "../lib/errors.js";
|
|
21
21
|
import { comfyHealth, submitAndWait, downloadImage, uploadImage } from "../lib/comfyui.js";
|
|
22
22
|
import { assertNotFrozenBySubject } from "../lib/freeze-gate.js";
|
|
23
|
+
import { isResumable, logResumedSkip } from "./_resume.js";
|
|
23
24
|
|
|
24
25
|
const DEFAULTS = {
|
|
25
26
|
checkpoint: "dreamshaperXL_v21TurboDPMSDE.safetensors",
|
|
@@ -33,27 +34,32 @@ const DEFAULTS = {
|
|
|
33
34
|
base_seed: 27100,
|
|
34
35
|
};
|
|
35
36
|
|
|
37
|
+
// H4: takeFlagValue() (lib/args.js) recognizes BOTH `--flag value` and
|
|
38
|
+
// `--flag=value`. The prior `argv.indexOf(flag)`-based get() only matched a
|
|
39
|
+
// token EXACTLY equal to e.g. "--weight" — false for the single token
|
|
40
|
+
// "--weight=1.1" — so the equals form silently fell back to the default
|
|
41
|
+
// with no error and no warning.
|
|
36
42
|
function parseLocalArgs(argv) {
|
|
37
|
-
const get = (
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
const v = argv[i + 1];
|
|
41
|
-
if (typeof v === 'string' && v.startsWith('--')) {
|
|
42
|
-
throw inputError('INPUT_MISSING_VALUE', `Flag ${flag} is missing its value (got another flag: ${v}).`);
|
|
43
|
-
}
|
|
44
|
-
return v;
|
|
43
|
+
const get = (flagName, def) => {
|
|
44
|
+
const v = takeFlagValue(argv, flagName);
|
|
45
|
+
return v === undefined ? def : v;
|
|
45
46
|
};
|
|
46
47
|
return {
|
|
47
|
-
subject: get("
|
|
48
|
-
guidePath: get("
|
|
49
|
-
prompt: get("
|
|
50
|
-
promptFile: get("
|
|
51
|
-
negative: get("
|
|
52
|
-
seeds: parseNumberFlag('seeds', get("
|
|
53
|
-
weight: parseNumberFlag('weight', get("
|
|
54
|
-
guidanceEnd: parseNumberFlag('guidance-end', get("
|
|
55
|
-
controlModel: get("
|
|
48
|
+
subject: get("subject", "unknown"),
|
|
49
|
+
guidePath: get("guide", null),
|
|
50
|
+
prompt: get("prompt", null),
|
|
51
|
+
promptFile: get("prompt-file", null),
|
|
52
|
+
negative: get("negative", null),
|
|
53
|
+
seeds: parseNumberFlag('seeds', get("seeds", "4"), { int: true, min: 1 }),
|
|
54
|
+
weight: parseNumberFlag('weight', get("weight", "0.70"), { min: 0, max: 2 }),
|
|
55
|
+
guidanceEnd: parseNumberFlag('guidance-end', get("guidance-end", "0.65"), { min: 0, max: 1 }),
|
|
56
|
+
controlModel: get("model", "controlnet-canny-sdxl-1.0.safetensors"),
|
|
56
57
|
dryRun: argv.includes("--dry-run"),
|
|
58
|
+
// H5: skip a seed slot whose output image already exists. This script
|
|
59
|
+
// (unlike generate.js / generate-identity.js) never writes a
|
|
60
|
+
// records/<id>.json — only the output PNG — so resumability here is
|
|
61
|
+
// image-existence only; see scripts/_resume.js's requireRecord doc.
|
|
62
|
+
resume: argv.includes("--resume"),
|
|
57
63
|
};
|
|
58
64
|
}
|
|
59
65
|
|
|
@@ -169,7 +175,7 @@ function buildControlNetWorkflow(prompt, negativePrompt, seed, guideImagePath, c
|
|
|
169
175
|
|
|
170
176
|
export async function run(argv = process.argv.slice(2)) {
|
|
171
177
|
const projectName = getProjectName(argv);
|
|
172
|
-
const GAME_ROOT =
|
|
178
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
173
179
|
const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
|
|
174
180
|
|
|
175
181
|
const opts = parseLocalArgs(argv);
|
|
@@ -180,8 +186,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
180
186
|
|
|
181
187
|
// Freeze gate (advisory): refuse if the subject's canon entry is frozen.
|
|
182
188
|
// Supports --i-know + --reason bypass for soft-advisory entries.
|
|
183
|
-
const
|
|
184
|
-
const bypassReason = reasonFlagIdx >= 0 && argv[reasonFlagIdx + 1] ? argv[reasonFlagIdx + 1] : null;
|
|
189
|
+
const bypassReason = takeFlagValue(argv, 'reason') ?? null;
|
|
185
190
|
await assertNotFrozenBySubject(GAME_ROOT, opts.subject, {
|
|
186
191
|
action: 'generate:controlnet',
|
|
187
192
|
allowSoftAdvisoryBypass: argv.includes('--i-know'),
|
|
@@ -236,14 +241,28 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
236
241
|
console.log(`\x1b[32m+\x1b[0m Guide uploaded: ${guideFilename}`);
|
|
237
242
|
}
|
|
238
243
|
|
|
239
|
-
|
|
244
|
+
// SDL-M11: only create the output dir on a real run — a --dry-run preview
|
|
245
|
+
// must be side-effect-free (README.md promises "preview the effect first").
|
|
246
|
+
if (!opts.dryRun) {
|
|
247
|
+
await mkdir(join(GAME_ROOT, "outputs/candidates"), { recursive: true });
|
|
248
|
+
}
|
|
240
249
|
|
|
241
|
-
let successes = 0, failures = 0;
|
|
250
|
+
let successes = 0, failures = 0, skipped = 0;
|
|
242
251
|
for (let si = 0; si < opts.seeds; si++) {
|
|
243
252
|
const seed = DEFAULTS.base_seed + si;
|
|
244
253
|
const assetId = `${opts.subject}_cn_s${si}`;
|
|
245
254
|
const destPath = `outputs/candidates/${assetId}.png`;
|
|
246
255
|
|
|
256
|
+
// H5: --resume skips a seed slot whose output image already exists.
|
|
257
|
+
// Seed here is derived from the loop index (si), not an externally
|
|
258
|
+
// mutated running counter, so skipping never desyncs a later seed — the
|
|
259
|
+
// for-loop's own si++ always fires regardless of this continue.
|
|
260
|
+
if (opts.resume && !opts.dryRun && isResumable(GAME_ROOT, assetId, { requireRecord: false })) {
|
|
261
|
+
logResumedSkip(si + 1, opts.seeds, assetId);
|
|
262
|
+
skipped++;
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
|
|
247
266
|
console.log(` [${si + 1}/${opts.seeds}] ${assetId} (seed: ${seed})`);
|
|
248
267
|
|
|
249
268
|
if (opts.dryRun) continue;
|
|
@@ -285,7 +304,8 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
285
304
|
}
|
|
286
305
|
}
|
|
287
306
|
|
|
288
|
-
|
|
307
|
+
const skippedSuffix = skipped > 0 ? `, ${skipped} resumed` : '';
|
|
308
|
+
console.log(`\n\x1b[32m+\x1b[0m ControlNet discovery complete (${successes} success, ${failures} failed${skippedSuffix})`);
|
|
289
309
|
if (opts.dryRun) console.log(" (dry run)");
|
|
290
310
|
if (!opts.dryRun && failures > 0 && successes === 0) {
|
|
291
311
|
throw runtimeError('RUNTIME_ALL_FAILED', `All ${opts.seeds} ControlNet generation attempts failed.`);
|
|
@@ -18,16 +18,20 @@
|
|
|
18
18
|
* --phase MODE Generation phase: discovery | follow_on (default: discovery)
|
|
19
19
|
* --anchor FILE Anchor source image path (required for --phase follow_on)
|
|
20
20
|
* --denoise N Denoise strength for follow_on phase (default: 0.38)
|
|
21
|
+
* --resume Skip shots whose record + output image already exist
|
|
22
|
+
* (seed slots still advance, so a resumed run stays
|
|
23
|
+
* bit-identical to an uninterrupted one)
|
|
21
24
|
*/
|
|
22
25
|
|
|
23
26
|
import { writeFile, mkdir } from "node:fs/promises";
|
|
24
27
|
import { join } from "node:path";
|
|
25
|
-
import { getProjectName, parseNumberFlag } from "../lib/args.js";
|
|
26
|
-
import {
|
|
28
|
+
import { getProjectName, parseNumberFlag, takeFlagValue } from "../lib/args.js";
|
|
29
|
+
import { getProjectRoot, resolveSafeProjectPath } from "../lib/paths.js";
|
|
27
30
|
import { readJsonFile, loadProjectMeta, resolveGpuModel } from "../lib/config.js";
|
|
28
31
|
import { inputError, runtimeError, handleCliError } from "../lib/errors.js";
|
|
29
32
|
import { comfyHealth, submitAndWait, downloadImage } from "../lib/comfyui.js";
|
|
30
33
|
import { assertNotFrozenBySubject } from "../lib/freeze-gate.js";
|
|
34
|
+
import { isResumable, logResumedSkip } from "./_resume.js";
|
|
31
35
|
|
|
32
36
|
// ── Defaults (match existing lab pipeline) ──
|
|
33
37
|
|
|
@@ -378,12 +382,12 @@ function buildRecord(pack, subject, shot, seed, seedIndex, imgData, destPath, el
|
|
|
378
382
|
|
|
379
383
|
export async function run(argv = process.argv.slice(2)) {
|
|
380
384
|
const projectName = getProjectName(argv);
|
|
381
|
-
const GAME_ROOT =
|
|
385
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
382
386
|
const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
|
|
383
387
|
const gpuModel = resolveGpuModel(loadProjectMeta(GAME_ROOT)?.defaults);
|
|
384
388
|
|
|
385
389
|
// Collect positionals, skipping values that belong to known flags
|
|
386
|
-
const knownFlagsWithValue = new Set(['--subject', '--seeds', '--phase', '--anchor', '--denoise', '--project', '--game']);
|
|
390
|
+
const knownFlagsWithValue = new Set(['--subject', '--seeds', '--phase', '--anchor', '--denoise', '--project', '--game', '--reason']);
|
|
387
391
|
const positionals = [];
|
|
388
392
|
for (let k = 0; k < argv.length; k++) {
|
|
389
393
|
const a = argv[k];
|
|
@@ -395,16 +399,23 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
395
399
|
}
|
|
396
400
|
const packPath = positionals[0];
|
|
397
401
|
const dryRun = argv.includes("--dry-run");
|
|
398
|
-
const
|
|
399
|
-
|
|
400
|
-
|
|
402
|
+
const resume = argv.includes("--resume");
|
|
403
|
+
|
|
404
|
+
// H4: takeFlagValue() (lib/args.js) recognizes BOTH `--flag value` and
|
|
405
|
+
// `--flag=value`. The previous `argv.includes('--x') ?
|
|
406
|
+
// argv[argv.indexOf('--x')+1] : default` pattern checked argv for a token
|
|
407
|
+
// EXACTLY EQUAL to "--x" — false for the single token "--phase=follow_on",
|
|
408
|
+
// so the equals form silently fell back to the default. For --phase that
|
|
409
|
+
// meant `--phase=follow_on` ran the discovery (txt2img) workflow instead
|
|
410
|
+
// of follow_on (img2img) — silently, on an hours-long GPU job. Same gap
|
|
411
|
+
// closed here for --seeds / --anchor / --denoise / --subject / --reason.
|
|
412
|
+
const subjectFilter = takeFlagValue(argv, 'subject') ?? null;
|
|
401
413
|
|
|
402
414
|
// Freeze gate (advisory): when --subject is given, refuse if that subject's
|
|
403
415
|
// canon entry is frozen. No-op when --subject is absent (pack-driven) or
|
|
404
416
|
// when the project has no canon-build config.
|
|
405
417
|
if (subjectFilter) {
|
|
406
|
-
const
|
|
407
|
-
const bypassReason = reasonFlagIdx >= 0 && argv[reasonFlagIdx + 1] ? argv[reasonFlagIdx + 1] : null;
|
|
418
|
+
const bypassReason = takeFlagValue(argv, 'reason') ?? null;
|
|
408
419
|
await assertNotFrozenBySubject(GAME_ROOT, subjectFilter, {
|
|
409
420
|
action: 'generate:identity',
|
|
410
421
|
allowSoftAdvisoryBypass: argv.includes('--i-know'),
|
|
@@ -412,17 +423,15 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
412
423
|
by: 'mike',
|
|
413
424
|
});
|
|
414
425
|
}
|
|
415
|
-
const
|
|
416
|
-
|
|
426
|
+
const seedsRaw = takeFlagValue(argv, 'seeds');
|
|
427
|
+
const seedCount = seedsRaw !== undefined
|
|
428
|
+
? parseNumberFlag('seeds', seedsRaw, { int: true, min: 1 })
|
|
417
429
|
: 3;
|
|
418
|
-
const phase = argv
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
const
|
|
422
|
-
?
|
|
423
|
-
: null;
|
|
424
|
-
const denoise = argv.includes("--denoise")
|
|
425
|
-
? parseNumberFlag('denoise', argv[argv.indexOf("--denoise") + 1], { min: 0, max: 1 })
|
|
430
|
+
const phase = takeFlagValue(argv, 'phase') ?? "discovery";
|
|
431
|
+
const anchorPath = takeFlagValue(argv, 'anchor') ?? null;
|
|
432
|
+
const denoiseRaw = takeFlagValue(argv, 'denoise');
|
|
433
|
+
const denoise = denoiseRaw !== undefined
|
|
434
|
+
? parseNumberFlag('denoise', denoiseRaw, { min: 0, max: 1 })
|
|
426
435
|
: 0.38;
|
|
427
436
|
|
|
428
437
|
if (!packPath) {
|
|
@@ -474,11 +483,16 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
474
483
|
console.log("\x1b[32m+\x1b[0m ComfyUI online");
|
|
475
484
|
}
|
|
476
485
|
|
|
477
|
-
|
|
478
|
-
|
|
486
|
+
// SDL-M11: only create output dirs on a real run — a --dry-run preview must
|
|
487
|
+
// be side-effect-free (README.md promises "preview the effect first").
|
|
488
|
+
if (!dryRun) {
|
|
489
|
+
await mkdir(join(GAME_ROOT, "outputs/candidates"), { recursive: true });
|
|
490
|
+
await mkdir(join(GAME_ROOT, "records"), { recursive: true });
|
|
491
|
+
}
|
|
479
492
|
|
|
480
493
|
let generated = 0;
|
|
481
494
|
let errors = 0;
|
|
495
|
+
let skipped = 0;
|
|
482
496
|
let imageIndex = 0;
|
|
483
497
|
|
|
484
498
|
for (const subject of subjects) {
|
|
@@ -498,6 +512,21 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
498
512
|
const assetId = `${shot.id}_s${si}`;
|
|
499
513
|
const destPath = `outputs/candidates/${assetId}.png`;
|
|
500
514
|
|
|
515
|
+
// H5: --resume skips a shot whose record AND output image already
|
|
516
|
+
// exist, but imageIndex (and therefore seed) still advances exactly
|
|
517
|
+
// as it would have if the shot had been processed — a resumed run
|
|
518
|
+
// reproduces the identical seed sequence an uninterrupted run would
|
|
519
|
+
// have produced. Ported from generate.js, the one generator that had
|
|
520
|
+
// this; generate-identity is the most expensive per-image generator
|
|
521
|
+
// in the pipeline, so it is the one that most needed it.
|
|
522
|
+
if (resume && !dryRun && isResumable(GAME_ROOT, assetId)) {
|
|
523
|
+
logResumedSkip(generated + 1, totalImages, assetId);
|
|
524
|
+
generated++;
|
|
525
|
+
skipped++;
|
|
526
|
+
imageIndex++;
|
|
527
|
+
continue;
|
|
528
|
+
}
|
|
529
|
+
|
|
501
530
|
console.log(` [${generated + 1}/${totalImages}] ${assetId} (seed: ${seed})`);
|
|
502
531
|
|
|
503
532
|
if (dryRun) {
|
|
@@ -572,8 +601,9 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
572
601
|
}
|
|
573
602
|
}
|
|
574
603
|
|
|
575
|
-
const succeeded = generated - errors;
|
|
576
|
-
|
|
604
|
+
const succeeded = generated - errors - skipped;
|
|
605
|
+
const skippedSuffix = skipped > 0 ? `, ${skipped} resumed` : '';
|
|
606
|
+
console.log(`\n\x1b[32m+\x1b[0m Generated ${succeeded} images (${errors} errors${skippedSuffix})`);
|
|
577
607
|
if (dryRun) console.log(" (dry run — no images produced)");
|
|
578
608
|
if (!dryRun && errors > 0 && succeeded === 0) {
|
|
579
609
|
throw runtimeError('RUNTIME_ALL_FAILED', `All ${generated} generation attempts failed.`);
|
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
|
|
14
14
|
import { writeFile, mkdir } from "node:fs/promises";
|
|
15
15
|
import { join } from "node:path";
|
|
16
|
-
import { getProjectName, parseNumberFlag } from "../lib/args.js";
|
|
17
|
-
import {
|
|
16
|
+
import { getProjectName, parseNumberFlag, takeFlagValue } from "../lib/args.js";
|
|
17
|
+
import { getProjectRoot, resolveSafeProjectPath } from "../lib/paths.js";
|
|
18
18
|
import { inputError, runtimeError, handleCliError } from "../lib/errors.js";
|
|
19
19
|
import { comfyHealth, submitAndWait, downloadImage, uploadImage } from "../lib/comfyui.js";
|
|
20
20
|
import { assertNotFrozenBySubject } from "../lib/freeze-gate.js";
|
|
21
|
+
import { isResumable, logResumedSkip } from "./_resume.js";
|
|
21
22
|
|
|
22
23
|
const DEFAULTS = {
|
|
23
24
|
checkpoint: "dreamshaperXL_v21TurboDPMSDE.safetensors",
|
|
@@ -33,26 +34,31 @@ const DEFAULTS = {
|
|
|
33
34
|
base_seed: 27200,
|
|
34
35
|
};
|
|
35
36
|
|
|
37
|
+
// H4: takeFlagValue() (lib/args.js) recognizes BOTH `--flag value` and
|
|
38
|
+
// `--flag=value`. The prior `argv.indexOf(flag)`-based get() only matched a
|
|
39
|
+
// token EXACTLY equal to e.g. "--weight" — false for the single token
|
|
40
|
+
// "--weight=1.1" — so the equals form silently fell back to the default
|
|
41
|
+
// with no error and no warning.
|
|
36
42
|
function parseLocalArgs(argv) {
|
|
37
|
-
const get = (
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
const v = argv[i + 1];
|
|
41
|
-
if (typeof v === 'string' && v.startsWith('--')) {
|
|
42
|
-
throw inputError('INPUT_MISSING_VALUE', `Flag ${flag} is missing its value (got another flag: ${v}).`);
|
|
43
|
-
}
|
|
44
|
-
return v;
|
|
43
|
+
const get = (flagName, def) => {
|
|
44
|
+
const v = takeFlagValue(argv, flagName);
|
|
45
|
+
return v === undefined ? def : v;
|
|
45
46
|
};
|
|
46
47
|
return {
|
|
47
|
-
subject: get("
|
|
48
|
-
refPath: get("
|
|
49
|
-
prompt: get("
|
|
50
|
-
negative: get("
|
|
51
|
-
seeds: parseNumberFlag('seeds', get("
|
|
52
|
-
weight: parseNumberFlag('weight', get("
|
|
53
|
-
start: parseNumberFlag('start', get("
|
|
54
|
-
end: parseNumberFlag('end', get("
|
|
48
|
+
subject: get("subject", "unknown"),
|
|
49
|
+
refPath: get("ref", null),
|
|
50
|
+
prompt: get("prompt", null),
|
|
51
|
+
negative: get("negative", null),
|
|
52
|
+
seeds: parseNumberFlag('seeds', get("seeds", "4"), { int: true, min: 1 }),
|
|
53
|
+
weight: parseNumberFlag('weight', get("weight", "0.55"), { min: 0, max: 2 }),
|
|
54
|
+
start: parseNumberFlag('start', get("start", "0.0"), { min: 0, max: 1 }),
|
|
55
|
+
end: parseNumberFlag('end', get("end", "0.8"), { min: 0, max: 1 }),
|
|
55
56
|
dryRun: argv.includes("--dry-run"),
|
|
57
|
+
// H5: skip a seed slot whose output image already exists. This script
|
|
58
|
+
// (unlike generate.js / generate-identity.js) never writes a
|
|
59
|
+
// records/<id>.json — only the output PNG — so resumability here is
|
|
60
|
+
// image-existence only; see scripts/_resume.js's requireRecord doc.
|
|
61
|
+
resume: argv.includes("--resume"),
|
|
56
62
|
};
|
|
57
63
|
}
|
|
58
64
|
|
|
@@ -175,7 +181,7 @@ function buildIPAdapterWorkflow(prompt, negativePrompt, seed, refImageName, weig
|
|
|
175
181
|
|
|
176
182
|
export async function run(argv = process.argv.slice(2)) {
|
|
177
183
|
const projectName = getProjectName(argv);
|
|
178
|
-
const GAME_ROOT =
|
|
184
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
179
185
|
const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
|
|
180
186
|
|
|
181
187
|
const opts = parseLocalArgs(argv);
|
|
@@ -188,8 +194,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
188
194
|
}
|
|
189
195
|
|
|
190
196
|
// Freeze gate (advisory): refuse if the subject's canon entry is frozen.
|
|
191
|
-
const
|
|
192
|
-
const bypassReason = reasonFlagIdx >= 0 && argv[reasonFlagIdx + 1] ? argv[reasonFlagIdx + 1] : null;
|
|
197
|
+
const bypassReason = takeFlagValue(argv, 'reason') ?? null;
|
|
193
198
|
await assertNotFrozenBySubject(GAME_ROOT, opts.subject, {
|
|
194
199
|
action: 'generate:ipadapter',
|
|
195
200
|
allowSoftAdvisoryBypass: argv.includes('--i-know'),
|
|
@@ -224,13 +229,28 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
224
229
|
console.log(`\x1b[32m+\x1b[0m Reference uploaded: ${refFilename}`);
|
|
225
230
|
}
|
|
226
231
|
|
|
227
|
-
|
|
232
|
+
// SDL-M11: only create the output dir on a real run — a --dry-run preview
|
|
233
|
+
// must be side-effect-free (README.md promises "preview the effect first").
|
|
234
|
+
if (!opts.dryRun) {
|
|
235
|
+
await mkdir(join(GAME_ROOT, "outputs/candidates"), { recursive: true });
|
|
236
|
+
}
|
|
228
237
|
|
|
238
|
+
let skipped = 0;
|
|
229
239
|
for (let si = 0; si < opts.seeds; si++) {
|
|
230
240
|
const seed = DEFAULTS.base_seed + si;
|
|
231
241
|
const assetId = `${opts.subject}_ipa_s${si}`;
|
|
232
242
|
const destPath = `outputs/candidates/${assetId}.png`;
|
|
233
243
|
|
|
244
|
+
// H5: --resume skips a seed slot whose output image already exists.
|
|
245
|
+
// Seed here is derived from the loop index (si), not an externally
|
|
246
|
+
// mutated running counter, so skipping never desyncs a later seed — the
|
|
247
|
+
// for-loop's own si++ always fires regardless of this continue.
|
|
248
|
+
if (opts.resume && !opts.dryRun && isResumable(GAME_ROOT, assetId, { requireRecord: false })) {
|
|
249
|
+
logResumedSkip(si + 1, opts.seeds, assetId);
|
|
250
|
+
skipped++;
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
253
|
+
|
|
234
254
|
console.log(` [${si + 1}/${opts.seeds}] ${assetId} (seed: ${seed})`);
|
|
235
255
|
|
|
236
256
|
if (opts.dryRun) continue;
|
|
@@ -269,7 +289,8 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
269
289
|
}
|
|
270
290
|
}
|
|
271
291
|
|
|
272
|
-
|
|
292
|
+
const skippedSuffix = skipped > 0 ? ` (${skipped} resumed)` : '';
|
|
293
|
+
console.log(`\n\x1b[32m+\x1b[0m IP-Adapter discovery complete${skippedSuffix}`);
|
|
273
294
|
if (opts.dryRun) console.log(" (dry run)");
|
|
274
295
|
}
|
|
275
296
|
|
package/scripts/generate.js
CHANGED
|
@@ -14,14 +14,14 @@ import { writeFile, mkdir } from "node:fs/promises";
|
|
|
14
14
|
import { existsSync } from "node:fs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { parseArgs, getProjectName } from "../lib/args.js";
|
|
17
|
-
import {
|
|
17
|
+
import { getProjectRoot, resolveSafeProjectPath } from "../lib/paths.js";
|
|
18
18
|
import { readJsonFile, loadProjectMeta, resolveGpuModel } from "../lib/config.js";
|
|
19
19
|
import { runtimeError, inputError, handleCliError } from "../lib/errors.js";
|
|
20
20
|
import { comfyHealth, submitAndWait, downloadImage } from "../lib/comfyui.js";
|
|
21
21
|
import { pickOutputImage } from "../lib/comfyui-output.js";
|
|
22
22
|
import { buildQwenGraph, QWEN_DEFAULTS } from "../lib/adapters/comfyui-workflows.js";
|
|
23
23
|
import { buildPinning, SEED_POLICIES, createHashCache, flushHashCache } from "../lib/run-manifest.js";
|
|
24
|
-
import { info, result } from "../lib/log.js";
|
|
24
|
+
import { info, result, warn } from "../lib/log.js";
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
27
|
* Format milliseconds as a human ETA string like "6m 12s" or "42s".
|
|
@@ -179,6 +179,34 @@ function buildWorkflow(prompt, negativePrompt, checkpoint, loras, seed, steps, c
|
|
|
179
179
|
return { nodes, saveNodeId: saveId };
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
+
/**
|
|
183
|
+
* Coordinator addendum (same session as H4/H5): every domain template's
|
|
184
|
+
* example-wave.json ships subject prompts with a literal `[DESCRIBE ...]`
|
|
185
|
+
* placeholder (e.g. "[DESCRIBE YOUR HERO — head and shoulders, ...]") that
|
|
186
|
+
* the user is meant to replace before generating. `sdlab init`'s own
|
|
187
|
+
* printed next-steps run generate before "edit the pack" is ever a
|
|
188
|
+
* mandatory step, so following them literally on an unedited pack burns a
|
|
189
|
+
* GPU wave on template text with no feedback until the images come back
|
|
190
|
+
* looking like a literal instruction string.
|
|
191
|
+
*
|
|
192
|
+
* All 5 domain templates use this exact "[DESCRIBE " shape (verified across
|
|
193
|
+
* templates/domains/*\/inputs/prompts/example-wave.json), so the check is
|
|
194
|
+
* narrow and specific to it — it does NOT flag ordinary SD prompt-editing
|
|
195
|
+
* bracket syntax such as "[word]" or "[from:to:0.5]", neither of which
|
|
196
|
+
* starts with the literal word DESCRIBE.
|
|
197
|
+
*/
|
|
198
|
+
const PLACEHOLDER_PROMPT_RE = /\[DESCRIBE\b/;
|
|
199
|
+
|
|
200
|
+
function findPlaceholderPrompts(jobs) {
|
|
201
|
+
const hits = [];
|
|
202
|
+
for (const job of jobs) {
|
|
203
|
+
if (typeof job.prompt === 'string' && PLACEHOLDER_PROMPT_RE.test(job.prompt)) {
|
|
204
|
+
hits.push(job.assetId);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return hits;
|
|
208
|
+
}
|
|
209
|
+
|
|
182
210
|
/**
|
|
183
211
|
* Detect the input schema and normalize to a unified shape:
|
|
184
212
|
* { lane, style_prefix, negative, defaults, base, jobs:[{assetId, subjectId, variationId, prompt, seed, loras}] }
|
|
@@ -264,7 +292,7 @@ function normalizeGenerationInput(raw, projectMeta) {
|
|
|
264
292
|
|
|
265
293
|
export async function run(argv = process.argv.slice(2)) {
|
|
266
294
|
const projectName = getProjectName(argv);
|
|
267
|
-
const GAME_ROOT =
|
|
295
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
268
296
|
const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
|
|
269
297
|
|
|
270
298
|
const parsed = parseArgs(argv, {
|
|
@@ -323,6 +351,17 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
323
351
|
console.log(` Total candidates: ${pack.jobs.length}`);
|
|
324
352
|
console.log("");
|
|
325
353
|
|
|
354
|
+
// Coordinator addendum: catch unedited template text before it burns GPU
|
|
355
|
+
// time (or, under --dry-run, before the user thinks the pack is ready).
|
|
356
|
+
const placeholderHits = findPlaceholderPrompts(pack.jobs);
|
|
357
|
+
if (placeholderHits.length > 0) {
|
|
358
|
+
warn(
|
|
359
|
+
`${placeholderHits.length} of ${pack.jobs.length} prompt(s) still contain unedited "[DESCRIBE ...]" template ` +
|
|
360
|
+
`text (e.g. "${placeholderHits[0]}"). Edit the prompt pack before generating for real — otherwise you'll spend ` +
|
|
361
|
+
`GPU time rendering placeholder instructions instead of your subject.`
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
|
|
326
365
|
if (!dryRun) {
|
|
327
366
|
const online = await comfyHealth(COMFY_URL);
|
|
328
367
|
if (!online) {
|
|
@@ -331,8 +370,12 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
331
370
|
console.log("\x1b[32m✓\x1b[0m ComfyUI online");
|
|
332
371
|
}
|
|
333
372
|
|
|
334
|
-
|
|
335
|
-
|
|
373
|
+
// SDL-M11: only create output dirs on a real run — a --dry-run preview must
|
|
374
|
+
// be side-effect-free (README.md promises "preview the effect first").
|
|
375
|
+
if (!dryRun) {
|
|
376
|
+
await mkdir(join(GAME_ROOT, "outputs/candidates"), { recursive: true });
|
|
377
|
+
await mkdir(join(GAME_ROOT, "records"), { recursive: true });
|
|
378
|
+
}
|
|
336
379
|
|
|
337
380
|
let generated = 0;
|
|
338
381
|
let errors = 0;
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { join } from 'node:path';
|
|
13
13
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
14
|
-
import {
|
|
14
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
15
15
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
16
16
|
import { listTrainingManifests } from '../lib/training-manifests.js';
|
|
17
17
|
import { buildImplementationPack, listImplementationPacks, loadImplementationPack } from '../lib/implementation-packs.js';
|
|
@@ -26,7 +26,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
26
26
|
});
|
|
27
27
|
|
|
28
28
|
const projectName = flags.project || getProjectName(argv);
|
|
29
|
-
const projectRoot =
|
|
29
|
+
const projectRoot = getProjectRoot(projectName);
|
|
30
30
|
const subcommand = positionals.find(a => !a.startsWith('impl-') && !a.startsWith('tm-')) || 'list';
|
|
31
31
|
|
|
32
32
|
if (subcommand === 'build') {
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* ingest.js — Bring a directory of pre-existing images into the pipeline
|
|
5
|
+
* as bare, uncurated candidate records.
|
|
6
|
+
*
|
|
7
|
+
* Closes the gap where generation happened OUTSIDE sdlab entirely (a hand
|
|
8
|
+
* batch, external Python against Comfy Cloud, salvage from somewhere else)
|
|
9
|
+
* and there was no CLI path to get those images into records/ for review.
|
|
10
|
+
* Unlike `sdlab reingest generated` (which requires a trained LoRA's
|
|
11
|
+
* training manifest) or `sdlab reingest selected` (which requires a
|
|
12
|
+
* selection id only the production loop can mint), this takes any folder
|
|
13
|
+
* of images and any project — no pipeline history required.
|
|
14
|
+
*
|
|
15
|
+
* Usage:
|
|
16
|
+
* sdlab ingest <dir> --project <name>
|
|
17
|
+
* sdlab ingest <dir> --project <name> --wave inputs/prompts/wave-v2.json
|
|
18
|
+
* sdlab ingest <dir> --project <name> --dry-run
|
|
19
|
+
* sdlab ingest <dir> --project <name> --json
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
23
|
+
import { REPO_ROOT, getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
|
|
24
|
+
import { inputError, handleCliError } from '../lib/errors.js';
|
|
25
|
+
import { ingestDirectory } from '../lib/ingest.js';
|
|
26
|
+
import { result } from '../lib/log.js';
|
|
27
|
+
|
|
28
|
+
const USAGE = `Usage: sdlab ingest <dir> --project <name> [--wave <wave.json>] [--dry-run] [--json]
|
|
29
|
+
|
|
30
|
+
Scan <dir> for images and create bare candidate records (judgment: null,
|
|
31
|
+
canon: null, provenance.source: "external") ready for "sdlab curate".
|
|
32
|
+
|
|
33
|
+
Positional:
|
|
34
|
+
dir Directory of images to ingest (png/jpg/jpeg/webp)
|
|
35
|
+
|
|
36
|
+
Flags:
|
|
37
|
+
--project <name> Project to operate on
|
|
38
|
+
--wave <path> Optional wave JSON (inputs/prompts/*.json shape) to
|
|
39
|
+
attach real generation provenance (seed, prompt, ...)
|
|
40
|
+
by matching each image's filename stem to an item id.
|
|
41
|
+
A sibling receipts.jsonl next to <dir> is picked up
|
|
42
|
+
automatically when present (prompt_id, seconds, status).
|
|
43
|
+
--dry-run Preview what would be created — no files written
|
|
44
|
+
--json Emit the result as JSON on stdout
|
|
45
|
+
|
|
46
|
+
Next steps after ingesting:
|
|
47
|
+
1. Review: sdlab curate <id> approved/rejected "explanation"
|
|
48
|
+
2. Bind: sdlab bind --project <name>`;
|
|
49
|
+
|
|
50
|
+
export async function run(argv = process.argv.slice(2)) {
|
|
51
|
+
if (argv.includes('--help') || argv.includes('-h')) {
|
|
52
|
+
console.log(USAGE);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const { flags, positionals } = parseArgs(argv, {
|
|
57
|
+
flags: {
|
|
58
|
+
project: { type: 'string' },
|
|
59
|
+
wave: { type: 'string' },
|
|
60
|
+
'dry-run': { type: 'boolean', default: false },
|
|
61
|
+
json: { type: 'boolean', default: false },
|
|
62
|
+
},
|
|
63
|
+
deprecated: { game: 'project' },
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const dirArg = positionals[0];
|
|
67
|
+
if (!dirArg) {
|
|
68
|
+
throw inputError(
|
|
69
|
+
'INPUT_MISSING_ARGS',
|
|
70
|
+
'A source directory is required.',
|
|
71
|
+
USAGE
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const projectName = flags.project || getProjectName(argv);
|
|
76
|
+
const projectRoot = getProjectRoot(projectName);
|
|
77
|
+
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
78
|
+
|
|
79
|
+
// Mirrors scripts/reingest.js's --source handling: the source directory
|
|
80
|
+
// of pre-existing images is allowed to live anywhere under the workspace
|
|
81
|
+
// root, not just inside this specific project (e.g. a shared staging
|
|
82
|
+
// inbox), while still refusing to escape the workspace entirely.
|
|
83
|
+
const safeSourceDir = resolveSafeProjectPath(projectRoot, dirArg, { baseRoot: REPO_ROOT, flagName: 'dir' });
|
|
84
|
+
|
|
85
|
+
// Mirrors scripts/generate.js's pack-path handling: a --wave file is
|
|
86
|
+
// expected to live inside THIS project (inputs/prompts/*.json), so its
|
|
87
|
+
// containment root defaults to the project root, not the whole workspace.
|
|
88
|
+
const safeWavePath = flags.wave
|
|
89
|
+
? resolveSafeProjectPath(projectRoot, flags.wave, { flagName: 'wave' })
|
|
90
|
+
: null;
|
|
91
|
+
|
|
92
|
+
if (!flags.json) {
|
|
93
|
+
console.log(`\x1b[1msdlab ingest\x1b[0m — ${projectName}`);
|
|
94
|
+
console.log(` Source: ${dirArg}`);
|
|
95
|
+
if (flags.wave) console.log(` Wave: ${flags.wave}`);
|
|
96
|
+
if (dryRun) console.log(` Mode: DRY RUN (no files written)`);
|
|
97
|
+
console.log('');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const { created, skipped, rejected, receiptsFile } = await ingestDirectory(projectRoot, safeSourceDir, {
|
|
101
|
+
wavePath: safeWavePath,
|
|
102
|
+
dryRun,
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
if (flags.json) {
|
|
106
|
+
console.log(JSON.stringify({ created, skipped, rejected, receiptsFile, dryRun }, null, 2));
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (receiptsFile) {
|
|
111
|
+
console.log(` \x1b[2mreceipts: ${receiptsFile}\x1b[0m`);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
console.log(` \x1b[32m✓\x1b[0m Created: ${created.length} record(s)${dryRun ? ' (dry run — nothing written)' : ''}`);
|
|
115
|
+
if (created.length > 0 && created.length <= 25) {
|
|
116
|
+
for (const id of created) console.log(` ✓ ${id}`);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (skipped.length > 0) {
|
|
120
|
+
console.log(` Skipped: ${skipped.length} (record already exists)`);
|
|
121
|
+
if (skipped.length <= 25) {
|
|
122
|
+
for (const id of skipped) console.log(` ○ ${id}`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (rejected.length > 0) {
|
|
127
|
+
console.log(` \x1b[33m⚠\x1b[0m Rejected: ${rejected.length} (unsafe id derived from filename — rename and re-run)`);
|
|
128
|
+
for (const r of rejected) console.log(` ✗ ${r.file}: ${r.reason}`);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (!dryRun && created.length > 0) {
|
|
132
|
+
console.log('');
|
|
133
|
+
result(`Candidates: outputs/candidates/`);
|
|
134
|
+
result(`Records: records/`);
|
|
135
|
+
console.log('');
|
|
136
|
+
console.log(' Next steps:');
|
|
137
|
+
console.log(' 1. Review: sdlab curate <id> approved/rejected "explanation"');
|
|
138
|
+
console.log(' 2. Bind: sdlab bind --project ' + projectName);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Direct execution guard
|
|
143
|
+
if (process.argv[1] && (process.argv[1].endsWith('ingest.js') || process.argv[1].endsWith('ingest'))) {
|
|
144
|
+
run().catch(handleCliError);
|
|
145
|
+
}
|