@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.es.md +117 -55
- package/README.fr.md +125 -65
- package/README.hi.md +117 -68
- package/README.it.md +119 -58
- package/README.ja.md +124 -74
- package/README.md +73 -7
- package/README.pt-BR.md +128 -63
- package/README.zh.md +116 -65
- package/bin/sdlab.js +100 -4
- package/lib/adapters/ai-toolkit.js +224 -35
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +94 -11
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/asset-ingest.js +575 -0
- package/lib/asset-masks.js +323 -0
- package/lib/asset-source.js +1176 -0
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-author.js +350 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/canon-review-gate.js +205 -0
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/colab-launcher-guards.js +285 -0
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +16 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +194 -6
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/lane1-render.js +668 -0
- package/lib/measure.js +422 -0
- package/lib/memory-store.js +663 -0
- package/lib/model-licences.js +305 -0
- package/lib/npy-meta.js +130 -0
- package/lib/paths.js +196 -7
- package/lib/png-meta.js +512 -0
- package/lib/records.js +54 -8
- package/lib/reingest-selected.js +113 -17
- package/lib/reingest.js +27 -3
- package/lib/run-manifest.js +159 -25
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/train-runs.js +214 -0
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +3 -0
- package/lib/verify-chain.js +367 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +4 -3
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/asset-ingest.js +160 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-draft.js +82 -0
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +31 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
- package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
- package/scripts/colab/colab_window.sh +587 -0
- package/scripts/colab/lane1_render.py +970 -0
- package/scripts/colab/launch_guards.py +564 -0
- package/scripts/colab/launch_lane1_stub.sh +309 -0
- package/scripts/colab/launch_lane3_stub.sh +366 -0
- package/scripts/colab/remote_runner.py +59 -0
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +182 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +86 -17
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/verify.js +67 -0
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* migrate-records.js — One-time migration for curator-divergent records (Shape 2 + Shape 4)
|
|
4
|
+
*
|
|
5
|
+
* Normalizes verdict/scores/explanation/failures/curated_at/curator at top level
|
|
6
|
+
* into the standard judgment.{status,criteria_scores,explanation,failure_modes,reviewed_at,reviewer} shape.
|
|
7
|
+
*
|
|
8
|
+
* Also fixes asset_path from outputs/candidates/ to outputs/approved/ when the image exists in approved/.
|
|
9
|
+
*
|
|
10
|
+
* DOES NOT TOUCH Shape 3 (identity packet) records — those have judgment=null by design.
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* node scripts/migrate-records.js --dry-run # Preview changes
|
|
14
|
+
* node scripts/migrate-records.js # Apply changes
|
|
15
|
+
* sdlab project migrate --dry-run --project star-freight
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { readdir, readFile, writeFile, access } from 'node:fs/promises';
|
|
19
|
+
import { join, basename } from 'node:path';
|
|
20
|
+
import { getProjectName } from '../lib/args.js';
|
|
21
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
22
|
+
import { handleCliError } from '../lib/errors.js';
|
|
23
|
+
|
|
24
|
+
async function fileExists(p) {
|
|
25
|
+
try { await access(p); return true; } catch { return false; }
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function isCuratorDivergent(rec) {
|
|
29
|
+
return rec.judgment === null && typeof rec.verdict === 'string' && !rec.identity;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function isIdentityRecord(rec) {
|
|
33
|
+
return rec.identity != null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function migrateRecord(filePath, GAME_ROOT, DRY_RUN) {
|
|
37
|
+
const raw = await readFile(filePath, 'utf-8');
|
|
38
|
+
const rec = JSON.parse(raw);
|
|
39
|
+
|
|
40
|
+
if (isIdentityRecord(rec)) return { file: basename(filePath), action: 'skip-identity' };
|
|
41
|
+
|
|
42
|
+
if (!isCuratorDivergent(rec)) {
|
|
43
|
+
if (rec.asset_path && rec.asset_path.includes('outputs/candidates/')) {
|
|
44
|
+
const approvedPath = rec.asset_path.replace('outputs/candidates/', 'outputs/approved/');
|
|
45
|
+
const fullApproved = join(GAME_ROOT, approvedPath);
|
|
46
|
+
if (await fileExists(fullApproved)) {
|
|
47
|
+
rec.asset_path = approvedPath;
|
|
48
|
+
if (!DRY_RUN) await writeFile(filePath, JSON.stringify(rec, null, 2) + '\n');
|
|
49
|
+
return { file: basename(filePath), action: 'path-fix-only', from: 'candidates/', to: 'approved/' };
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { file: basename(filePath), action: 'skip-standard' };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const changes = [];
|
|
56
|
+
|
|
57
|
+
rec.judgment = {
|
|
58
|
+
status: rec.verdict || 'unknown',
|
|
59
|
+
reviewer: rec.curator || 'wave25-script',
|
|
60
|
+
reviewed_at: rec.curated_at || new Date().toISOString(),
|
|
61
|
+
explanation: rec.explanation || '',
|
|
62
|
+
criteria_scores: rec.scores || {},
|
|
63
|
+
failure_modes: rec.failures || [],
|
|
64
|
+
improvement_notes: null,
|
|
65
|
+
confidence: 0.8,
|
|
66
|
+
};
|
|
67
|
+
changes.push(`verdict="${rec.verdict}" → judgment.status`);
|
|
68
|
+
|
|
69
|
+
delete rec.verdict;
|
|
70
|
+
delete rec.scores;
|
|
71
|
+
delete rec.explanation;
|
|
72
|
+
delete rec.failures;
|
|
73
|
+
delete rec.curated_at;
|
|
74
|
+
delete rec.curator;
|
|
75
|
+
changes.push('removed top-level verdict/scores/explanation/failures/curated_at/curator');
|
|
76
|
+
|
|
77
|
+
if (rec.asset_path && rec.asset_path.includes('outputs/candidates/')) {
|
|
78
|
+
const approvedPath = rec.asset_path.replace('outputs/candidates/', 'outputs/approved/');
|
|
79
|
+
const fullApproved = join(GAME_ROOT, approvedPath);
|
|
80
|
+
if (await fileExists(fullApproved)) {
|
|
81
|
+
rec.asset_path = approvedPath;
|
|
82
|
+
changes.push(`asset_path: candidates/ → approved/`);
|
|
83
|
+
} else {
|
|
84
|
+
changes.push(`asset_path: candidates/ (image not in approved/, kept as-is)`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (!DRY_RUN) {
|
|
89
|
+
await writeFile(filePath, JSON.stringify(rec, null, 2) + '\n');
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return { file: basename(filePath), action: 'migrated', changes };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export async function run(argv = process.argv.slice(2)) {
|
|
96
|
+
const projectName = getProjectName(argv);
|
|
97
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
98
|
+
const RECORDS_DIR = join(GAME_ROOT, 'records');
|
|
99
|
+
const DRY_RUN = argv.includes('--dry-run');
|
|
100
|
+
|
|
101
|
+
console.log(DRY_RUN ? '=== DRY RUN ===' : '=== APPLYING MIGRATION ===');
|
|
102
|
+
|
|
103
|
+
const files = (await readdir(RECORDS_DIR)).filter(f => f.endsWith('.json'));
|
|
104
|
+
console.log(`Found ${files.length} record files\n`);
|
|
105
|
+
|
|
106
|
+
const results = { skipped: 0, migrated: 0, pathFixed: 0, identity: 0, errors: 0 };
|
|
107
|
+
|
|
108
|
+
for (const file of files) {
|
|
109
|
+
try {
|
|
110
|
+
const result = await migrateRecord(join(RECORDS_DIR, file), GAME_ROOT, DRY_RUN);
|
|
111
|
+
switch (result.action) {
|
|
112
|
+
case 'skip-standard': results.skipped++; break;
|
|
113
|
+
case 'skip-identity': results.identity++; break;
|
|
114
|
+
case 'migrated':
|
|
115
|
+
results.migrated++;
|
|
116
|
+
console.log(`MIGRATE: ${result.file}`);
|
|
117
|
+
result.changes.forEach(c => console.log(` → ${c}`));
|
|
118
|
+
break;
|
|
119
|
+
case 'path-fix-only':
|
|
120
|
+
results.pathFixed++;
|
|
121
|
+
console.log(`PATH-FIX: ${result.file} (${result.from} → ${result.to})`);
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
} catch (err) {
|
|
125
|
+
results.errors++;
|
|
126
|
+
console.error(`ERROR: ${file}: ${err.message}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
console.log(`\n=== Summary ===`);
|
|
131
|
+
console.log(`Standard (unchanged): ${results.skipped}`);
|
|
132
|
+
console.log(`Identity (untouched): ${results.identity}`);
|
|
133
|
+
console.log(`Migrated (schema fix): ${results.migrated}`);
|
|
134
|
+
console.log(`Path-fixed only: ${results.pathFixed}`);
|
|
135
|
+
console.log(`Errors: ${results.errors}`);
|
|
136
|
+
console.log(`Total: ${files.length}`);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Direct execution guard
|
|
140
|
+
if (process.argv[1] && (process.argv[1].endsWith('migrate-records.js') || process.argv[1].endsWith('migrate-records'))) {
|
|
141
|
+
run().catch(handleCliError);
|
|
142
|
+
}
|
|
@@ -12,8 +12,9 @@
|
|
|
12
12
|
import { writeFile, mkdir } from "node:fs/promises";
|
|
13
13
|
import { join } from "node:path";
|
|
14
14
|
import { getProjectName } from "../lib/args.js";
|
|
15
|
-
import {
|
|
15
|
+
import { getProjectRoot } from "../lib/paths.js";
|
|
16
16
|
import { submitAndWait, downloadImage, uploadImage } from "../lib/comfyui.js";
|
|
17
|
+
import { handleCliError } from "../lib/errors.js";
|
|
17
18
|
|
|
18
19
|
const PAINTERLY_PROMPT = [
|
|
19
20
|
"oil painting, visible brushstrokes, painterly concept art,",
|
|
@@ -72,7 +73,7 @@ function buildWorkflow(imagePath, denoise, loraWeight, seed) {
|
|
|
72
73
|
|
|
73
74
|
export async function run(argv = process.argv.slice(2)) {
|
|
74
75
|
const projectName = getProjectName(argv);
|
|
75
|
-
const GAME_ROOT =
|
|
76
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
76
77
|
const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
|
|
77
78
|
|
|
78
79
|
const outDir = join(GAME_ROOT, "outputs/painterly-test");
|
|
@@ -118,5 +119,5 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
118
119
|
|
|
119
120
|
// Direct execution guard
|
|
120
121
|
if (process.argv[1] && (process.argv[1].endsWith('painterly-test.js') || process.argv[1].endsWith('painterly-test'))) {
|
|
121
|
-
run().catch(
|
|
122
|
+
run().catch(handleCliError);
|
|
122
123
|
}
|
package/scripts/painterly.js
CHANGED
|
@@ -19,7 +19,7 @@ import { writeFile, mkdir, readdir } from "node:fs/promises";
|
|
|
19
19
|
import { join } from "node:path";
|
|
20
20
|
import { existsSync } from "node:fs";
|
|
21
21
|
import { getProjectName, parseNumberFlag } from "../lib/args.js";
|
|
22
|
-
import {
|
|
22
|
+
import { getProjectRoot, resolveSafeProjectPath } from "../lib/paths.js";
|
|
23
23
|
import { runtimeError, handleCliError } from "../lib/errors.js";
|
|
24
24
|
import { comfyHealth, submitAndWait, downloadImage, uploadImage } from "../lib/comfyui.js";
|
|
25
25
|
|
|
@@ -145,7 +145,7 @@ function buildImg2ImgWorkflow(imagePath, seed) {
|
|
|
145
145
|
|
|
146
146
|
export async function run(argv = process.argv.slice(2)) {
|
|
147
147
|
const projectName = getProjectName(argv);
|
|
148
|
-
const GAME_ROOT =
|
|
148
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
149
149
|
const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
|
|
150
150
|
|
|
151
151
|
const dryRun = argv.includes("--dry-run");
|
|
@@ -163,7 +163,11 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
163
163
|
|
|
164
164
|
const fullSourceDir = resolveSafeProjectPath(GAME_ROOT, sourceDir, { flagName: 'source' });
|
|
165
165
|
const outDir = join(GAME_ROOT, "outputs/painterly");
|
|
166
|
-
|
|
166
|
+
// SDL-M11: only create the output dir on a real run — a --dry-run preview
|
|
167
|
+
// must be side-effect-free (README.md promises "preview the effect first").
|
|
168
|
+
if (!dryRun) {
|
|
169
|
+
await mkdir(outDir, { recursive: true });
|
|
170
|
+
}
|
|
167
171
|
|
|
168
172
|
// Collect source PNGs
|
|
169
173
|
const allFiles = (await readdir(fullSourceDir))
|
package/scripts/qwen_generate.py
CHANGED
|
@@ -22,6 +22,7 @@ import os
|
|
|
22
22
|
import time
|
|
23
23
|
import urllib.request
|
|
24
24
|
import uuid
|
|
25
|
+
from decimal import Decimal
|
|
25
26
|
from pathlib import Path
|
|
26
27
|
|
|
27
28
|
from PIL import Image
|
|
@@ -58,20 +59,71 @@ _KIND_SUBDIRS = {
|
|
|
58
59
|
|
|
59
60
|
|
|
60
61
|
def _js_number(n) -> str:
|
|
61
|
-
"""Format a number the way JS
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
"""Format a number exactly the way JS Number::toString (ECMA-262
|
|
63
|
+
6.1.6.1.20) does, so the graph hash matches the JS helper
|
|
64
|
+
(lib/run-manifest.js stableStringify).
|
|
65
|
+
|
|
66
|
+
SDL-H3: the previous implementation delegated non-integer floats to
|
|
67
|
+
Python's repr(), which disagrees with JS in the [1e-6, 1e-4) magnitude
|
|
68
|
+
band and on exponent zero-padding:
|
|
69
|
+
0.00005 -> Python repr "5e-05" vs JS "0.00005"
|
|
70
|
+
0.000001 -> Python repr "1e-06" vs JS "0.000001"
|
|
71
|
+
9.9e-7 -> Python repr "9.9e-07" vs JS "9.9e-7"
|
|
72
|
+
Any wave with a LoRA weight or float param in that band produced a
|
|
73
|
+
different comfy_workflow_sha than the JS path for an identical graph.
|
|
74
|
+
|
|
75
|
+
Fix: Python's repr()/str() already compute the same "shortest
|
|
76
|
+
round-trip decimal digit string" JS engines use internally (both use a
|
|
77
|
+
Grisu/Ryu-class shortest-repr algorithm) — the two runtimes only
|
|
78
|
+
disagree on how that digit string is laid out (decimal point position,
|
|
79
|
+
when to switch to exponential, exponent zero-padding). So: borrow
|
|
80
|
+
Python's digit string via repr(), hand it to Decimal (which parses
|
|
81
|
+
repr()'s scientific-notation form natively, no manual string
|
|
82
|
+
splitting), then re-render per the ECMA spec's own branches.
|
|
83
|
+
"""
|
|
64
84
|
if isinstance(n, bool):
|
|
65
85
|
return "true" if n else "false"
|
|
66
86
|
if isinstance(n, int):
|
|
67
87
|
return str(n)
|
|
68
88
|
if isinstance(n, float):
|
|
69
|
-
if
|
|
89
|
+
if math.isnan(n) or math.isinf(n):
|
|
70
90
|
return "null"
|
|
71
|
-
|
|
91
|
+
if n == 0:
|
|
92
|
+
return "0" # covers -0.0 too — JS String(-0) is "0"
|
|
93
|
+
sign = "-" if n < 0 else ""
|
|
94
|
+
_, digits_tuple, exponent = Decimal(repr(abs(n))).as_tuple()
|
|
95
|
+
m = len(digits_tuple)
|
|
96
|
+
# value == 0.<all m digits> * 10**point_pos (this is ECMA's "n")
|
|
97
|
+
point_pos = exponent + m
|
|
98
|
+
digits = "".join(map(str, digits_tuple)).rstrip("0") or "0"
|
|
99
|
+
k = len(digits)
|
|
100
|
+
if k <= point_pos <= 21:
|
|
101
|
+
return sign + digits + ("0" * (point_pos - k))
|
|
102
|
+
if 0 < point_pos <= 21:
|
|
103
|
+
return sign + digits[:point_pos] + "." + digits[point_pos:]
|
|
104
|
+
if -6 < point_pos <= 0:
|
|
105
|
+
return sign + "0." + ("0" * -point_pos) + digits
|
|
106
|
+
mantissa = digits[0] if k == 1 else digits[0] + "." + digits[1:]
|
|
107
|
+
e = point_pos - 1
|
|
108
|
+
esign = "+" if e >= 0 else "-"
|
|
109
|
+
return f"{sign}{mantissa}e{esign}{abs(e)}"
|
|
72
110
|
return "null"
|
|
73
111
|
|
|
74
112
|
|
|
113
|
+
def _js_nullish(value, default):
|
|
114
|
+
"""Mirror JS `value ?? default` — substitutes only when value is None
|
|
115
|
+
(JS null/undefined), never for other falsy values like 0 or False.
|
|
116
|
+
|
|
117
|
+
SDL-H4: dict.get(key, default) is NOT equivalent — it only substitutes
|
|
118
|
+
when the KEY is missing, never when the key is present with an
|
|
119
|
+
explicit null. {"weight": None}.get("weight", 1.0) returns None, not
|
|
120
|
+
1.0, silently splitting the graph hash from the JS side's
|
|
121
|
+
`l.weight ?? 1.0` AND submitting a literal null strength_model to
|
|
122
|
+
ComfyUI's /prompt.
|
|
123
|
+
"""
|
|
124
|
+
return default if value is None else value
|
|
125
|
+
|
|
126
|
+
|
|
75
127
|
def _stable_stringify(value) -> str:
|
|
76
128
|
"""Deterministic serialization mirroring lib/run-manifest.js stableStringify:
|
|
77
129
|
object keys sorted, arrays in order, JS-style number/string formatting."""
|
|
@@ -204,7 +256,7 @@ def _build_pinning(graph, models, loras, seed_policy, do_hash, cache) -> dict:
|
|
|
204
256
|
lora_hashes = []
|
|
205
257
|
for lora in loras or []:
|
|
206
258
|
entry = _hash_model_file(lora.get("name"), "lora", do_hash, cache)
|
|
207
|
-
entry["weight"] = lora.get("weight", 1.0)
|
|
259
|
+
entry["weight"] = _js_nullish(lora.get("weight"), 1.0) # SDL-H4
|
|
208
260
|
lora_hashes.append(entry)
|
|
209
261
|
return {
|
|
210
262
|
"pinning_version": PINNING_VERSION,
|
|
@@ -216,11 +268,22 @@ def _build_pinning(graph, models, loras, seed_policy, do_hash, cache) -> dict:
|
|
|
216
268
|
|
|
217
269
|
|
|
218
270
|
def qwen_graph(pos: str, neg: str, seed: int, w: int, h: int, steps: int, cfg: float, prefix: str,
|
|
219
|
-
loras: list | None = None
|
|
271
|
+
loras: list | None = None, sampler: str = "euler", scheduler: str = "simple",
|
|
272
|
+
shift: float = 3.1) -> dict:
|
|
220
273
|
"""loras: [{name, weight}] chained as DiT-only LoraLoaderModelOnly nodes (ids from
|
|
221
274
|
"40", between UNETLoader and ModelSamplingAuraFlow) — mirrors sdlab buildQwenGraph.
|
|
222
275
|
Model-only because studio Qwen style LoRAs train the transformer with the text
|
|
223
|
-
encoder frozen.
|
|
276
|
+
encoder frozen.
|
|
277
|
+
|
|
278
|
+
SDL-H11: sampler/scheduler/shift used to be hardcoded literals here (always
|
|
279
|
+
"euler"/"simple"/3.1, regardless of what the wave declared) and were not
|
|
280
|
+
even accepted as parameters — so a wave asking for dpmpp_2m/karras/shift 5.0
|
|
281
|
+
silently generated with euler/simple/3.1 instead, produced a different
|
|
282
|
+
comfy_workflow_sha than the JS `sdlab generate` path for the "same" graph,
|
|
283
|
+
AND had those wrong values stamped into the provenance receipt. Defaults
|
|
284
|
+
here match QWEN_DEFAULTS in lib/adapters/comfyui-workflows.js — used only
|
|
285
|
+
when a wave omits the fields (mirrors JS's `rp.sampler || defaults.sampler
|
|
286
|
+
|| QWEN_DEFAULTS.sampler` fallback chain)."""
|
|
224
287
|
nodes = {
|
|
225
288
|
"37": {"class_type": "UNETLoader", "inputs": {"unet_name": QWEN_UNET, "weight_dtype": "default"}},
|
|
226
289
|
"38": {"class_type": "CLIPLoader", "inputs": {"clip_name": QWEN_CLIP, "type": "qwen_image"}},
|
|
@@ -232,13 +295,13 @@ def qwen_graph(pos: str, neg: str, seed: int, w: int, h: int, steps: int, cfg: f
|
|
|
232
295
|
for i, lora in enumerate(loras or []):
|
|
233
296
|
lid = str(40 + i)
|
|
234
297
|
nodes[lid] = {"class_type": "LoraLoaderModelOnly", "inputs": {
|
|
235
|
-
"lora_name": lora["name"], "strength_model": lora.get("weight", 1.0), "model": model_out}}
|
|
298
|
+
"lora_name": lora["name"], "strength_model": _js_nullish(lora.get("weight"), 1.0), "model": model_out}} # SDL-H4
|
|
236
299
|
model_out = [lid, 0]
|
|
237
300
|
nodes.update({
|
|
238
|
-
"66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift":
|
|
301
|
+
"66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift": shift, "model": model_out}},
|
|
239
302
|
"5": {"class_type": "EmptySD3LatentImage", "inputs": {"width": w, "height": h, "batch_size": 1}},
|
|
240
|
-
"3": {"class_type": "KSampler", "inputs": {"seed": seed, "steps": steps, "cfg": cfg, "sampler_name":
|
|
241
|
-
"scheduler":
|
|
303
|
+
"3": {"class_type": "KSampler", "inputs": {"seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler,
|
|
304
|
+
"scheduler": scheduler, "denoise": 1.0, "model": ["66", 0], "positive": ["6", 0], "negative": ["7", 0], "latent_image": ["5", 0]}},
|
|
242
305
|
"8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
|
|
243
306
|
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": prefix, "images": ["8", 0]}},
|
|
244
307
|
})
|
|
@@ -251,7 +314,9 @@ def _post(path: str, payload: dict):
|
|
|
251
314
|
|
|
252
315
|
|
|
253
316
|
def generate(pos: str, neg: str, seed: int, dst: Path, **kw) -> Path:
|
|
254
|
-
g = qwen_graph(pos, neg, seed, kw["w"], kw["h"], kw["steps"], kw["cfg"], dst.stem, loras=kw.get("loras")
|
|
317
|
+
g = qwen_graph(pos, neg, seed, kw["w"], kw["h"], kw["steps"], kw["cfg"], dst.stem, loras=kw.get("loras"),
|
|
318
|
+
sampler=kw.get("sampler", "euler"), scheduler=kw.get("scheduler", "simple"),
|
|
319
|
+
shift=kw.get("shift", 3.1))
|
|
255
320
|
pid = _post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
|
|
256
321
|
for _ in range(1200):
|
|
257
322
|
hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
|
|
@@ -284,10 +349,16 @@ def main() -> None:
|
|
|
284
349
|
steps, cfg = d.get("steps", 22), d.get("cfg", 3.5)
|
|
285
350
|
base_seed = d.get("base_seed", 1000)
|
|
286
351
|
loras = d.get("loras", [])
|
|
352
|
+
# SDL-H11: read sampler/scheduler/shift from the wave instead of hardcoding
|
|
353
|
+
# them — defaults match QWEN_DEFAULTS in lib/adapters/comfyui-workflows.js
|
|
354
|
+
# so an omitted field falls back to the same value the JS path would use.
|
|
355
|
+
sampler = d.get("sampler", "euler")
|
|
356
|
+
scheduler = d.get("scheduler", "simple")
|
|
357
|
+
shift = d.get("shift", 3.1)
|
|
287
358
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
288
359
|
|
|
289
360
|
receipt = {"wave": wave.get("wave"), "base": "qwen-image", "unet": QWEN_UNET, "clip": QWEN_CLIP,
|
|
290
|
-
"vae": QWEN_VAE, "sampler":
|
|
361
|
+
"vae": QWEN_VAE, "sampler": sampler, "scheduler": scheduler, "shift": shift,
|
|
291
362
|
"steps": steps, "cfg": cfg, "size": [w, h], "style_prefix": prefix, "negative": neg,
|
|
292
363
|
"loras": loras, "items": []}
|
|
293
364
|
|
|
@@ -296,7 +367,8 @@ def main() -> None:
|
|
|
296
367
|
# out and recorded per item in items[]). seed_policy is "explicit-per-item"
|
|
297
368
|
# because each item's seed is written verbatim below. Mirror of the JS contract.
|
|
298
369
|
hash_cache = _load_hash_cache()
|
|
299
|
-
pin_graph = qwen_graph(prefix or "x", neg, base_seed, w, h, steps, cfg, "sdl", loras=loras
|
|
370
|
+
pin_graph = qwen_graph(prefix or "x", neg, base_seed, w, h, steps, cfg, "sdl", loras=loras,
|
|
371
|
+
sampler=sampler, scheduler=scheduler, shift=shift)
|
|
300
372
|
receipt["pinning"] = _build_pinning(
|
|
301
373
|
pin_graph,
|
|
302
374
|
{"unet": QWEN_UNET, "clip": QWEN_CLIP, "vae": QWEN_VAE},
|
|
@@ -306,21 +378,67 @@ def main() -> None:
|
|
|
306
378
|
hash_cache,
|
|
307
379
|
)
|
|
308
380
|
|
|
381
|
+
receipt_path = args.out / "generation.json"
|
|
382
|
+
|
|
383
|
+
def _save_receipt() -> None:
|
|
384
|
+
# M3: incremental checkpoint (mirrors lib/batch-runs.js's per-slot
|
|
385
|
+
# atomicWriteJson pattern) — written after EVERY item, not once at
|
|
386
|
+
# the end. Before this fix, the subject loop called generate() with
|
|
387
|
+
# no try/except (it can raise the named ComfyUI timeout at its
|
|
388
|
+
# RuntimeError above), and generation.json was written exactly once,
|
|
389
|
+
# after the loop finished. A 200-image wave failing at item 130 left
|
|
390
|
+
# 129 rendered PNGs on disk with NO generation.json at all — the
|
|
391
|
+
# PIN_PER_STEP provenance this file's own docstring promises, fully
|
|
392
|
+
# lost, for every item including the ones that succeeded.
|
|
393
|
+
receipt_path.write_text(json.dumps(receipt, indent=2), encoding="utf-8")
|
|
394
|
+
|
|
309
395
|
seed = base_seed
|
|
396
|
+
error_count = 0
|
|
310
397
|
for subj in wave["subjects"]:
|
|
311
398
|
nvar = args.variations if args.variations is not None else subj.get("variations", 1)
|
|
312
399
|
pos = f"{prefix}, {subj['prompt']}" if prefix else subj["prompt"]
|
|
313
400
|
for v in range(nvar):
|
|
314
401
|
dst = args.out / f"{subj['id']}_v{v}.png"
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
402
|
+
item = {"id": subj["id"], "variation": v, "seed": seed, "file": dst.name, "prompt": pos}
|
|
403
|
+
try:
|
|
404
|
+
generate(pos, neg, seed, dst, w=w, h=h, steps=steps, cfg=cfg, loras=loras,
|
|
405
|
+
sampler=sampler, scheduler=scheduler, shift=shift)
|
|
406
|
+
except Exception as exc:
|
|
407
|
+
# A single item's failure no longer takes the whole wave's
|
|
408
|
+
# provenance down with it — record the failure and keep
|
|
409
|
+
# going. The receipt is checkpointed below regardless of
|
|
410
|
+
# per-item outcome.
|
|
411
|
+
item["status"] = "error"
|
|
412
|
+
item["error"] = str(exc)
|
|
413
|
+
error_count += 1
|
|
414
|
+
print(f" {subj['id']} v{v} (seed {seed}) -> ERROR: {exc}", flush=True)
|
|
415
|
+
else:
|
|
416
|
+
item["status"] = "ok"
|
|
417
|
+
print(f" {subj['id']} v{v} (seed {seed}) -> {dst.name}", flush=True)
|
|
418
|
+
receipt["items"].append(item)
|
|
318
419
|
seed += 1
|
|
420
|
+
_save_receipt()
|
|
319
421
|
|
|
320
422
|
if args.hash_models:
|
|
321
423
|
_save_hash_cache(hash_cache)
|
|
322
|
-
(
|
|
323
|
-
|
|
424
|
+
_save_receipt()
|
|
425
|
+
ok_count = len(receipt["items"]) - error_count
|
|
426
|
+
# Plain ASCII only (no em-dash) — matches the rest of this file's
|
|
427
|
+
# runtime print()/error strings. A Windows console not running in UTF-8
|
|
428
|
+
# mode mangles non-ASCII on stdout; the docstring/comments can afford
|
|
429
|
+
# Unicode since those never reach a terminal at runtime.
|
|
430
|
+
summary = f"\n{ok_count}/{len(receipt['items'])} images -> {args.out} (+ generation.json provenance)"
|
|
431
|
+
if error_count:
|
|
432
|
+
summary += f" ({error_count} error(s) - see generation.json for per-item detail)"
|
|
433
|
+
print(summary)
|
|
434
|
+
|
|
435
|
+
# Mirror scripts/generate.js's RUNTIME_ALL_FAILED convention: a run where
|
|
436
|
+
# EVERY item failed exits non-zero (nothing to salvage); a partial
|
|
437
|
+
# failure still exits 0 — the per-item errors are visible above and in
|
|
438
|
+
# the checkpointed receipt, and individual item failure is tolerated
|
|
439
|
+
# everywhere else in this pipeline.
|
|
440
|
+
if receipt["items"] and error_count == len(receipt["items"]):
|
|
441
|
+
raise SystemExit(f"all {error_count} generation attempt(s) failed - see generation.json for per-item errors")
|
|
324
442
|
|
|
325
443
|
|
|
326
444
|
if __name__ == "__main__":
|
package/scripts/reingest.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { join } from 'node:path';
|
|
13
13
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
14
|
-
import { REPO_ROOT, resolveSafeProjectPath } from '../lib/paths.js';
|
|
14
|
+
import { REPO_ROOT, getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
|
|
15
15
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
16
16
|
import { reingestGenerated, auditReingest } from '../lib/reingest.js';
|
|
17
17
|
|
|
@@ -29,7 +29,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
29
29
|
});
|
|
30
30
|
|
|
31
31
|
const projectName = flags.project || getProjectName(argv);
|
|
32
|
-
const projectRoot =
|
|
32
|
+
const projectRoot = getProjectRoot(projectName);
|
|
33
33
|
const subcommand = positionals[0] || 'audit';
|
|
34
34
|
|
|
35
35
|
if (subcommand === 'generated') {
|
package/scripts/sheet.js
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* sheet.js — Render an HTML contact sheet over a directory of images.
|
|
5
|
+
*
|
|
6
|
+
* No batch manifest required. Points the SAME HTML renderer that powers
|
|
7
|
+
* `sdlab batch sheet` (lib/batch-sheet-render.js, via lib/candidate-sheet.js)
|
|
8
|
+
* at an arbitrary directory under a project — outputs/candidates/,
|
|
9
|
+
* outputs/approved/, outputs/, whatever — instead of requiring a
|
|
10
|
+
* batches/<id>/manifest.json that only `sdlab batch generate` can produce.
|
|
11
|
+
*
|
|
12
|
+
* Each image is matched to its record by asset_path (when a record exists)
|
|
13
|
+
* for id / curation status / prompt. Images with no matching record
|
|
14
|
+
* (pre-ingest — generated but not yet re-ingested) still render, labeled by
|
|
15
|
+
* filename instead of crashing or being skipped.
|
|
16
|
+
*
|
|
17
|
+
* Usage:
|
|
18
|
+
* sdlab sheet [<dir>] --project <name> [--out <path>] [--dry-run]
|
|
19
|
+
*
|
|
20
|
+
* Examples:
|
|
21
|
+
* sdlab sheet --project star-freight # outputs/candidates (default)
|
|
22
|
+
* sdlab sheet outputs/approved --project star-freight
|
|
23
|
+
* sdlab sheet outputs --project star-freight --dry-run # whole tree, no write
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { existsSync } from 'node:fs';
|
|
27
|
+
import { relative, dirname } from 'node:path';
|
|
28
|
+
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
29
|
+
import { getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
|
|
30
|
+
import { inputError, handleCliError } from '../lib/errors.js';
|
|
31
|
+
import { result } from '../lib/log.js';
|
|
32
|
+
import {
|
|
33
|
+
buildCandidateSheetItems,
|
|
34
|
+
renderCandidateSheetHTML,
|
|
35
|
+
saveCandidateSheet,
|
|
36
|
+
defaultSheetOutputPath,
|
|
37
|
+
} from '../lib/candidate-sheet.js';
|
|
38
|
+
|
|
39
|
+
const DEFAULT_DIR = 'outputs/candidates';
|
|
40
|
+
|
|
41
|
+
function posix(p) {
|
|
42
|
+
return p.replace(/\\/g, '/');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function run(argv = process.argv.slice(2)) {
|
|
46
|
+
// Checked against the raw argv, not via parseArgs' flag defs: parseArgs
|
|
47
|
+
// only recognizes "--"-prefixed flags (see lib/args.js), so a bare "-h"
|
|
48
|
+
// would otherwise fall through to the positional `dir` slot instead of
|
|
49
|
+
// showing help. Mirrors bin/sdlab.js's own argvHasHelp() check.
|
|
50
|
+
if (argv.includes('--help') || argv.includes('-h')) {
|
|
51
|
+
printUsage();
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const { flags, positionals } = parseArgs(argv, {
|
|
56
|
+
flags: {
|
|
57
|
+
project: { type: 'string' },
|
|
58
|
+
out: { type: 'string' },
|
|
59
|
+
'dry-run': { type: 'boolean', default: false },
|
|
60
|
+
},
|
|
61
|
+
positionals: ['dir'],
|
|
62
|
+
deprecated: { game: 'project' },
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
const projectName = flags.project || getProjectName(argv);
|
|
66
|
+
const projectRoot = getProjectRoot(projectName);
|
|
67
|
+
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
68
|
+
|
|
69
|
+
const dirArg = positionals[0] || DEFAULT_DIR;
|
|
70
|
+
// Containment check — a user-supplied path must never be joined by hand.
|
|
71
|
+
const safeDir = resolveSafeProjectPath(projectRoot, dirArg, { flagName: 'dir' });
|
|
72
|
+
const relDir = posix(relative(projectRoot, safeDir)) || '.';
|
|
73
|
+
|
|
74
|
+
if (!existsSync(safeDir)) {
|
|
75
|
+
throw inputError(
|
|
76
|
+
'INPUT_DIR_NOT_FOUND',
|
|
77
|
+
`Directory not found: ${relDir} (looked under ${projectRoot})`,
|
|
78
|
+
'Try outputs/candidates, outputs/approved, outputs/rejected, outputs/borderline, or outputs (the whole tree).'
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const items = await buildCandidateSheetItems(projectRoot, safeDir);
|
|
83
|
+
|
|
84
|
+
console.log(`\x1b[1mstyle-dataset-lab\x1b[0m sheet — ${projectName}`);
|
|
85
|
+
console.log(` Source: ${relDir}`);
|
|
86
|
+
|
|
87
|
+
if (items.length === 0) {
|
|
88
|
+
console.log(' No images found — nothing to render.');
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const matched = items.filter((i) => i.hasRecord).length;
|
|
93
|
+
console.log(` Images: ${items.length} (${matched} with records, ${items.length - matched} pre-ingest)`);
|
|
94
|
+
|
|
95
|
+
const outRel = flags.out
|
|
96
|
+
? flags.out
|
|
97
|
+
: posix(relative(projectRoot, defaultSheetOutputPath(projectRoot, relDir)));
|
|
98
|
+
const safeOut = resolveSafeProjectPath(projectRoot, outRel, { flagName: 'out' });
|
|
99
|
+
|
|
100
|
+
if (dryRun) {
|
|
101
|
+
console.log(` \x1b[33m(dry-run)\x1b[0m would write: ${posix(relative(projectRoot, safeOut))}`);
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const html = renderCandidateSheetHTML({
|
|
106
|
+
items,
|
|
107
|
+
title: `Contact sheet — ${projectName} — ${relDir}`,
|
|
108
|
+
sourceLabel: relDir,
|
|
109
|
+
sheetOutputDir: dirname(safeOut),
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
const written = await saveCandidateSheet(safeOut, html);
|
|
113
|
+
result('sheet', posix(relative(projectRoot, written)));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function printUsage() {
|
|
117
|
+
console.log('Usage: sdlab sheet [<dir>] --project <name> [--out <path>] [--dry-run]');
|
|
118
|
+
console.log('');
|
|
119
|
+
console.log('Render an HTML contact sheet over a directory of images — no batch');
|
|
120
|
+
console.log('manifest required. Matches each image to its record (by asset_path)');
|
|
121
|
+
console.log('when one exists; degrades to filename when it does not (pre-ingest).');
|
|
122
|
+
console.log('');
|
|
123
|
+
console.log('Positional:');
|
|
124
|
+
console.log(' dir Directory to sheet, relative to the project root');
|
|
125
|
+
console.log(` (default: ${DEFAULT_DIR})`);
|
|
126
|
+
console.log('');
|
|
127
|
+
console.log('Flags:');
|
|
128
|
+
console.log(' --project <name> Project to operate on');
|
|
129
|
+
console.log(' --out <path> Output HTML path, relative to project root');
|
|
130
|
+
console.log(' (default: outputs/sheets/<slugified-dir>.html)');
|
|
131
|
+
console.log(' --dry-run Report what would be written without writing it');
|
|
132
|
+
console.log('');
|
|
133
|
+
console.log('Examples:');
|
|
134
|
+
console.log(' sdlab sheet --project star-freight');
|
|
135
|
+
console.log(' sdlab sheet outputs/approved --project star-freight');
|
|
136
|
+
console.log(' sdlab sheet outputs --project star-freight --dry-run');
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Direct execution guard — same pattern as every other script in this repo.
|
|
140
|
+
if (process.argv[1] && (process.argv[1].endsWith('sheet.js') || process.argv[1].endsWith('sheet'))) {
|
|
141
|
+
run().catch(handleCliError);
|
|
142
|
+
}
|
package/scripts/snapshot.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { readFile } from 'node:fs/promises';
|
|
14
14
|
import { join } from 'node:path';
|
|
15
15
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
16
|
-
import {
|
|
16
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
17
17
|
import { loadSelectionProfile } from '../lib/config.js';
|
|
18
18
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
19
19
|
import { createSnapshot, loadSnapshot, listSnapshots, diffSnapshots } from '../lib/snapshot.js';
|
|
@@ -30,7 +30,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
const projectName = flags.project || getProjectName(argv);
|
|
33
|
-
const projectRoot =
|
|
33
|
+
const projectRoot = getProjectRoot(projectName);
|
|
34
34
|
const subcommand = positionals[0] || 'list';
|
|
35
35
|
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
36
36
|
|
package/scripts/split.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
15
|
-
import {
|
|
15
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
16
16
|
import { loadSplitProfile } from '../lib/config.js';
|
|
17
17
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
18
18
|
import { listSnapshots } from '../lib/snapshot.js';
|
|
@@ -30,7 +30,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
const projectName = flags.project || getProjectName(argv);
|
|
33
|
-
const projectRoot =
|
|
33
|
+
const projectRoot = getProjectRoot(projectName);
|
|
34
34
|
const subcommand = positionals[0] || 'list';
|
|
35
35
|
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
36
36
|
|