@mcptoolshop/style-dataset-lab 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -1
- package/README.es.md +40 -19
- package/README.fr.md +74 -53
- package/README.hi.md +67 -46
- package/README.it.md +42 -21
- package/README.ja.md +44 -23
- package/README.md +25 -4
- package/README.pt-BR.md +42 -21
- package/README.zh.md +66 -45
- package/bin/sdlab.js +51 -4
- package/lib/adapters/ai-toolkit.js +37 -5
- package/lib/adapters/comfyui-runner.js +114 -27
- package/lib/adapters/comfyui-workflows.js +10 -2
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +82 -3
- package/lib/critique-render.js +45 -0
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +69 -16
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +16 -1
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/measure.js +422 -0
- package/lib/paths.js +196 -7
- package/lib/reingest-selected.js +98 -16
- package/lib/run-manifest.js +394 -0
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +121 -16
- package/lib/split.js +364 -48
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +67 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +3 -2
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +27 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +48 -8
- package/scripts/curate.js +23 -2
- package/scripts/doctor.js +181 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +72 -5
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +334 -13
- package/scripts/reingest.js +2 -2
- package/scripts/run-generate.js +3 -0
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
package/scripts/qwen_generate.py
CHANGED
|
@@ -15,10 +15,14 @@ embedded python (has PIL): start the watchdog (E:/AI/training/_watchdog.ps1) bef
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
17
|
import argparse
|
|
18
|
+
import hashlib
|
|
18
19
|
import json
|
|
20
|
+
import math
|
|
21
|
+
import os
|
|
19
22
|
import time
|
|
20
23
|
import urllib.request
|
|
21
24
|
import uuid
|
|
25
|
+
from decimal import Decimal
|
|
22
26
|
from pathlib import Path
|
|
23
27
|
|
|
24
28
|
from PIL import Image
|
|
@@ -31,13 +35,255 @@ QWEN_UNET = "qwen_image_fp8_e4m3fn.safetensors"
|
|
|
31
35
|
QWEN_CLIP = "qwen_2.5_vl_7b_fp8_scaled.safetensors"
|
|
32
36
|
QWEN_VAE = "qwen_image_vae.safetensors"
|
|
33
37
|
|
|
38
|
+
# ─── PIN_PER_STEP pinning contract ────────────────────────────────────────────
|
|
39
|
+
# Mirror of lib/run-manifest.js (the JS helper is the field-name authority). This
|
|
40
|
+
# receipt carries the IDENTICAL `pinning` block so a wave produced by this bridge
|
|
41
|
+
# generator is pinned the same way as one produced through the JS `sdlab generate`
|
|
42
|
+
# / `sdlab run generate` paths. Keep PINNING_VERSION + the field names in lockstep
|
|
43
|
+
# with lib/run-manifest.js.
|
|
44
|
+
PINNING_VERSION = "1.0.0"
|
|
45
|
+
MODELS_DIR = os.environ.get(
|
|
46
|
+
"SDLAB_MODELS_DIR", r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/models"
|
|
47
|
+
)
|
|
48
|
+
HASH_CACHE_FILE = os.environ.get("SDLAB_HASH_CACHE") or str(
|
|
49
|
+
Path(MODELS_DIR) / ".sdlab-hash-cache.json"
|
|
50
|
+
)
|
|
51
|
+
# kind → candidate subdirs under the models dir, in search order (mirror of JS).
|
|
52
|
+
_KIND_SUBDIRS = {
|
|
53
|
+
"unet": ["diffusion_models", "unet"],
|
|
54
|
+
"clip": ["text_encoders", "clip"],
|
|
55
|
+
"vae": ["vae"],
|
|
56
|
+
"checkpoint": ["checkpoints"],
|
|
57
|
+
"lora": ["loras"],
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _js_number(n) -> str:
|
|
62
|
+
"""Format a number exactly the way JS Number::toString (ECMA-262
|
|
63
|
+
6.1.6.1.20) does, so the graph hash matches the JS helper
|
|
64
|
+
(lib/run-manifest.js stableStringify).
|
|
65
|
+
|
|
66
|
+
SDL-H3: the previous implementation delegated non-integer floats to
|
|
67
|
+
Python's repr(), which disagrees with JS in the [1e-6, 1e-4) magnitude
|
|
68
|
+
band and on exponent zero-padding:
|
|
69
|
+
0.00005 -> Python repr "5e-05" vs JS "0.00005"
|
|
70
|
+
0.000001 -> Python repr "1e-06" vs JS "0.000001"
|
|
71
|
+
9.9e-7 -> Python repr "9.9e-07" vs JS "9.9e-7"
|
|
72
|
+
Any wave with a LoRA weight or float param in that band produced a
|
|
73
|
+
different comfy_workflow_sha than the JS path for an identical graph.
|
|
74
|
+
|
|
75
|
+
Fix: Python's repr()/str() already compute the same "shortest
|
|
76
|
+
round-trip decimal digit string" JS engines use internally (both use a
|
|
77
|
+
Grisu/Ryu-class shortest-repr algorithm) — the two runtimes only
|
|
78
|
+
disagree on how that digit string is laid out (decimal point position,
|
|
79
|
+
when to switch to exponential, exponent zero-padding). So: borrow
|
|
80
|
+
Python's digit string via repr(), hand it to Decimal (which parses
|
|
81
|
+
repr()'s scientific-notation form natively, no manual string
|
|
82
|
+
splitting), then re-render per the ECMA spec's own branches.
|
|
83
|
+
"""
|
|
84
|
+
if isinstance(n, bool):
|
|
85
|
+
return "true" if n else "false"
|
|
86
|
+
if isinstance(n, int):
|
|
87
|
+
return str(n)
|
|
88
|
+
if isinstance(n, float):
|
|
89
|
+
if math.isnan(n) or math.isinf(n):
|
|
90
|
+
return "null"
|
|
91
|
+
if n == 0:
|
|
92
|
+
return "0" # covers -0.0 too — JS String(-0) is "0"
|
|
93
|
+
sign = "-" if n < 0 else ""
|
|
94
|
+
_, digits_tuple, exponent = Decimal(repr(abs(n))).as_tuple()
|
|
95
|
+
m = len(digits_tuple)
|
|
96
|
+
# value == 0.<all m digits> * 10**point_pos (this is ECMA's "n")
|
|
97
|
+
point_pos = exponent + m
|
|
98
|
+
digits = "".join(map(str, digits_tuple)).rstrip("0") or "0"
|
|
99
|
+
k = len(digits)
|
|
100
|
+
if k <= point_pos <= 21:
|
|
101
|
+
return sign + digits + ("0" * (point_pos - k))
|
|
102
|
+
if 0 < point_pos <= 21:
|
|
103
|
+
return sign + digits[:point_pos] + "." + digits[point_pos:]
|
|
104
|
+
if -6 < point_pos <= 0:
|
|
105
|
+
return sign + "0." + ("0" * -point_pos) + digits
|
|
106
|
+
mantissa = digits[0] if k == 1 else digits[0] + "." + digits[1:]
|
|
107
|
+
e = point_pos - 1
|
|
108
|
+
esign = "+" if e >= 0 else "-"
|
|
109
|
+
return f"{sign}{mantissa}e{esign}{abs(e)}"
|
|
110
|
+
return "null"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _js_nullish(value, default):
|
|
114
|
+
"""Mirror JS `value ?? default` — substitutes only when value is None
|
|
115
|
+
(JS null/undefined), never for other falsy values like 0 or False.
|
|
116
|
+
|
|
117
|
+
SDL-H4: dict.get(key, default) is NOT equivalent — it only substitutes
|
|
118
|
+
when the KEY is missing, never when the key is present with an
|
|
119
|
+
explicit null. {"weight": None}.get("weight", 1.0) returns None, not
|
|
120
|
+
1.0, silently splitting the graph hash from the JS side's
|
|
121
|
+
`l.weight ?? 1.0` AND submitting a literal null strength_model to
|
|
122
|
+
ComfyUI's /prompt.
|
|
123
|
+
"""
|
|
124
|
+
return default if value is None else value
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _stable_stringify(value) -> str:
|
|
128
|
+
"""Deterministic serialization mirroring lib/run-manifest.js stableStringify:
|
|
129
|
+
object keys sorted, arrays in order, JS-style number/string formatting."""
|
|
130
|
+
if value is None:
|
|
131
|
+
return "null"
|
|
132
|
+
if isinstance(value, bool):
|
|
133
|
+
return "true" if value else "false"
|
|
134
|
+
if isinstance(value, (int, float)):
|
|
135
|
+
return _js_number(value)
|
|
136
|
+
if isinstance(value, str):
|
|
137
|
+
return json.dumps(value, ensure_ascii=False)
|
|
138
|
+
if isinstance(value, (list, tuple)):
|
|
139
|
+
return "[" + ",".join(_stable_stringify(v) for v in value) + "]"
|
|
140
|
+
if isinstance(value, dict):
|
|
141
|
+
# JS omits `undefined` keys; Python has no undefined, and None maps to JS
|
|
142
|
+
# `null` which stableStringify DOES serialize — so serialize every key.
|
|
143
|
+
parts = [
|
|
144
|
+
json.dumps(k, ensure_ascii=False) + ":" + _stable_stringify(value[k])
|
|
145
|
+
for k in sorted(value.keys())
|
|
146
|
+
]
|
|
147
|
+
return "{" + ",".join(parts) + "}"
|
|
148
|
+
return "null"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _normalize_graph_for_hash(graph: dict) -> dict:
|
|
152
|
+
"""Blank per-item nondeterminism (seed, prompt text, filename_prefix) so the
|
|
153
|
+
whole wave shares one comfy_workflow_sha. Mirror of the JS normalizer."""
|
|
154
|
+
out = {}
|
|
155
|
+
for nid, node in (graph or {}).items():
|
|
156
|
+
inputs = dict(node.get("inputs", {}))
|
|
157
|
+
cls = node.get("class_type")
|
|
158
|
+
if cls in ("KSampler", "KSamplerAdvanced"):
|
|
159
|
+
if "seed" in inputs:
|
|
160
|
+
inputs["seed"] = 0
|
|
161
|
+
if "noise_seed" in inputs:
|
|
162
|
+
inputs["noise_seed"] = 0
|
|
163
|
+
elif cls == "CLIPTextEncode":
|
|
164
|
+
if "text" in inputs:
|
|
165
|
+
inputs["text"] = ""
|
|
166
|
+
elif cls == "SaveImage":
|
|
167
|
+
if "filename_prefix" in inputs:
|
|
168
|
+
inputs["filename_prefix"] = ""
|
|
169
|
+
out[nid] = {"class_type": cls, "inputs": inputs}
|
|
170
|
+
return out
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _comfy_workflow_sha(graph: dict):
|
|
174
|
+
if not graph:
|
|
175
|
+
return None
|
|
176
|
+
payload = _stable_stringify(_normalize_graph_for_hash(graph))
|
|
177
|
+
return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _resolve_model_path(name: str, kind: str):
|
|
181
|
+
if not name:
|
|
182
|
+
return None
|
|
183
|
+
subdirs = list(_KIND_SUBDIRS.get(kind, []))
|
|
184
|
+
for group in _KIND_SUBDIRS.values():
|
|
185
|
+
subdirs.extend(group)
|
|
186
|
+
subdirs.append(".")
|
|
187
|
+
seen = set()
|
|
188
|
+
for sub in subdirs:
|
|
189
|
+
cand = os.path.join(MODELS_DIR, sub, name)
|
|
190
|
+
if cand in seen:
|
|
191
|
+
continue
|
|
192
|
+
seen.add(cand)
|
|
193
|
+
if os.path.exists(cand):
|
|
194
|
+
return cand
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _sha256_file(path: str) -> str:
|
|
199
|
+
h = hashlib.sha256()
|
|
200
|
+
with open(path, "rb") as f:
|
|
201
|
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
202
|
+
h.update(chunk)
|
|
203
|
+
return "sha256:" + h.hexdigest()
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _load_hash_cache() -> dict:
|
|
207
|
+
try:
|
|
208
|
+
with open(HASH_CACHE_FILE, "r", encoding="utf-8") as f:
|
|
209
|
+
return json.load(f)
|
|
210
|
+
except Exception:
|
|
211
|
+
return {}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _save_hash_cache(cache: dict) -> None:
|
|
215
|
+
try:
|
|
216
|
+
with open(HASH_CACHE_FILE, "w", encoding="utf-8") as f:
|
|
217
|
+
json.dump(cache, f, indent=2)
|
|
218
|
+
except Exception:
|
|
219
|
+
pass
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _hash_model_file(name: str, kind: str, do_hash: bool, cache: dict) -> dict:
|
|
223
|
+
"""Best-effort {name, size_bytes, sha256} for a model/LoRA file (mirror of the
|
|
224
|
+
JS hashModelFile). size_bytes is always filled when resolvable; sha256 is
|
|
225
|
+
gated behind do_hash and cached by (name, size, mtime)."""
|
|
226
|
+
if not name:
|
|
227
|
+
return {"name": name, "size_bytes": None, "sha256": None, "hash_note": "no filename"}
|
|
228
|
+
resolved = _resolve_model_path(name, kind)
|
|
229
|
+
if not resolved:
|
|
230
|
+
return {"name": name, "size_bytes": None, "sha256": None, "hash_note": f"unresolved under {MODELS_DIR}"}
|
|
231
|
+
try:
|
|
232
|
+
st = os.stat(resolved)
|
|
233
|
+
size_bytes = st.st_size
|
|
234
|
+
mtime_ms = round(st.st_mtime * 1000)
|
|
235
|
+
except OSError:
|
|
236
|
+
return {"name": name, "size_bytes": None, "sha256": None, "hash_note": "stat failed"}
|
|
237
|
+
if not do_hash:
|
|
238
|
+
return {"name": name, "size_bytes": size_bytes, "sha256": None, "hash_note": "hash skipped (pass --hash-models)"}
|
|
239
|
+
key = f"{name}:{size_bytes}:{mtime_ms}"
|
|
240
|
+
if key in cache:
|
|
241
|
+
return {"name": name, "size_bytes": size_bytes, "sha256": cache[key]}
|
|
242
|
+
try:
|
|
243
|
+
sha = _sha256_file(resolved)
|
|
244
|
+
except OSError:
|
|
245
|
+
return {"name": name, "size_bytes": size_bytes, "sha256": None, "hash_note": "hash failed (read error)"}
|
|
246
|
+
cache[key] = sha
|
|
247
|
+
return {"name": name, "size_bytes": size_bytes, "sha256": sha}
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _build_pinning(graph, models, loras, seed_policy, do_hash, cache) -> dict:
|
|
251
|
+
"""Assemble the pinning block — identical shape to lib/run-manifest.js buildPinning."""
|
|
252
|
+
model_hashes = {}
|
|
253
|
+
for role, kind in (("unet", "unet"), ("clip", "clip"), ("vae", "vae"), ("checkpoint", "checkpoint")):
|
|
254
|
+
if models.get(role):
|
|
255
|
+
model_hashes[role] = _hash_model_file(models[role], kind, do_hash, cache)
|
|
256
|
+
lora_hashes = []
|
|
257
|
+
for lora in loras or []:
|
|
258
|
+
entry = _hash_model_file(lora.get("name"), "lora", do_hash, cache)
|
|
259
|
+
entry["weight"] = _js_nullish(lora.get("weight"), 1.0) # SDL-H4
|
|
260
|
+
lora_hashes.append(entry)
|
|
261
|
+
return {
|
|
262
|
+
"pinning_version": PINNING_VERSION,
|
|
263
|
+
"comfy_workflow_sha": _comfy_workflow_sha(graph),
|
|
264
|
+
"seed_policy": seed_policy,
|
|
265
|
+
"models": model_hashes,
|
|
266
|
+
"loras": lora_hashes,
|
|
267
|
+
}
|
|
268
|
+
|
|
34
269
|
|
|
35
270
|
def qwen_graph(pos: str, neg: str, seed: int, w: int, h: int, steps: int, cfg: float, prefix: str,
|
|
36
|
-
loras: list | None = None
|
|
271
|
+
loras: list | None = None, sampler: str = "euler", scheduler: str = "simple",
|
|
272
|
+
shift: float = 3.1) -> dict:
|
|
37
273
|
"""loras: [{name, weight}] chained as DiT-only LoraLoaderModelOnly nodes (ids from
|
|
38
274
|
"40", between UNETLoader and ModelSamplingAuraFlow) — mirrors sdlab buildQwenGraph.
|
|
39
275
|
Model-only because studio Qwen style LoRAs train the transformer with the text
|
|
40
|
-
encoder frozen.
|
|
276
|
+
encoder frozen.
|
|
277
|
+
|
|
278
|
+
SDL-H11: sampler/scheduler/shift used to be hardcoded literals here (always
|
|
279
|
+
"euler"/"simple"/3.1, regardless of what the wave declared) and were not
|
|
280
|
+
even accepted as parameters — so a wave asking for dpmpp_2m/karras/shift 5.0
|
|
281
|
+
silently generated with euler/simple/3.1 instead, produced a different
|
|
282
|
+
comfy_workflow_sha than the JS `sdlab generate` path for the "same" graph,
|
|
283
|
+
AND had those wrong values stamped into the provenance receipt. Defaults
|
|
284
|
+
here match QWEN_DEFAULTS in lib/adapters/comfyui-workflows.js — used only
|
|
285
|
+
when a wave omits the fields (mirrors JS's `rp.sampler || defaults.sampler
|
|
286
|
+
|| QWEN_DEFAULTS.sampler` fallback chain)."""
|
|
41
287
|
nodes = {
|
|
42
288
|
"37": {"class_type": "UNETLoader", "inputs": {"unet_name": QWEN_UNET, "weight_dtype": "default"}},
|
|
43
289
|
"38": {"class_type": "CLIPLoader", "inputs": {"clip_name": QWEN_CLIP, "type": "qwen_image"}},
|
|
@@ -49,13 +295,13 @@ def qwen_graph(pos: str, neg: str, seed: int, w: int, h: int, steps: int, cfg: f
|
|
|
49
295
|
for i, lora in enumerate(loras or []):
|
|
50
296
|
lid = str(40 + i)
|
|
51
297
|
nodes[lid] = {"class_type": "LoraLoaderModelOnly", "inputs": {
|
|
52
|
-
"lora_name": lora["name"], "strength_model": lora.get("weight", 1.0), "model": model_out}}
|
|
298
|
+
"lora_name": lora["name"], "strength_model": _js_nullish(lora.get("weight"), 1.0), "model": model_out}} # SDL-H4
|
|
53
299
|
model_out = [lid, 0]
|
|
54
300
|
nodes.update({
|
|
55
|
-
"66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift":
|
|
301
|
+
"66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift": shift, "model": model_out}},
|
|
56
302
|
"5": {"class_type": "EmptySD3LatentImage", "inputs": {"width": w, "height": h, "batch_size": 1}},
|
|
57
|
-
"3": {"class_type": "KSampler", "inputs": {"seed": seed, "steps": steps, "cfg": cfg, "sampler_name":
|
|
58
|
-
"scheduler":
|
|
303
|
+
"3": {"class_type": "KSampler", "inputs": {"seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler,
|
|
304
|
+
"scheduler": scheduler, "denoise": 1.0, "model": ["66", 0], "positive": ["6", 0], "negative": ["7", 0], "latent_image": ["5", 0]}},
|
|
59
305
|
"8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
|
|
60
306
|
"9": {"class_type": "SaveImage", "inputs": {"filename_prefix": prefix, "images": ["8", 0]}},
|
|
61
307
|
})
|
|
@@ -68,7 +314,9 @@ def _post(path: str, payload: dict):
|
|
|
68
314
|
|
|
69
315
|
|
|
70
316
|
def generate(pos: str, neg: str, seed: int, dst: Path, **kw) -> Path:
|
|
71
|
-
g = qwen_graph(pos, neg, seed, kw["w"], kw["h"], kw["steps"], kw["cfg"], dst.stem, loras=kw.get("loras")
|
|
317
|
+
g = qwen_graph(pos, neg, seed, kw["w"], kw["h"], kw["steps"], kw["cfg"], dst.stem, loras=kw.get("loras"),
|
|
318
|
+
sampler=kw.get("sampler", "euler"), scheduler=kw.get("scheduler", "simple"),
|
|
319
|
+
shift=kw.get("shift", 3.1))
|
|
72
320
|
pid = _post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
|
|
73
321
|
for _ in range(1200):
|
|
74
322
|
hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
|
|
@@ -89,6 +337,8 @@ def main() -> None:
|
|
|
89
337
|
ap.add_argument("--wave", required=True, type=Path)
|
|
90
338
|
ap.add_argument("--out", required=True, type=Path)
|
|
91
339
|
ap.add_argument("--variations", type=int, default=None, help="override per-subject variation count")
|
|
340
|
+
ap.add_argument("--hash-models", action="store_true",
|
|
341
|
+
help="content-hash the unet/clip/vae/LoRA files into the pinning block (best-effort, cached)")
|
|
92
342
|
args = ap.parse_args()
|
|
93
343
|
|
|
94
344
|
wave = json.loads(args.wave.read_text(encoding="utf-8"))
|
|
@@ -99,25 +349,96 @@ def main() -> None:
|
|
|
99
349
|
steps, cfg = d.get("steps", 22), d.get("cfg", 3.5)
|
|
100
350
|
base_seed = d.get("base_seed", 1000)
|
|
101
351
|
loras = d.get("loras", [])
|
|
352
|
+
# SDL-H11: read sampler/scheduler/shift from the wave instead of hardcoding
|
|
353
|
+
# them — defaults match QWEN_DEFAULTS in lib/adapters/comfyui-workflows.js
|
|
354
|
+
# so an omitted field falls back to the same value the JS path would use.
|
|
355
|
+
sampler = d.get("sampler", "euler")
|
|
356
|
+
scheduler = d.get("scheduler", "simple")
|
|
357
|
+
shift = d.get("shift", 3.1)
|
|
102
358
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
103
359
|
|
|
104
360
|
receipt = {"wave": wave.get("wave"), "base": "qwen-image", "unet": QWEN_UNET, "clip": QWEN_CLIP,
|
|
105
|
-
"vae": QWEN_VAE, "sampler":
|
|
361
|
+
"vae": QWEN_VAE, "sampler": sampler, "scheduler": scheduler, "shift": shift,
|
|
106
362
|
"steps": steps, "cfg": cfg, "size": [w, h], "style_prefix": prefix, "negative": neg,
|
|
107
363
|
"loras": loras, "items": []}
|
|
364
|
+
|
|
365
|
+
# PIN_PER_STEP: pin the exact graph + model/LoRA content identity + seed policy.
|
|
366
|
+
# The wave shares ONE comfy_workflow_sha (per-item seed + prompt are normalized
|
|
367
|
+
# out and recorded per item in items[]). seed_policy is "explicit-per-item"
|
|
368
|
+
# because each item's seed is written verbatim below. Mirror of the JS contract.
|
|
369
|
+
hash_cache = _load_hash_cache()
|
|
370
|
+
pin_graph = qwen_graph(prefix or "x", neg, base_seed, w, h, steps, cfg, "sdl", loras=loras,
|
|
371
|
+
sampler=sampler, scheduler=scheduler, shift=shift)
|
|
372
|
+
receipt["pinning"] = _build_pinning(
|
|
373
|
+
pin_graph,
|
|
374
|
+
{"unet": QWEN_UNET, "clip": QWEN_CLIP, "vae": QWEN_VAE},
|
|
375
|
+
loras,
|
|
376
|
+
"explicit-per-item",
|
|
377
|
+
args.hash_models,
|
|
378
|
+
hash_cache,
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
receipt_path = args.out / "generation.json"
|
|
382
|
+
|
|
383
|
+
def _save_receipt() -> None:
|
|
384
|
+
# M3: incremental checkpoint (mirrors lib/batch-runs.js's per-slot
|
|
385
|
+
# atomicWriteJson pattern) — written after EVERY item, not once at
|
|
386
|
+
# the end. Before this fix, the subject loop called generate() with
|
|
387
|
+
# no try/except (it can raise the named ComfyUI timeout at its
|
|
388
|
+
# RuntimeError above), and generation.json was written exactly once,
|
|
389
|
+
# after the loop finished. A 200-image wave failing at item 130 left
|
|
390
|
+
# 129 rendered PNGs on disk with NO generation.json at all — the
|
|
391
|
+
# PIN_PER_STEP provenance this file's own docstring promises, fully
|
|
392
|
+
# lost, for every item including the ones that succeeded.
|
|
393
|
+
receipt_path.write_text(json.dumps(receipt, indent=2), encoding="utf-8")
|
|
394
|
+
|
|
108
395
|
seed = base_seed
|
|
396
|
+
error_count = 0
|
|
109
397
|
for subj in wave["subjects"]:
|
|
110
398
|
nvar = args.variations if args.variations is not None else subj.get("variations", 1)
|
|
111
399
|
pos = f"{prefix}, {subj['prompt']}" if prefix else subj["prompt"]
|
|
112
400
|
for v in range(nvar):
|
|
113
401
|
dst = args.out / f"{subj['id']}_v{v}.png"
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
402
|
+
item = {"id": subj["id"], "variation": v, "seed": seed, "file": dst.name, "prompt": pos}
|
|
403
|
+
try:
|
|
404
|
+
generate(pos, neg, seed, dst, w=w, h=h, steps=steps, cfg=cfg, loras=loras,
|
|
405
|
+
sampler=sampler, scheduler=scheduler, shift=shift)
|
|
406
|
+
except Exception as exc:
|
|
407
|
+
# A single item's failure no longer takes the whole wave's
|
|
408
|
+
# provenance down with it — record the failure and keep
|
|
409
|
+
# going. The receipt is checkpointed below regardless of
|
|
410
|
+
# per-item outcome.
|
|
411
|
+
item["status"] = "error"
|
|
412
|
+
item["error"] = str(exc)
|
|
413
|
+
error_count += 1
|
|
414
|
+
print(f" {subj['id']} v{v} (seed {seed}) -> ERROR: {exc}", flush=True)
|
|
415
|
+
else:
|
|
416
|
+
item["status"] = "ok"
|
|
417
|
+
print(f" {subj['id']} v{v} (seed {seed}) -> {dst.name}", flush=True)
|
|
418
|
+
receipt["items"].append(item)
|
|
117
419
|
seed += 1
|
|
420
|
+
_save_receipt()
|
|
421
|
+
|
|
422
|
+
if args.hash_models:
|
|
423
|
+
_save_hash_cache(hash_cache)
|
|
424
|
+
_save_receipt()
|
|
425
|
+
ok_count = len(receipt["items"]) - error_count
|
|
426
|
+
# Plain ASCII only (no em-dash) — matches the rest of this file's
|
|
427
|
+
# runtime print()/error strings. A Windows console not running in UTF-8
|
|
428
|
+
# mode mangles non-ASCII on stdout; the docstring/comments can afford
|
|
429
|
+
# Unicode since those never reach a terminal at runtime.
|
|
430
|
+
summary = f"\n{ok_count}/{len(receipt['items'])} images -> {args.out} (+ generation.json provenance)"
|
|
431
|
+
if error_count:
|
|
432
|
+
summary += f" ({error_count} error(s) - see generation.json for per-item detail)"
|
|
433
|
+
print(summary)
|
|
118
434
|
|
|
119
|
-
|
|
120
|
-
|
|
435
|
+
# Mirror scripts/generate.js's RUNTIME_ALL_FAILED convention: a run where
|
|
436
|
+
# EVERY item failed exits non-zero (nothing to salvage); a partial
|
|
437
|
+
# failure still exits 0 — the per-item errors are visible above and in
|
|
438
|
+
# the checkpointed receipt, and individual item failure is tolerated
|
|
439
|
+
# everywhere else in this pipeline.
|
|
440
|
+
if receipt["items"] and error_count == len(receipt["items"]):
|
|
441
|
+
raise SystemExit(f"all {error_count} generation attempt(s) failed - see generation.json for per-item errors")
|
|
121
442
|
|
|
122
443
|
|
|
123
444
|
if __name__ == "__main__":
|
package/scripts/reingest.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { join } from 'node:path';
|
|
13
13
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
14
|
-
import { REPO_ROOT, resolveSafeProjectPath } from '../lib/paths.js';
|
|
14
|
+
import { REPO_ROOT, getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
|
|
15
15
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
16
16
|
import { reingestGenerated, auditReingest } from '../lib/reingest.js';
|
|
17
17
|
|
|
@@ -29,7 +29,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
29
29
|
});
|
|
30
30
|
|
|
31
31
|
const projectName = flags.project || getProjectName(argv);
|
|
32
|
-
const projectRoot =
|
|
32
|
+
const projectRoot = getProjectRoot(projectName);
|
|
33
33
|
const subcommand = positionals[0] || 'audit';
|
|
34
34
|
|
|
35
35
|
if (subcommand === 'generated') {
|
package/scripts/run-generate.js
CHANGED
|
@@ -24,6 +24,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
24
24
|
brief: { type: 'string' },
|
|
25
25
|
seed: { type: 'string' },
|
|
26
26
|
'dry-run': { type: 'boolean' },
|
|
27
|
+
'hash-models': { type: 'boolean' },
|
|
27
28
|
json: { type: 'boolean' },
|
|
28
29
|
},
|
|
29
30
|
deprecated: { game: 'project' },
|
|
@@ -36,6 +37,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
36
37
|
console.log(' --brief <id> Compiled brief ID (required)');
|
|
37
38
|
console.log(' --seed <n> Override base seed');
|
|
38
39
|
console.log(' --dry-run Prepare run dir without submitting to ComfyUI');
|
|
40
|
+
console.log(' --hash-models Content-hash the model/LoRA files into the run manifest (best-effort, cached)');
|
|
39
41
|
console.log(' --json Output manifest as JSON');
|
|
40
42
|
return;
|
|
41
43
|
}
|
|
@@ -59,6 +61,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
59
61
|
brief,
|
|
60
62
|
baseSeed: flags.seed ? parseNumberFlag('seed', flags.seed, { int: true, min: 0 }) : undefined,
|
|
61
63
|
dryRun: flags['dry-run'] || false,
|
|
64
|
+
hashModels: flags['hash-models'] || false,
|
|
62
65
|
});
|
|
63
66
|
|
|
64
67
|
// Save summary
|
package/scripts/sheet.js
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* sheet.js — Render an HTML contact sheet over a directory of images.
|
|
5
|
+
*
|
|
6
|
+
* No batch manifest required. Points the SAME HTML renderer that powers
|
|
7
|
+
* `sdlab batch sheet` (lib/batch-sheet-render.js, via lib/candidate-sheet.js)
|
|
8
|
+
* at an arbitrary directory under a project — outputs/candidates/,
|
|
9
|
+
* outputs/approved/, outputs/, whatever — instead of requiring a
|
|
10
|
+
* batches/<id>/manifest.json that only `sdlab batch generate` can produce.
|
|
11
|
+
*
|
|
12
|
+
* Each image is matched to its record by asset_path (when a record exists)
|
|
13
|
+
* for id / curation status / prompt. Images with no matching record
|
|
14
|
+
* (pre-ingest — generated but not yet re-ingested) still render, labeled by
|
|
15
|
+
* filename instead of crashing or being skipped.
|
|
16
|
+
*
|
|
17
|
+
* Usage:
|
|
18
|
+
* sdlab sheet [<dir>] --project <name> [--out <path>] [--dry-run]
|
|
19
|
+
*
|
|
20
|
+
* Examples:
|
|
21
|
+
* sdlab sheet --project star-freight # outputs/candidates (default)
|
|
22
|
+
* sdlab sheet outputs/approved --project star-freight
|
|
23
|
+
* sdlab sheet outputs --project star-freight --dry-run # whole tree, no write
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { existsSync } from 'node:fs';
|
|
27
|
+
import { relative, dirname } from 'node:path';
|
|
28
|
+
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
29
|
+
import { getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
|
|
30
|
+
import { inputError, handleCliError } from '../lib/errors.js';
|
|
31
|
+
import { result } from '../lib/log.js';
|
|
32
|
+
import {
|
|
33
|
+
buildCandidateSheetItems,
|
|
34
|
+
renderCandidateSheetHTML,
|
|
35
|
+
saveCandidateSheet,
|
|
36
|
+
defaultSheetOutputPath,
|
|
37
|
+
} from '../lib/candidate-sheet.js';
|
|
38
|
+
|
|
39
|
+
const DEFAULT_DIR = 'outputs/candidates';
|
|
40
|
+
|
|
41
|
+
function posix(p) {
|
|
42
|
+
return p.replace(/\\/g, '/');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function run(argv = process.argv.slice(2)) {
|
|
46
|
+
// Checked against the raw argv, not via parseArgs' flag defs: parseArgs
|
|
47
|
+
// only recognizes "--"-prefixed flags (see lib/args.js), so a bare "-h"
|
|
48
|
+
// would otherwise fall through to the positional `dir` slot instead of
|
|
49
|
+
// showing help. Mirrors bin/sdlab.js's own argvHasHelp() check.
|
|
50
|
+
if (argv.includes('--help') || argv.includes('-h')) {
|
|
51
|
+
printUsage();
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const { flags, positionals } = parseArgs(argv, {
|
|
56
|
+
flags: {
|
|
57
|
+
project: { type: 'string' },
|
|
58
|
+
out: { type: 'string' },
|
|
59
|
+
'dry-run': { type: 'boolean', default: false },
|
|
60
|
+
},
|
|
61
|
+
positionals: ['dir'],
|
|
62
|
+
deprecated: { game: 'project' },
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
const projectName = flags.project || getProjectName(argv);
|
|
66
|
+
const projectRoot = getProjectRoot(projectName);
|
|
67
|
+
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
68
|
+
|
|
69
|
+
const dirArg = positionals[0] || DEFAULT_DIR;
|
|
70
|
+
// Containment check — a user-supplied path must never be joined by hand.
|
|
71
|
+
const safeDir = resolveSafeProjectPath(projectRoot, dirArg, { flagName: 'dir' });
|
|
72
|
+
const relDir = posix(relative(projectRoot, safeDir)) || '.';
|
|
73
|
+
|
|
74
|
+
if (!existsSync(safeDir)) {
|
|
75
|
+
throw inputError(
|
|
76
|
+
'INPUT_DIR_NOT_FOUND',
|
|
77
|
+
`Directory not found: ${relDir} (looked under ${projectRoot})`,
|
|
78
|
+
'Try outputs/candidates, outputs/approved, outputs/rejected, outputs/borderline, or outputs (the whole tree).'
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const items = await buildCandidateSheetItems(projectRoot, safeDir);
|
|
83
|
+
|
|
84
|
+
console.log(`\x1b[1mstyle-dataset-lab\x1b[0m sheet — ${projectName}`);
|
|
85
|
+
console.log(` Source: ${relDir}`);
|
|
86
|
+
|
|
87
|
+
if (items.length === 0) {
|
|
88
|
+
console.log(' No images found — nothing to render.');
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const matched = items.filter((i) => i.hasRecord).length;
|
|
93
|
+
console.log(` Images: ${items.length} (${matched} with records, ${items.length - matched} pre-ingest)`);
|
|
94
|
+
|
|
95
|
+
const outRel = flags.out
|
|
96
|
+
? flags.out
|
|
97
|
+
: posix(relative(projectRoot, defaultSheetOutputPath(projectRoot, relDir)));
|
|
98
|
+
const safeOut = resolveSafeProjectPath(projectRoot, outRel, { flagName: 'out' });
|
|
99
|
+
|
|
100
|
+
if (dryRun) {
|
|
101
|
+
console.log(` \x1b[33m(dry-run)\x1b[0m would write: ${posix(relative(projectRoot, safeOut))}`);
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const html = renderCandidateSheetHTML({
|
|
106
|
+
items,
|
|
107
|
+
title: `Contact sheet — ${projectName} — ${relDir}`,
|
|
108
|
+
sourceLabel: relDir,
|
|
109
|
+
sheetOutputDir: dirname(safeOut),
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
const written = await saveCandidateSheet(safeOut, html);
|
|
113
|
+
result('sheet', posix(relative(projectRoot, written)));
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function printUsage() {
|
|
117
|
+
console.log('Usage: sdlab sheet [<dir>] --project <name> [--out <path>] [--dry-run]');
|
|
118
|
+
console.log('');
|
|
119
|
+
console.log('Render an HTML contact sheet over a directory of images — no batch');
|
|
120
|
+
console.log('manifest required. Matches each image to its record (by asset_path)');
|
|
121
|
+
console.log('when one exists; degrades to filename when it does not (pre-ingest).');
|
|
122
|
+
console.log('');
|
|
123
|
+
console.log('Positional:');
|
|
124
|
+
console.log(' dir Directory to sheet, relative to the project root');
|
|
125
|
+
console.log(` (default: ${DEFAULT_DIR})`);
|
|
126
|
+
console.log('');
|
|
127
|
+
console.log('Flags:');
|
|
128
|
+
console.log(' --project <name> Project to operate on');
|
|
129
|
+
console.log(' --out <path> Output HTML path, relative to project root');
|
|
130
|
+
console.log(' (default: outputs/sheets/<slugified-dir>.html)');
|
|
131
|
+
console.log(' --dry-run Report what would be written without writing it');
|
|
132
|
+
console.log('');
|
|
133
|
+
console.log('Examples:');
|
|
134
|
+
console.log(' sdlab sheet --project star-freight');
|
|
135
|
+
console.log(' sdlab sheet outputs/approved --project star-freight');
|
|
136
|
+
console.log(' sdlab sheet outputs --project star-freight --dry-run');
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Direct execution guard — same pattern as every other script in this repo.
|
|
140
|
+
if (process.argv[1] && (process.argv[1].endsWith('sheet.js') || process.argv[1].endsWith('sheet'))) {
|
|
141
|
+
run().catch(handleCliError);
|
|
142
|
+
}
|
package/scripts/snapshot.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { readFile } from 'node:fs/promises';
|
|
14
14
|
import { join } from 'node:path';
|
|
15
15
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
16
|
-
import {
|
|
16
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
17
17
|
import { loadSelectionProfile } from '../lib/config.js';
|
|
18
18
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
19
19
|
import { createSnapshot, loadSnapshot, listSnapshots, diffSnapshots } from '../lib/snapshot.js';
|
|
@@ -30,7 +30,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
const projectName = flags.project || getProjectName(argv);
|
|
33
|
-
const projectRoot =
|
|
33
|
+
const projectRoot = getProjectRoot(projectName);
|
|
34
34
|
const subcommand = positionals[0] || 'list';
|
|
35
35
|
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
36
36
|
|
package/scripts/split.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
15
|
-
import {
|
|
15
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
16
16
|
import { loadSplitProfile } from '../lib/config.js';
|
|
17
17
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
18
18
|
import { listSnapshots } from '../lib/snapshot.js';
|
|
@@ -30,7 +30,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
const projectName = flags.project || getProjectName(argv);
|
|
33
|
-
const projectRoot =
|
|
33
|
+
const projectRoot = getProjectRoot(projectName);
|
|
34
34
|
const subcommand = positionals[0] || 'list';
|
|
35
35
|
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
36
36
|
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { join } from 'node:path';
|
|
14
14
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
15
|
-
import {
|
|
15
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
16
16
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
17
17
|
import { listExports } from '../lib/export.js';
|
|
18
18
|
import {
|
|
@@ -35,7 +35,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
35
35
|
});
|
|
36
36
|
|
|
37
37
|
const projectName = flags.project || getProjectName(argv);
|
|
38
|
-
const projectRoot =
|
|
38
|
+
const projectRoot = getProjectRoot(projectName);
|
|
39
39
|
const subcommand = positionals.find(a => !a.startsWith('tm-')) || 'list';
|
|
40
40
|
|
|
41
41
|
if (subcommand === 'create') {
|