@mcptoolshop/style-dataset-lab 3.2.0 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/CHANGELOG.md +58 -1
  2. package/README.es.md +40 -19
  3. package/README.fr.md +74 -53
  4. package/README.hi.md +67 -46
  5. package/README.it.md +42 -21
  6. package/README.ja.md +44 -23
  7. package/README.md +25 -4
  8. package/README.pt-BR.md +42 -21
  9. package/README.zh.md +66 -45
  10. package/bin/sdlab.js +51 -4
  11. package/lib/adapters/ai-toolkit.js +37 -5
  12. package/lib/adapters/comfyui-runner.js +114 -27
  13. package/lib/adapters/comfyui-workflows.js +10 -2
  14. package/lib/adapters/diffusers-lora.js +37 -5
  15. package/lib/adapters/generic-image-caption.js +43 -5
  16. package/lib/args.js +16 -1
  17. package/lib/batch-modes.js +29 -8
  18. package/lib/batch-runs.js +29 -1
  19. package/lib/batch-sheet-render.js +13 -3
  20. package/lib/brief-compiler.js +19 -0
  21. package/lib/candidate-sheet.js +327 -0
  22. package/lib/canon-build/build.js +146 -93
  23. package/lib/captions.js +57 -0
  24. package/lib/card.js +74 -2
  25. package/lib/comfyui.js +116 -18
  26. package/lib/config.js +116 -12
  27. package/lib/critique-engine.js +82 -3
  28. package/lib/critique-render.js +45 -0
  29. package/lib/eligibility.js +121 -9
  30. package/lib/eval-pack.js +14 -14
  31. package/lib/eval-runs.js +33 -0
  32. package/lib/export.js +122 -34
  33. package/lib/freeze-gate.js +69 -16
  34. package/lib/freeze-stamp.js +28 -3
  35. package/lib/generated-provenance.js +16 -1
  36. package/lib/implementation-packs.js +95 -17
  37. package/lib/ingest.js +396 -0
  38. package/lib/judgment-provenance.js +220 -0
  39. package/lib/measure.js +422 -0
  40. package/lib/paths.js +196 -7
  41. package/lib/reingest-selected.js +98 -16
  42. package/lib/run-manifest.js +394 -0
  43. package/lib/run-summary.js +12 -6
  44. package/lib/runtime-runs.js +45 -4
  45. package/lib/snapshot.js +121 -16
  46. package/lib/split.js +364 -48
  47. package/lib/training-manifests.js +114 -21
  48. package/lib/training-packages.js +147 -15
  49. package/lib/verifier.js +67 -0
  50. package/lib/workflow-profiles.js +24 -3
  51. package/package.json +3 -2
  52. package/runtime/comfyui/txt2img-qwen.json +38 -0
  53. package/runtime/comfyui/txt2img-standard.json +21 -0
  54. package/scripts/_resume.js +62 -0
  55. package/scripts/_shared-args.js +44 -0
  56. package/scripts/brief-compile.js +2 -1
  57. package/scripts/brief-show.js +2 -1
  58. package/scripts/canon-bind.js +2 -2
  59. package/scripts/canon-build.js +27 -20
  60. package/scripts/canon-drift.js +16 -7
  61. package/scripts/canon-freeze-status.js +14 -6
  62. package/scripts/canon-freeze.js +27 -11
  63. package/scripts/canon-unfreeze.js +17 -9
  64. package/scripts/card.js +2 -2
  65. package/scripts/compare.js +2 -2
  66. package/scripts/critique.js +48 -8
  67. package/scripts/curate.js +23 -2
  68. package/scripts/doctor.js +181 -55
  69. package/scripts/eligibility.js +42 -2
  70. package/scripts/eval-pack.js +2 -2
  71. package/scripts/eval-run.js +2 -2
  72. package/scripts/export.js +2 -2
  73. package/scripts/generate-controlnet.js +45 -25
  74. package/scripts/generate-identity.js +53 -23
  75. package/scripts/generate-ipadapter.js +44 -23
  76. package/scripts/generate.js +72 -5
  77. package/scripts/implementation-pack.js +2 -2
  78. package/scripts/ingest.js +145 -0
  79. package/scripts/init.js +73 -13
  80. package/scripts/measure.js +170 -0
  81. package/scripts/measure_image.py +458 -0
  82. package/scripts/migrate-records.js +142 -0
  83. package/scripts/painterly-test.js +4 -3
  84. package/scripts/painterly.js +7 -3
  85. package/scripts/qwen_generate.py +334 -13
  86. package/scripts/reingest.js +2 -2
  87. package/scripts/run-generate.js +3 -0
  88. package/scripts/sheet.js +142 -0
  89. package/scripts/snapshot.js +2 -2
  90. package/scripts/split.js +2 -2
  91. package/scripts/training-manifest.js +2 -2
  92. package/scripts/training-package.js +2 -2
  93. package/scripts/training-profile.js +2 -2
  94. package/scripts/workflow-list.js +2 -1
  95. package/scripts/workflow-show.js +2 -1
@@ -15,10 +15,14 @@ embedded python (has PIL): start the watchdog (E:/AI/training/_watchdog.ps1) bef
15
15
  from __future__ import annotations
16
16
 
17
17
  import argparse
18
+ import hashlib
18
19
  import json
20
+ import math
21
+ import os
19
22
  import time
20
23
  import urllib.request
21
24
  import uuid
25
+ from decimal import Decimal
22
26
  from pathlib import Path
23
27
 
24
28
  from PIL import Image
@@ -31,13 +35,255 @@ QWEN_UNET = "qwen_image_fp8_e4m3fn.safetensors"
31
35
  QWEN_CLIP = "qwen_2.5_vl_7b_fp8_scaled.safetensors"
32
36
  QWEN_VAE = "qwen_image_vae.safetensors"
33
37
 
38
+ # ─── PIN_PER_STEP pinning contract ────────────────────────────────────────────
39
+ # Mirror of lib/run-manifest.js (the JS helper is the field-name authority). This
40
+ # receipt carries the IDENTICAL `pinning` block so a wave produced by this bridge
41
+ # generator is pinned the same way as one produced through the JS `sdlab generate`
42
+ # / `sdlab run generate` paths. Keep PINNING_VERSION + the field names in lockstep
43
+ # with lib/run-manifest.js.
44
+ PINNING_VERSION = "1.0.0"
45
+ MODELS_DIR = os.environ.get(
46
+ "SDLAB_MODELS_DIR", r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/models"
47
+ )
48
+ HASH_CACHE_FILE = os.environ.get("SDLAB_HASH_CACHE") or str(
49
+ Path(MODELS_DIR) / ".sdlab-hash-cache.json"
50
+ )
51
+ # kind → candidate subdirs under the models dir, in search order (mirror of JS).
52
+ _KIND_SUBDIRS = {
53
+ "unet": ["diffusion_models", "unet"],
54
+ "clip": ["text_encoders", "clip"],
55
+ "vae": ["vae"],
56
+ "checkpoint": ["checkpoints"],
57
+ "lora": ["loras"],
58
+ }
59
+
60
+
61
+ def _js_number(n) -> str:
62
+ """Format a number exactly the way JS Number::toString (ECMA-262
63
+ 6.1.6.1.20) does, so the graph hash matches the JS helper
64
+ (lib/run-manifest.js stableStringify).
65
+
66
+ SDL-H3: the previous implementation delegated non-integer floats to
67
+ Python's repr(), which disagrees with JS in the [1e-6, 1e-4) magnitude
68
+ band and on exponent zero-padding:
69
+ 0.00005 -> Python repr "5e-05" vs JS "0.00005"
70
+ 0.000001 -> Python repr "1e-06" vs JS "0.000001"
71
+ 9.9e-7 -> Python repr "9.9e-07" vs JS "9.9e-7"
72
+ Any wave with a LoRA weight or float param in that band produced a
73
+ different comfy_workflow_sha than the JS path for an identical graph.
74
+
75
+ Fix: Python's repr()/str() already compute the same "shortest
76
+ round-trip decimal digit string" JS engines use internally (both use a
77
+ Grisu/Ryu-class shortest-repr algorithm) — the two runtimes only
78
+ disagree on how that digit string is laid out (decimal point position,
79
+ when to switch to exponential, exponent zero-padding). So: borrow
80
+ Python's digit string via repr(), hand it to Decimal (which parses
81
+ repr()'s scientific-notation form natively, no manual string
82
+ splitting), then re-render per the ECMA spec's own branches.
83
+ """
84
+ if isinstance(n, bool):
85
+ return "true" if n else "false"
86
+ if isinstance(n, int):
87
+ return str(n)
88
+ if isinstance(n, float):
89
+ if math.isnan(n) or math.isinf(n):
90
+ return "null"
91
+ if n == 0:
92
+ return "0" # covers -0.0 too — JS String(-0) is "0"
93
+ sign = "-" if n < 0 else ""
94
+ _, digits_tuple, exponent = Decimal(repr(abs(n))).as_tuple()
95
+ m = len(digits_tuple)
96
+ # value == 0.<all m digits> * 10**point_pos (this is ECMA's "n")
97
+ point_pos = exponent + m
98
+ digits = "".join(map(str, digits_tuple)).rstrip("0") or "0"
99
+ k = len(digits)
100
+ if k <= point_pos <= 21:
101
+ return sign + digits + ("0" * (point_pos - k))
102
+ if 0 < point_pos <= 21:
103
+ return sign + digits[:point_pos] + "." + digits[point_pos:]
104
+ if -6 < point_pos <= 0:
105
+ return sign + "0." + ("0" * -point_pos) + digits
106
+ mantissa = digits[0] if k == 1 else digits[0] + "." + digits[1:]
107
+ e = point_pos - 1
108
+ esign = "+" if e >= 0 else "-"
109
+ return f"{sign}{mantissa}e{esign}{abs(e)}"
110
+ return "null"
111
+
112
+
113
+ def _js_nullish(value, default):
114
+ """Mirror JS `value ?? default` — substitutes only when value is None
115
+ (JS null/undefined), never for other falsy values like 0 or False.
116
+
117
+ SDL-H4: dict.get(key, default) is NOT equivalent — it only substitutes
118
+ when the KEY is missing, never when the key is present with an
119
+ explicit null. {"weight": None}.get("weight", 1.0) returns None, not
120
+ 1.0, silently splitting the graph hash from the JS side's
121
+ `l.weight ?? 1.0` AND submitting a literal null strength_model to
122
+ ComfyUI's /prompt.
123
+ """
124
+ return default if value is None else value
125
+
126
+
127
+ def _stable_stringify(value) -> str:
128
+ """Deterministic serialization mirroring lib/run-manifest.js stableStringify:
129
+ object keys sorted, arrays in order, JS-style number/string formatting."""
130
+ if value is None:
131
+ return "null"
132
+ if isinstance(value, bool):
133
+ return "true" if value else "false"
134
+ if isinstance(value, (int, float)):
135
+ return _js_number(value)
136
+ if isinstance(value, str):
137
+ return json.dumps(value, ensure_ascii=False)
138
+ if isinstance(value, (list, tuple)):
139
+ return "[" + ",".join(_stable_stringify(v) for v in value) + "]"
140
+ if isinstance(value, dict):
141
+ # JS omits `undefined` keys; Python has no undefined, and None maps to JS
142
+ # `null` which stableStringify DOES serialize — so serialize every key.
143
+ parts = [
144
+ json.dumps(k, ensure_ascii=False) + ":" + _stable_stringify(value[k])
145
+ for k in sorted(value.keys())
146
+ ]
147
+ return "{" + ",".join(parts) + "}"
148
+ return "null"
149
+
150
+
151
+ def _normalize_graph_for_hash(graph: dict) -> dict:
152
+ """Blank per-item nondeterminism (seed, prompt text, filename_prefix) so the
153
+ whole wave shares one comfy_workflow_sha. Mirror of the JS normalizer."""
154
+ out = {}
155
+ for nid, node in (graph or {}).items():
156
+ inputs = dict(node.get("inputs", {}))
157
+ cls = node.get("class_type")
158
+ if cls in ("KSampler", "KSamplerAdvanced"):
159
+ if "seed" in inputs:
160
+ inputs["seed"] = 0
161
+ if "noise_seed" in inputs:
162
+ inputs["noise_seed"] = 0
163
+ elif cls == "CLIPTextEncode":
164
+ if "text" in inputs:
165
+ inputs["text"] = ""
166
+ elif cls == "SaveImage":
167
+ if "filename_prefix" in inputs:
168
+ inputs["filename_prefix"] = ""
169
+ out[nid] = {"class_type": cls, "inputs": inputs}
170
+ return out
171
+
172
+
173
+ def _comfy_workflow_sha(graph: dict):
174
+ if not graph:
175
+ return None
176
+ payload = _stable_stringify(_normalize_graph_for_hash(graph))
177
+ return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()
178
+
179
+
180
+ def _resolve_model_path(name: str, kind: str):
181
+ if not name:
182
+ return None
183
+ subdirs = list(_KIND_SUBDIRS.get(kind, []))
184
+ for group in _KIND_SUBDIRS.values():
185
+ subdirs.extend(group)
186
+ subdirs.append(".")
187
+ seen = set()
188
+ for sub in subdirs:
189
+ cand = os.path.join(MODELS_DIR, sub, name)
190
+ if cand in seen:
191
+ continue
192
+ seen.add(cand)
193
+ if os.path.exists(cand):
194
+ return cand
195
+ return None
196
+
197
+
198
+ def _sha256_file(path: str) -> str:
199
+ h = hashlib.sha256()
200
+ with open(path, "rb") as f:
201
+ for chunk in iter(lambda: f.read(1 << 20), b""):
202
+ h.update(chunk)
203
+ return "sha256:" + h.hexdigest()
204
+
205
+
206
+ def _load_hash_cache() -> dict:
207
+ try:
208
+ with open(HASH_CACHE_FILE, "r", encoding="utf-8") as f:
209
+ return json.load(f)
210
+ except Exception:
211
+ return {}
212
+
213
+
214
+ def _save_hash_cache(cache: dict) -> None:
215
+ try:
216
+ with open(HASH_CACHE_FILE, "w", encoding="utf-8") as f:
217
+ json.dump(cache, f, indent=2)
218
+ except Exception:
219
+ pass
220
+
221
+
222
+ def _hash_model_file(name: str, kind: str, do_hash: bool, cache: dict) -> dict:
223
+ """Best-effort {name, size_bytes, sha256} for a model/LoRA file (mirror of the
224
+ JS hashModelFile). size_bytes is always filled when resolvable; sha256 is
225
+ gated behind do_hash and cached by (name, size, mtime)."""
226
+ if not name:
227
+ return {"name": name, "size_bytes": None, "sha256": None, "hash_note": "no filename"}
228
+ resolved = _resolve_model_path(name, kind)
229
+ if not resolved:
230
+ return {"name": name, "size_bytes": None, "sha256": None, "hash_note": f"unresolved under {MODELS_DIR}"}
231
+ try:
232
+ st = os.stat(resolved)
233
+ size_bytes = st.st_size
234
+ mtime_ms = round(st.st_mtime * 1000)
235
+ except OSError:
236
+ return {"name": name, "size_bytes": None, "sha256": None, "hash_note": "stat failed"}
237
+ if not do_hash:
238
+ return {"name": name, "size_bytes": size_bytes, "sha256": None, "hash_note": "hash skipped (pass --hash-models)"}
239
+ key = f"{name}:{size_bytes}:{mtime_ms}"
240
+ if key in cache:
241
+ return {"name": name, "size_bytes": size_bytes, "sha256": cache[key]}
242
+ try:
243
+ sha = _sha256_file(resolved)
244
+ except OSError:
245
+ return {"name": name, "size_bytes": size_bytes, "sha256": None, "hash_note": "hash failed (read error)"}
246
+ cache[key] = sha
247
+ return {"name": name, "size_bytes": size_bytes, "sha256": sha}
248
+
249
+
250
+ def _build_pinning(graph, models, loras, seed_policy, do_hash, cache) -> dict:
251
+ """Assemble the pinning block — identical shape to lib/run-manifest.js buildPinning."""
252
+ model_hashes = {}
253
+ for role, kind in (("unet", "unet"), ("clip", "clip"), ("vae", "vae"), ("checkpoint", "checkpoint")):
254
+ if models.get(role):
255
+ model_hashes[role] = _hash_model_file(models[role], kind, do_hash, cache)
256
+ lora_hashes = []
257
+ for lora in loras or []:
258
+ entry = _hash_model_file(lora.get("name"), "lora", do_hash, cache)
259
+ entry["weight"] = _js_nullish(lora.get("weight"), 1.0) # SDL-H4
260
+ lora_hashes.append(entry)
261
+ return {
262
+ "pinning_version": PINNING_VERSION,
263
+ "comfy_workflow_sha": _comfy_workflow_sha(graph),
264
+ "seed_policy": seed_policy,
265
+ "models": model_hashes,
266
+ "loras": lora_hashes,
267
+ }
268
+
34
269
 
35
270
  def qwen_graph(pos: str, neg: str, seed: int, w: int, h: int, steps: int, cfg: float, prefix: str,
36
- loras: list | None = None) -> dict:
271
+ loras: list | None = None, sampler: str = "euler", scheduler: str = "simple",
272
+ shift: float = 3.1) -> dict:
37
273
  """loras: [{name, weight}] chained as DiT-only LoraLoaderModelOnly nodes (ids from
38
274
  "40", between UNETLoader and ModelSamplingAuraFlow) — mirrors sdlab buildQwenGraph.
39
275
  Model-only because studio Qwen style LoRAs train the transformer with the text
40
- encoder frozen."""
276
+ encoder frozen.
277
+
278
+ SDL-H11: sampler/scheduler/shift used to be hardcoded literals here (always
279
+ "euler"/"simple"/3.1, regardless of what the wave declared) and were not
280
+ even accepted as parameters — so a wave asking for dpmpp_2m/karras/shift 5.0
281
+ silently generated with euler/simple/3.1 instead, produced a different
282
+ comfy_workflow_sha than the JS `sdlab generate` path for the "same" graph,
283
+ AND had those wrong values stamped into the provenance receipt. Defaults
284
+ here match QWEN_DEFAULTS in lib/adapters/comfyui-workflows.js — used only
285
+ when a wave omits the fields (mirrors JS's `rp.sampler || defaults.sampler
286
+ || QWEN_DEFAULTS.sampler` fallback chain)."""
41
287
  nodes = {
42
288
  "37": {"class_type": "UNETLoader", "inputs": {"unet_name": QWEN_UNET, "weight_dtype": "default"}},
43
289
  "38": {"class_type": "CLIPLoader", "inputs": {"clip_name": QWEN_CLIP, "type": "qwen_image"}},
@@ -49,13 +295,13 @@ def qwen_graph(pos: str, neg: str, seed: int, w: int, h: int, steps: int, cfg: f
49
295
  for i, lora in enumerate(loras or []):
50
296
  lid = str(40 + i)
51
297
  nodes[lid] = {"class_type": "LoraLoaderModelOnly", "inputs": {
52
- "lora_name": lora["name"], "strength_model": lora.get("weight", 1.0), "model": model_out}}
298
+ "lora_name": lora["name"], "strength_model": _js_nullish(lora.get("weight"), 1.0), "model": model_out}} # SDL-H4
53
299
  model_out = [lid, 0]
54
300
  nodes.update({
55
- "66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift": 3.1, "model": model_out}},
301
+ "66": {"class_type": "ModelSamplingAuraFlow", "inputs": {"shift": shift, "model": model_out}},
56
302
  "5": {"class_type": "EmptySD3LatentImage", "inputs": {"width": w, "height": h, "batch_size": 1}},
57
- "3": {"class_type": "KSampler", "inputs": {"seed": seed, "steps": steps, "cfg": cfg, "sampler_name": "euler",
58
- "scheduler": "simple", "denoise": 1.0, "model": ["66", 0], "positive": ["6", 0], "negative": ["7", 0], "latent_image": ["5", 0]}},
303
+ "3": {"class_type": "KSampler", "inputs": {"seed": seed, "steps": steps, "cfg": cfg, "sampler_name": sampler,
304
+ "scheduler": scheduler, "denoise": 1.0, "model": ["66", 0], "positive": ["6", 0], "negative": ["7", 0], "latent_image": ["5", 0]}},
59
305
  "8": {"class_type": "VAEDecode", "inputs": {"samples": ["3", 0], "vae": ["39", 0]}},
60
306
  "9": {"class_type": "SaveImage", "inputs": {"filename_prefix": prefix, "images": ["8", 0]}},
61
307
  })
@@ -68,7 +314,9 @@ def _post(path: str, payload: dict):
68
314
 
69
315
 
70
316
  def generate(pos: str, neg: str, seed: int, dst: Path, **kw) -> Path:
71
- g = qwen_graph(pos, neg, seed, kw["w"], kw["h"], kw["steps"], kw["cfg"], dst.stem, loras=kw.get("loras"))
317
+ g = qwen_graph(pos, neg, seed, kw["w"], kw["h"], kw["steps"], kw["cfg"], dst.stem, loras=kw.get("loras"),
318
+ sampler=kw.get("sampler", "euler"), scheduler=kw.get("scheduler", "simple"),
319
+ shift=kw.get("shift", 3.1))
72
320
  pid = _post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
73
321
  for _ in range(1200):
74
322
  hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
@@ -89,6 +337,8 @@ def main() -> None:
89
337
  ap.add_argument("--wave", required=True, type=Path)
90
338
  ap.add_argument("--out", required=True, type=Path)
91
339
  ap.add_argument("--variations", type=int, default=None, help="override per-subject variation count")
340
+ ap.add_argument("--hash-models", action="store_true",
341
+ help="content-hash the unet/clip/vae/LoRA files into the pinning block (best-effort, cached)")
92
342
  args = ap.parse_args()
93
343
 
94
344
  wave = json.loads(args.wave.read_text(encoding="utf-8"))
@@ -99,25 +349,96 @@ def main() -> None:
99
349
  steps, cfg = d.get("steps", 22), d.get("cfg", 3.5)
100
350
  base_seed = d.get("base_seed", 1000)
101
351
  loras = d.get("loras", [])
352
+ # SDL-H11: read sampler/scheduler/shift from the wave instead of hardcoding
353
+ # them — defaults match QWEN_DEFAULTS in lib/adapters/comfyui-workflows.js
354
+ # so an omitted field falls back to the same value the JS path would use.
355
+ sampler = d.get("sampler", "euler")
356
+ scheduler = d.get("scheduler", "simple")
357
+ shift = d.get("shift", 3.1)
102
358
  args.out.mkdir(parents=True, exist_ok=True)
103
359
 
104
360
  receipt = {"wave": wave.get("wave"), "base": "qwen-image", "unet": QWEN_UNET, "clip": QWEN_CLIP,
105
- "vae": QWEN_VAE, "sampler": "euler", "scheduler": "simple", "shift": 3.1,
361
+ "vae": QWEN_VAE, "sampler": sampler, "scheduler": scheduler, "shift": shift,
106
362
  "steps": steps, "cfg": cfg, "size": [w, h], "style_prefix": prefix, "negative": neg,
107
363
  "loras": loras, "items": []}
364
+
365
+ # PIN_PER_STEP: pin the exact graph + model/LoRA content identity + seed policy.
366
+ # The wave shares ONE comfy_workflow_sha (per-item seed + prompt are normalized
367
+ # out and recorded per item in items[]). seed_policy is "explicit-per-item"
368
+ # because each item's seed is written verbatim below. Mirror of the JS contract.
369
+ hash_cache = _load_hash_cache()
370
+ pin_graph = qwen_graph(prefix or "x", neg, base_seed, w, h, steps, cfg, "sdl", loras=loras,
371
+ sampler=sampler, scheduler=scheduler, shift=shift)
372
+ receipt["pinning"] = _build_pinning(
373
+ pin_graph,
374
+ {"unet": QWEN_UNET, "clip": QWEN_CLIP, "vae": QWEN_VAE},
375
+ loras,
376
+ "explicit-per-item",
377
+ args.hash_models,
378
+ hash_cache,
379
+ )
380
+
381
+ receipt_path = args.out / "generation.json"
382
+
383
+ def _save_receipt() -> None:
384
+ # M3: incremental checkpoint (mirrors lib/batch-runs.js's per-slot
385
+ # atomicWriteJson pattern) — written after EVERY item, not once at
386
+ # the end. Before this fix, the subject loop called generate() with
387
+ # no try/except (it can raise the named ComfyUI timeout at its
388
+ # RuntimeError above), and generation.json was written exactly once,
389
+ # after the loop finished. A 200-image wave failing at item 130 left
390
+ # 129 rendered PNGs on disk with NO generation.json at all — the
391
+ # PIN_PER_STEP provenance this file's own docstring promises, fully
392
+ # lost, for every item including the ones that succeeded.
393
+ receipt_path.write_text(json.dumps(receipt, indent=2), encoding="utf-8")
394
+
108
395
  seed = base_seed
396
+ error_count = 0
109
397
  for subj in wave["subjects"]:
110
398
  nvar = args.variations if args.variations is not None else subj.get("variations", 1)
111
399
  pos = f"{prefix}, {subj['prompt']}" if prefix else subj["prompt"]
112
400
  for v in range(nvar):
113
401
  dst = args.out / f"{subj['id']}_v{v}.png"
114
- generate(pos, neg, seed, dst, w=w, h=h, steps=steps, cfg=cfg, loras=loras)
115
- print(f" {subj['id']} v{v} (seed {seed}) -> {dst.name}", flush=True)
116
- receipt["items"].append({"id": subj["id"], "variation": v, "seed": seed, "file": dst.name, "prompt": pos})
402
+ item = {"id": subj["id"], "variation": v, "seed": seed, "file": dst.name, "prompt": pos}
403
+ try:
404
+ generate(pos, neg, seed, dst, w=w, h=h, steps=steps, cfg=cfg, loras=loras,
405
+ sampler=sampler, scheduler=scheduler, shift=shift)
406
+ except Exception as exc:
407
+ # A single item's failure no longer takes the whole wave's
408
+ # provenance down with it — record the failure and keep
409
+ # going. The receipt is checkpointed below regardless of
410
+ # per-item outcome.
411
+ item["status"] = "error"
412
+ item["error"] = str(exc)
413
+ error_count += 1
414
+ print(f" {subj['id']} v{v} (seed {seed}) -> ERROR: {exc}", flush=True)
415
+ else:
416
+ item["status"] = "ok"
417
+ print(f" {subj['id']} v{v} (seed {seed}) -> {dst.name}", flush=True)
418
+ receipt["items"].append(item)
117
419
  seed += 1
420
+ _save_receipt()
421
+
422
+ if args.hash_models:
423
+ _save_hash_cache(hash_cache)
424
+ _save_receipt()
425
+ ok_count = len(receipt["items"]) - error_count
426
+ # Plain ASCII only (no em-dash) — matches the rest of this file's
427
+ # runtime print()/error strings. A Windows console not running in UTF-8
428
+ # mode mangles non-ASCII on stdout; the docstring/comments can afford
429
+ # Unicode since those never reach a terminal at runtime.
430
+ summary = f"\n{ok_count}/{len(receipt['items'])} images -> {args.out} (+ generation.json provenance)"
431
+ if error_count:
432
+ summary += f" ({error_count} error(s) - see generation.json for per-item detail)"
433
+ print(summary)
118
434
 
119
- (args.out / "generation.json").write_text(json.dumps(receipt, indent=2), encoding="utf-8")
120
- print(f"\n{len(receipt['items'])} images -> {args.out} (+ generation.json provenance)")
435
+ # Mirror scripts/generate.js's RUNTIME_ALL_FAILED convention: a run where
436
+ # EVERY item failed exits non-zero (nothing to salvage); a partial
437
+ # failure still exits 0 — the per-item errors are visible above and in
438
+ # the checkpointed receipt, and individual item failure is tolerated
439
+ # everywhere else in this pipeline.
440
+ if receipt["items"] and error_count == len(receipt["items"]):
441
+ raise SystemExit(f"all {error_count} generation attempt(s) failed - see generation.json for per-item errors")
121
442
 
122
443
 
123
444
  if __name__ == "__main__":
@@ -11,7 +11,7 @@
11
11
 
12
12
  import { join } from 'node:path';
13
13
  import { parseArgs, getProjectName } from '../lib/args.js';
14
- import { REPO_ROOT, resolveSafeProjectPath } from '../lib/paths.js';
14
+ import { REPO_ROOT, getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
15
15
  import { inputError, handleCliError } from '../lib/errors.js';
16
16
  import { reingestGenerated, auditReingest } from '../lib/reingest.js';
17
17
 
@@ -29,7 +29,7 @@ export async function run(argv = process.argv.slice(2)) {
29
29
  });
30
30
 
31
31
  const projectName = flags.project || getProjectName(argv);
32
- const projectRoot = join(REPO_ROOT, 'projects', projectName);
32
+ const projectRoot = getProjectRoot(projectName);
33
33
  const subcommand = positionals[0] || 'audit';
34
34
 
35
35
  if (subcommand === 'generated') {
@@ -24,6 +24,7 @@ export async function run(argv = process.argv.slice(2)) {
24
24
  brief: { type: 'string' },
25
25
  seed: { type: 'string' },
26
26
  'dry-run': { type: 'boolean' },
27
+ 'hash-models': { type: 'boolean' },
27
28
  json: { type: 'boolean' },
28
29
  },
29
30
  deprecated: { game: 'project' },
@@ -36,6 +37,7 @@ export async function run(argv = process.argv.slice(2)) {
36
37
  console.log(' --brief <id> Compiled brief ID (required)');
37
38
  console.log(' --seed <n> Override base seed');
38
39
  console.log(' --dry-run Prepare run dir without submitting to ComfyUI');
40
+ console.log(' --hash-models Content-hash the model/LoRA files into the run manifest (best-effort, cached)');
39
41
  console.log(' --json Output manifest as JSON');
40
42
  return;
41
43
  }
@@ -59,6 +61,7 @@ export async function run(argv = process.argv.slice(2)) {
59
61
  brief,
60
62
  baseSeed: flags.seed ? parseNumberFlag('seed', flags.seed, { int: true, min: 0 }) : undefined,
61
63
  dryRun: flags['dry-run'] || false,
64
+ hashModels: flags['hash-models'] || false,
62
65
  });
63
66
 
64
67
  // Save summary
@@ -0,0 +1,142 @@
1
+ #!/usr/bin/env node
2
+
3
+ /**
4
+ * sheet.js — Render an HTML contact sheet over a directory of images.
5
+ *
6
+ * No batch manifest required. Points the SAME HTML renderer that powers
7
+ * `sdlab batch sheet` (lib/batch-sheet-render.js, via lib/candidate-sheet.js)
8
+ * at an arbitrary directory under a project — outputs/candidates/,
9
+ * outputs/approved/, outputs/, whatever — instead of requiring a
10
+ * batches/<id>/manifest.json that only `sdlab batch generate` can produce.
11
+ *
12
+ * Each image is matched to its record by asset_path (when a record exists)
13
+ * for id / curation status / prompt. Images with no matching record
14
+ * (pre-ingest — generated but not yet re-ingested) still render, labeled by
15
+ * filename instead of crashing or being skipped.
16
+ *
17
+ * Usage:
18
+ * sdlab sheet [<dir>] --project <name> [--out <path>] [--dry-run]
19
+ *
20
+ * Examples:
21
+ * sdlab sheet --project star-freight # outputs/candidates (default)
22
+ * sdlab sheet outputs/approved --project star-freight
23
+ * sdlab sheet outputs --project star-freight --dry-run # whole tree, no write
24
+ */
25
+
26
+ import { existsSync } from 'node:fs';
27
+ import { relative, dirname } from 'node:path';
28
+ import { parseArgs, getProjectName } from '../lib/args.js';
29
+ import { getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
30
+ import { inputError, handleCliError } from '../lib/errors.js';
31
+ import { result } from '../lib/log.js';
32
+ import {
33
+ buildCandidateSheetItems,
34
+ renderCandidateSheetHTML,
35
+ saveCandidateSheet,
36
+ defaultSheetOutputPath,
37
+ } from '../lib/candidate-sheet.js';
38
+
39
+ const DEFAULT_DIR = 'outputs/candidates';
40
+
41
+ function posix(p) {
42
+ return p.replace(/\\/g, '/');
43
+ }
44
+
45
+ export async function run(argv = process.argv.slice(2)) {
46
+ // Checked against the raw argv, not via parseArgs' flag defs: parseArgs
47
+ // only recognizes "--"-prefixed flags (see lib/args.js), so a bare "-h"
48
+ // would otherwise fall through to the positional `dir` slot instead of
49
+ // showing help. Mirrors bin/sdlab.js's own argvHasHelp() check.
50
+ if (argv.includes('--help') || argv.includes('-h')) {
51
+ printUsage();
52
+ return;
53
+ }
54
+
55
+ const { flags, positionals } = parseArgs(argv, {
56
+ flags: {
57
+ project: { type: 'string' },
58
+ out: { type: 'string' },
59
+ 'dry-run': { type: 'boolean', default: false },
60
+ },
61
+ positionals: ['dir'],
62
+ deprecated: { game: 'project' },
63
+ });
64
+
65
+ const projectName = flags.project || getProjectName(argv);
66
+ const projectRoot = getProjectRoot(projectName);
67
+ const dryRun = flags['dry-run'] || flags.dryRun;
68
+
69
+ const dirArg = positionals[0] || DEFAULT_DIR;
70
+ // Containment check — a user-supplied path must never be joined by hand.
71
+ const safeDir = resolveSafeProjectPath(projectRoot, dirArg, { flagName: 'dir' });
72
+ const relDir = posix(relative(projectRoot, safeDir)) || '.';
73
+
74
+ if (!existsSync(safeDir)) {
75
+ throw inputError(
76
+ 'INPUT_DIR_NOT_FOUND',
77
+ `Directory not found: ${relDir} (looked under ${projectRoot})`,
78
+ 'Try outputs/candidates, outputs/approved, outputs/rejected, outputs/borderline, or outputs (the whole tree).'
79
+ );
80
+ }
81
+
82
+ const items = await buildCandidateSheetItems(projectRoot, safeDir);
83
+
84
+ console.log(`\x1b[1mstyle-dataset-lab\x1b[0m sheet — ${projectName}`);
85
+ console.log(` Source: ${relDir}`);
86
+
87
+ if (items.length === 0) {
88
+ console.log(' No images found — nothing to render.');
89
+ return;
90
+ }
91
+
92
+ const matched = items.filter((i) => i.hasRecord).length;
93
+ console.log(` Images: ${items.length} (${matched} with records, ${items.length - matched} pre-ingest)`);
94
+
95
+ const outRel = flags.out
96
+ ? flags.out
97
+ : posix(relative(projectRoot, defaultSheetOutputPath(projectRoot, relDir)));
98
+ const safeOut = resolveSafeProjectPath(projectRoot, outRel, { flagName: 'out' });
99
+
100
+ if (dryRun) {
101
+ console.log(` \x1b[33m(dry-run)\x1b[0m would write: ${posix(relative(projectRoot, safeOut))}`);
102
+ return;
103
+ }
104
+
105
+ const html = renderCandidateSheetHTML({
106
+ items,
107
+ title: `Contact sheet — ${projectName} — ${relDir}`,
108
+ sourceLabel: relDir,
109
+ sheetOutputDir: dirname(safeOut),
110
+ });
111
+
112
+ const written = await saveCandidateSheet(safeOut, html);
113
+ result('sheet', posix(relative(projectRoot, written)));
114
+ }
115
+
116
+ function printUsage() {
117
+ console.log('Usage: sdlab sheet [<dir>] --project <name> [--out <path>] [--dry-run]');
118
+ console.log('');
119
+ console.log('Render an HTML contact sheet over a directory of images — no batch');
120
+ console.log('manifest required. Matches each image to its record (by asset_path)');
121
+ console.log('when one exists; degrades to filename when it does not (pre-ingest).');
122
+ console.log('');
123
+ console.log('Positional:');
124
+ console.log(' dir Directory to sheet, relative to the project root');
125
+ console.log(` (default: ${DEFAULT_DIR})`);
126
+ console.log('');
127
+ console.log('Flags:');
128
+ console.log(' --project <name> Project to operate on');
129
+ console.log(' --out <path> Output HTML path, relative to project root');
130
+ console.log(' (default: outputs/sheets/<slugified-dir>.html)');
131
+ console.log(' --dry-run Report what would be written without writing it');
132
+ console.log('');
133
+ console.log('Examples:');
134
+ console.log(' sdlab sheet --project star-freight');
135
+ console.log(' sdlab sheet outputs/approved --project star-freight');
136
+ console.log(' sdlab sheet outputs --project star-freight --dry-run');
137
+ }
138
+
139
+ // Direct execution guard — same pattern as every other script in this repo.
140
+ if (process.argv[1] && (process.argv[1].endsWith('sheet.js') || process.argv[1].endsWith('sheet'))) {
141
+ run().catch(handleCliError);
142
+ }
@@ -13,7 +13,7 @@
13
13
  import { readFile } from 'node:fs/promises';
14
14
  import { join } from 'node:path';
15
15
  import { parseArgs, getProjectName } from '../lib/args.js';
16
- import { REPO_ROOT } from '../lib/paths.js';
16
+ import { getProjectRoot } from '../lib/paths.js';
17
17
  import { loadSelectionProfile } from '../lib/config.js';
18
18
  import { inputError, handleCliError } from '../lib/errors.js';
19
19
  import { createSnapshot, loadSnapshot, listSnapshots, diffSnapshots } from '../lib/snapshot.js';
@@ -30,7 +30,7 @@ export async function run(argv = process.argv.slice(2)) {
30
30
  });
31
31
 
32
32
  const projectName = flags.project || getProjectName(argv);
33
- const projectRoot = join(REPO_ROOT, 'projects', projectName);
33
+ const projectRoot = getProjectRoot(projectName);
34
34
  const subcommand = positionals[0] || 'list';
35
35
  const dryRun = flags['dry-run'] || flags.dryRun;
36
36
 
package/scripts/split.js CHANGED
@@ -12,7 +12,7 @@
12
12
 
13
13
  import { join } from 'node:path';
14
14
  import { parseArgs, getProjectName } from '../lib/args.js';
15
- import { REPO_ROOT } from '../lib/paths.js';
15
+ import { getProjectRoot } from '../lib/paths.js';
16
16
  import { loadSplitProfile } from '../lib/config.js';
17
17
  import { inputError, handleCliError } from '../lib/errors.js';
18
18
  import { listSnapshots } from '../lib/snapshot.js';
@@ -30,7 +30,7 @@ export async function run(argv = process.argv.slice(2)) {
30
30
  });
31
31
 
32
32
  const projectName = flags.project || getProjectName(argv);
33
- const projectRoot = join(REPO_ROOT, 'projects', projectName);
33
+ const projectRoot = getProjectRoot(projectName);
34
34
  const subcommand = positionals[0] || 'list';
35
35
  const dryRun = flags['dry-run'] || flags.dryRun;
36
36
 
@@ -12,7 +12,7 @@
12
12
 
13
13
  import { join } from 'node:path';
14
14
  import { parseArgs, getProjectName } from '../lib/args.js';
15
- import { REPO_ROOT } from '../lib/paths.js';
15
+ import { getProjectRoot } from '../lib/paths.js';
16
16
  import { inputError, handleCliError } from '../lib/errors.js';
17
17
  import { listExports } from '../lib/export.js';
18
18
  import {
@@ -35,7 +35,7 @@ export async function run(argv = process.argv.slice(2)) {
35
35
  });
36
36
 
37
37
  const projectName = flags.project || getProjectName(argv);
38
- const projectRoot = join(REPO_ROOT, 'projects', projectName);
38
+ const projectRoot = getProjectRoot(projectName);
39
39
  const subcommand = positionals.find(a => !a.startsWith('tm-')) || 'list';
40
40
 
41
41
  if (subcommand === 'create') {