@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/README.es.md +117 -55
  3. package/README.fr.md +125 -65
  4. package/README.hi.md +117 -68
  5. package/README.it.md +119 -58
  6. package/README.ja.md +124 -74
  7. package/README.md +73 -7
  8. package/README.pt-BR.md +128 -63
  9. package/README.zh.md +116 -65
  10. package/bin/sdlab.js +100 -4
  11. package/lib/adapters/ai-toolkit.js +224 -35
  12. package/lib/adapters/comfyui-runner.js +79 -26
  13. package/lib/adapters/comfyui-workflows.js +94 -11
  14. package/lib/adapters/diffusers-lora.js +37 -5
  15. package/lib/adapters/generic-image-caption.js +43 -5
  16. package/lib/args.js +16 -1
  17. package/lib/asset-ingest.js +575 -0
  18. package/lib/asset-masks.js +323 -0
  19. package/lib/asset-source.js +1176 -0
  20. package/lib/batch-modes.js +29 -8
  21. package/lib/batch-runs.js +29 -1
  22. package/lib/batch-sheet-render.js +13 -3
  23. package/lib/brief-compiler.js +19 -0
  24. package/lib/candidate-sheet.js +327 -0
  25. package/lib/canon-author.js +350 -0
  26. package/lib/canon-build/build.js +146 -93
  27. package/lib/canon-review-gate.js +205 -0
  28. package/lib/captions.js +57 -0
  29. package/lib/card.js +74 -2
  30. package/lib/colab-launcher-guards.js +285 -0
  31. package/lib/comfyui.js +116 -18
  32. package/lib/config.js +116 -12
  33. package/lib/critique-engine.js +36 -2
  34. package/lib/eligibility.js +121 -9
  35. package/lib/eval-pack.js +14 -14
  36. package/lib/eval-runs.js +33 -0
  37. package/lib/export.js +122 -34
  38. package/lib/freeze-gate.js +16 -3
  39. package/lib/freeze-stamp.js +28 -3
  40. package/lib/generated-provenance.js +194 -6
  41. package/lib/implementation-packs.js +95 -17
  42. package/lib/ingest.js +396 -0
  43. package/lib/judgment-provenance.js +220 -0
  44. package/lib/lane1-render.js +668 -0
  45. package/lib/measure.js +422 -0
  46. package/lib/memory-store.js +663 -0
  47. package/lib/model-licences.js +305 -0
  48. package/lib/npy-meta.js +130 -0
  49. package/lib/paths.js +196 -7
  50. package/lib/png-meta.js +512 -0
  51. package/lib/records.js +54 -8
  52. package/lib/reingest-selected.js +113 -17
  53. package/lib/reingest.js +27 -3
  54. package/lib/run-manifest.js +159 -25
  55. package/lib/run-summary.js +12 -6
  56. package/lib/runtime-runs.js +45 -4
  57. package/lib/snapshot.js +114 -15
  58. package/lib/split.js +364 -48
  59. package/lib/train-runs.js +214 -0
  60. package/lib/training-manifests.js +114 -21
  61. package/lib/training-packages.js +147 -15
  62. package/lib/verifier.js +3 -0
  63. package/lib/verify-chain.js +367 -0
  64. package/lib/workflow-profiles.js +24 -3
  65. package/package.json +4 -3
  66. package/runtime/comfyui/txt2img-qwen.json +38 -0
  67. package/runtime/comfyui/txt2img-standard.json +21 -0
  68. package/scripts/_resume.js +62 -0
  69. package/scripts/_shared-args.js +44 -0
  70. package/scripts/asset-ingest.js +160 -0
  71. package/scripts/brief-compile.js +2 -1
  72. package/scripts/brief-show.js +2 -1
  73. package/scripts/canon-bind.js +2 -2
  74. package/scripts/canon-build.js +27 -20
  75. package/scripts/canon-draft.js +82 -0
  76. package/scripts/canon-drift.js +16 -7
  77. package/scripts/canon-freeze-status.js +14 -6
  78. package/scripts/canon-freeze.js +31 -11
  79. package/scripts/canon-unfreeze.js +17 -9
  80. package/scripts/card.js +2 -2
  81. package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
  82. package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
  83. package/scripts/colab/colab_window.sh +587 -0
  84. package/scripts/colab/lane1_render.py +970 -0
  85. package/scripts/colab/launch_guards.py +564 -0
  86. package/scripts/colab/launch_lane1_stub.sh +309 -0
  87. package/scripts/colab/launch_lane3_stub.sh +366 -0
  88. package/scripts/colab/remote_runner.py +59 -0
  89. package/scripts/compare.js +2 -2
  90. package/scripts/critique.js +16 -1
  91. package/scripts/curate.js +2 -2
  92. package/scripts/doctor.js +182 -55
  93. package/scripts/eligibility.js +42 -2
  94. package/scripts/eval-pack.js +2 -2
  95. package/scripts/eval-run.js +2 -2
  96. package/scripts/export.js +2 -2
  97. package/scripts/generate-controlnet.js +45 -25
  98. package/scripts/generate-identity.js +53 -23
  99. package/scripts/generate-ipadapter.js +44 -23
  100. package/scripts/generate.js +86 -17
  101. package/scripts/implementation-pack.js +2 -2
  102. package/scripts/ingest.js +145 -0
  103. package/scripts/init.js +73 -13
  104. package/scripts/measure.js +170 -0
  105. package/scripts/measure_image.py +458 -0
  106. package/scripts/migrate-records.js +142 -0
  107. package/scripts/painterly-test.js +4 -3
  108. package/scripts/painterly.js +7 -3
  109. package/scripts/qwen_generate.py +138 -20
  110. package/scripts/reingest.js +2 -2
  111. package/scripts/sheet.js +142 -0
  112. package/scripts/snapshot.js +2 -2
  113. package/scripts/split.js +2 -2
  114. package/scripts/training-manifest.js +2 -2
  115. package/scripts/training-package.js +2 -2
  116. package/scripts/training-profile.js +2 -2
  117. package/scripts/verify.js +67 -0
  118. package/scripts/workflow-list.js +2 -1
  119. package/scripts/workflow-show.js +2 -1
@@ -6,9 +6,9 @@
6
6
  * canon-binding flow as everything else. No bypass.
7
7
  */
8
8
 
9
- import { writeFile, mkdir, copyFile } from 'node:fs/promises';
9
+ import { writeFile, mkdir, copyFile, rename, unlink } from 'node:fs/promises';
10
10
  import { join, extname } from 'node:path';
11
- import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
11
+ import { existsSync, readFileSync, statSync } from 'node:fs';
12
12
  import { getRunsDir } from './paths.js';
13
13
  import { getBatchesDir } from './batch-runs.js';
14
14
  import { getSelectionsDir, loadSelection } from './selections.js';
@@ -16,6 +16,35 @@ import { buildGeneratedProvenance } from './generated-provenance.js';
16
16
  import { buildBaseRecord } from './records.js';
17
17
  import { inputError } from './errors.js';
18
18
 
19
+ // SDL-H7: the only shape a selection id can ever legitimately have,
20
+ // mirroring generateSelectionId() in lib/selections.js (selection_<date>_<seq>,
21
+ // same prefix/date/seq scheme as run/batch ids). Validated BEFORE the id is
22
+ // used anywhere — including before loadSelection(), which does its own
23
+ // unguarded join — so a traversal payload never reaches the filesystem, and
24
+ // before the join at `selectionDir` below, which this module writes
25
+ // provenance.jsonl into. A mismatch reuses loadSelection's own
26
+ // SELECTION_NOT_FOUND shape (not a distinct code) because no real selection
27
+ // can ever have an id outside this shape.
28
+ const SELECTION_ID_RE = /^selection_\d{4}-\d{2}-\d{2}_\d{3}$/;
29
+
30
+ /**
31
+ * B03: atomic write helper — write temp + rename, mirroring the same
32
+ * temp-file-then-rename checkpoint pattern lib/batch-runs.js uses for its
33
+ * per-slot manifest checkpoint. Rename is atomic on a single filesystem, so
34
+ * a crash mid-write can never leave a half-written provenance.jsonl in
35
+ * place of a good one.
36
+ */
37
+ async function atomicWriteText(targetPath, text) {
38
+ const tmpPath = `${targetPath}.tmp-${process.pid}-${Date.now()}`;
39
+ try {
40
+ await writeFile(tmpPath, text);
41
+ await rename(tmpPath, targetPath);
42
+ } catch (err) {
43
+ try { await unlink(tmpPath); } catch { /* ignore */ }
44
+ throw err;
45
+ }
46
+ }
47
+
19
48
  /**
20
49
  * Re-ingest a selection — creates candidate records from chosen outputs.
21
50
  *
@@ -27,6 +56,14 @@ import { inputError } from './errors.js';
27
56
  * @returns {Promise<{created: string[], skipped: string[]}>}
28
57
  */
29
58
  export async function reingestSelection({ projectRoot, projectId, selectionId, dryRun }) {
59
+ if (typeof selectionId !== 'string' || !SELECTION_ID_RE.test(selectionId)) {
60
+ throw inputError(
61
+ 'SELECTION_NOT_FOUND',
62
+ `Selection "${selectionId}" not found`,
63
+ 'Check the selection ID or run: sdlab select --run <id> --approve <files>',
64
+ );
65
+ }
66
+
30
67
  const manifest = loadSelection(projectRoot, selectionId);
31
68
 
32
69
  if (!manifest.reingest_ready) {
@@ -69,7 +106,46 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
69
106
 
70
107
  const created = [];
71
108
  const skipped = [];
109
+
110
+ // B03: provenance.jsonl is now built incrementally — checkpointed (atomic
111
+ // full-rewrite, temp+rename) after every item — instead of accumulated in
112
+ // memory and overwritten once at the end. `provenanceLines` seeds from
113
+ // whatever is ALREADY on disk (e.g. left by an interrupted prior run of
114
+ // this same selection) so: (a) a rerun never duplicates a line for a
115
+ // record it already logged, and (b) a partial rerun can never ERASE lines
116
+ // a previous call already committed — the old end-of-run `writeFile`
117
+ // overwrote with only the CURRENT call's accumulator, so a second call
118
+ // touching fewer items than the first could silently wipe out provenance
119
+ // the first call had already written to disk.
120
+ const provenancePath = join(selectionDir, 'provenance.jsonl');
72
121
  const provenanceLines = [];
122
+ const seenProvenanceIds = new Set();
123
+ if (existsSync(provenancePath)) {
124
+ for (const line of readFileSync(provenancePath, 'utf-8').split('\n')) {
125
+ if (!line.trim()) continue;
126
+ provenanceLines.push(line);
127
+ try {
128
+ const parsed = JSON.parse(line);
129
+ if (parsed?.record_id) seenProvenanceIds.add(parsed.record_id);
130
+ } catch {
131
+ // Tolerate a corrupt/partial trailing line from a prior crash —
132
+ // keep it verbatim rather than dropping it, just don't index it.
133
+ }
134
+ }
135
+ }
136
+
137
+ /**
138
+ * Checkpoint one provenance line: skips dryRun, and skips a record_id
139
+ * that's already logged (idempotent — safe to call on both the fresh-write
140
+ * path and the skip/backfill path below).
141
+ */
142
+ async function checkpointProvenance(recordId, prov) {
143
+ if (dryRun) return;
144
+ if (seenProvenanceIds.has(recordId)) return;
145
+ provenanceLines.push(JSON.stringify({ record_id: recordId, ...prov }));
146
+ seenProvenanceIds.add(recordId);
147
+ await atomicWriteText(provenancePath, provenanceLines.join('\n') + '\n');
148
+ }
73
149
 
74
150
  // D-010: use entries() — not indexOf() — to avoid O(n^2) behavior and
75
151
  // silent duplicate-ID bugs when manifest.items contains structurally
@@ -88,12 +164,11 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
88
164
  seenIds.add(recordId);
89
165
  const recordPath = join(recordsDir, `${recordId}.json`);
90
166
 
91
- if (existsSync(recordPath)) {
92
- skipped.push(recordId);
93
- continue;
94
- }
95
-
96
- // Resolve the run manifest for this item
167
+ // Resolve the run manifest for this item + build its provenance BEFORE
168
+ // the existsSync check below — B03: a skipped (already-created) item
169
+ // still needs its provenance backfilled if an earlier interrupted run
170
+ // stopped short of appending it, so this can't be gated behind "the
171
+ // record is new".
97
172
  let runManifest;
98
173
  if (manifest.source_type === 'run') {
99
174
  runManifest = getRunManifest(manifest.source_id);
@@ -124,8 +199,19 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
124
199
  batchManifest,
125
200
  item,
126
201
  projectRoot,
202
+ canonSha: manifest.canon_sha ?? null,
127
203
  });
128
204
 
205
+ if (existsSync(recordPath)) {
206
+ // B03: the record already landed on disk (this item was completed by
207
+ // a prior, possibly-interrupted run of this same re-ingest). Backfill
208
+ // its provenance line if that prior run never got to append one —
209
+ // never let a record exist with no corresponding provenance entry.
210
+ await checkpointProvenance(recordId, prov);
211
+ skipped.push(recordId);
212
+ continue;
213
+ }
214
+
129
215
  // Image source in selection/chosen/
130
216
  const srcImage = join(selectionDir, 'chosen', item.filename);
131
217
  const ext = extname(item.filename) || '.png';
@@ -139,6 +225,15 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
139
225
  fileBytes = statSync(srcImage).size;
140
226
  }
141
227
 
228
+ const canonBlock =
229
+ prov.canon_sha || prov.entry_id || prov.entry_sha256
230
+ ? {
231
+ canon_sha: prov.canon_sha ?? null,
232
+ entry_id: prov.entry_id ?? null,
233
+ entry_sha256: prov.entry_sha256 ?? null,
234
+ }
235
+ : null;
236
+
142
237
  // Build record
143
238
  const record = buildBaseRecord(recordId, assetPath, {
144
239
  width: null,
@@ -148,9 +243,13 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
148
243
  source: 'generated',
149
244
  selection_id: selectionId,
150
245
  generation_provenance: prov,
246
+ }, {
247
+ canon: canonBlock,
151
248
  });
152
249
 
153
250
  record.schema_version = '2.2.0';
251
+ record.generation_provenance = prov;
252
+ if (projectId) record.project_id = projectId;
154
253
 
155
254
  // Tags
156
255
  const recordTags = ['generated', 'selected'];
@@ -166,18 +265,15 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
166
265
  }
167
266
  // Write record
168
267
  await writeFile(recordPath, JSON.stringify(record, null, 2) + '\n');
268
+ // B03: checkpoint immediately after the record write (per-item,
269
+ // matching lib/batch-runs.js's per-slot checkpoint pattern) — NOT
270
+ // batched to the end of the run. A crash on item 120 of 200 must
271
+ // leave provenance for 1-119 durably on disk, not lose it because
272
+ // item 200 never finished.
273
+ await checkpointProvenance(recordId, prov);
169
274
  }
170
275
 
171
276
  created.push(recordId);
172
- provenanceLines.push(JSON.stringify({ record_id: recordId, ...prov }));
173
- }
174
-
175
- // Write provenance log
176
- if (!dryRun && provenanceLines.length > 0) {
177
- await writeFile(
178
- join(selectionDir, 'provenance.jsonl'),
179
- provenanceLines.join('\n') + '\n',
180
- );
181
277
  }
182
278
 
183
279
  return { created, skipped };
package/lib/reingest.js CHANGED
@@ -12,6 +12,8 @@ import { join, basename, extname } from 'node:path';
12
12
  import { existsSync } from 'node:fs';
13
13
  import { loadTrainingManifest } from './training-manifests.js';
14
14
  import { buildBaseRecord } from './records.js';
15
+ import { resolveCanonLink } from './generated-provenance.js';
16
+ import { resolveModelLicence } from './model-licences.js';
15
17
 
16
18
  const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
17
19
 
@@ -22,12 +24,12 @@ const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
22
24
  * - provenance.source = "generated"
23
25
  * - provenance.training_manifest_id = <manifest-id>
24
26
  * - judgment = null (must be reviewed)
25
- * - canon = null (must be bound)
27
+ * - canon = resolved canon link when canon-build is present, else null
26
28
  *
27
29
  * @param {string} projectRoot
28
30
  * @param {string} sourcePath — directory containing generated images
29
31
  * @param {string} manifestId — training manifest that produced them
30
- * @param {Object} options — { dryRun? }
32
+ * @param {Object} options — { dryRun?, canonSha?, entryId? }
31
33
  * @returns {Promise<{created: number, skipped: number, records: string[]}>}
32
34
  */
33
35
  export async function reingestGenerated(projectRoot, sourcePath, manifestId, options = {}) {
@@ -66,12 +68,32 @@ export async function reingestGenerated(projectRoot, sourcePath, manifestId, opt
66
68
  // Relative asset path from project root
67
69
  const assetPath = `outputs/candidates/${file}`;
68
70
 
71
+ const canonLink = resolveCanonLink(projectRoot, {
72
+ canonSha: options.canonSha || manifest.canon_sha || null,
73
+ entryId: options.entryId || stem,
74
+ });
75
+ const baseLic = manifest.base_model ? resolveModelLicence(manifest.base_model) : null;
76
+
69
77
  const provenance = {
70
78
  source: 'generated',
71
79
  training_manifest_id: manifestId,
72
80
  training_profile_id: manifest.training_profile_id,
73
81
  adapter_target: manifest.adapter_target,
74
82
  base_model: manifest.base_model || null,
83
+ ...(baseLic
84
+ ? {
85
+ licence_id: baseLic.licence_id,
86
+ licence_url: baseLic.licence_url,
87
+ date_read: baseLic.date_read,
88
+ }
89
+ : {}),
90
+ ...(canonLink
91
+ ? {
92
+ canon_sha: canonLink.canon_sha,
93
+ entry_id: canonLink.entry_id,
94
+ entry_sha256: canonLink.entry_sha256,
95
+ }
96
+ : {}),
75
97
  source_export_id: manifest.source_export_id,
76
98
  generated_at: new Date().toISOString(),
77
99
  original_path: filePath,
@@ -81,7 +103,9 @@ export async function reingestGenerated(projectRoot, sourcePath, manifestId, opt
81
103
  width: null,
82
104
  height: null,
83
105
  bytes: fileStat.size,
84
- }, provenance);
106
+ }, provenance, {
107
+ canon: canonLink,
108
+ });
85
109
 
86
110
  record.schema_version = '2.1.0';
87
111
 
@@ -25,6 +25,20 @@
25
25
  import { createHash } from 'node:crypto';
26
26
  import { openSync, readSync, closeSync, statSync, existsSync, readFileSync, writeFileSync } from 'node:fs';
27
27
  import { join } from 'node:path';
28
+ import { warn } from './log.js';
29
+ import {
30
+ CLEARED_SPDX_IDS,
31
+ resolveModelLicence,
32
+ isClearedLicenceEntry,
33
+ assertClearedPinningLicences,
34
+ } from './model-licences.js';
35
+
36
+ export {
37
+ CLEARED_SPDX_IDS,
38
+ resolveModelLicence,
39
+ isClearedLicenceEntry,
40
+ assertClearedPinningLicences,
41
+ };
28
42
 
29
43
  /**
30
44
  * Contract version for the `pinning` block itself. Bump when the block's shape
@@ -35,6 +49,16 @@ import { join } from 'node:path';
35
49
  */
36
50
  export const PINNING_VERSION = '1.0.0';
37
51
 
52
+ /**
53
+ * The ONE hash_note value that means "hashing was never attempted because
54
+ * --hash-models wasn't passed" — NOT a failure. buildPinning() (B07) needs
55
+ * to tell this apart from every other hash_note (unresolved/stat
56
+ * failed/read error/no filename), all of which ARE failures worth warning
57
+ * about. Shared as a constant so hashModelFile's own note and buildPinning's
58
+ * filter can never drift apart.
59
+ */
60
+ const HASH_SKIPPED_NOTE = 'hash skipped (pass --hash-models)';
61
+
38
62
  /**
39
63
  * Seed policies — the INTENT behind seed selection, recorded alongside the
40
64
  * concrete per-item seed values so a reader knows how to regenerate.
@@ -144,17 +168,24 @@ const KIND_SUBDIRS = {
144
168
 
145
169
  /**
146
170
  * Resolve a model filename to an absolute path under the models dir. Searches
147
- * the kind-appropriate subdirs first, then every known subdir, then the root.
148
- * Returns null when nothing matches (unresolvable → sha256 null with a note).
171
+ * ONLY the kind-appropriate subdirs (plus the modelsDir root itself, which is
172
+ * kind-agnostic, not another kind's territory) — mirroring ComfyUI's own
173
+ * loaders, which never search across kinds (a UNETLoader never looks in
174
+ * checkpoints/, a VAELoader never looks in loras/, etc).
175
+ *
176
+ * SDL-M8: this used to also fall back to every OTHER kind's subdirs, so a
177
+ * missing unet could silently resolve to a same-named file under
178
+ * checkpoints/ and get pinned as though it were the unet ComfyUI actually
179
+ * loaded — a clean-looking wrong identity. An honest `null` (unresolved →
180
+ * sha256 null + a note, see hashModelFile) is the correct outcome when the
181
+ * kind-specific subdirs don't have the file.
182
+ *
183
+ * Returns null when nothing matches.
149
184
  */
150
185
  export function resolveModelPath(name, kind, modelsDir = defaultModelsDir()) {
151
186
  if (!name || !modelsDir) return null;
187
+ const subdirs = [...(KIND_SUBDIRS[kind] || []), '.'];
152
188
  const tried = new Set();
153
- const subdirs = [
154
- ...(KIND_SUBDIRS[kind] || []),
155
- ...Object.values(KIND_SUBDIRS).flat(),
156
- '.',
157
- ];
158
189
  for (const sub of subdirs) {
159
190
  const candidate = join(modelsDir, sub, name);
160
191
  if (tried.has(candidate)) continue;
@@ -183,12 +214,63 @@ export function sha256File(path) {
183
214
  return 'sha256:' + hash.digest('hex');
184
215
  }
185
216
 
217
+ // SDL-M9: a cheap partial-content fingerprint folded into the cache key
218
+ // alongside name/size/mtime. The key used to be bare name:size:round(mtime)
219
+ // with no content check — on a coarse-mtime mount (network shares, some
220
+ // FAT/exFAT mounts), a same-size in-place replacement (e.g. swapping a
221
+ // LoRA) can round to the SAME mtime as the file it replaced, so the cache
222
+ // would silently serve the sha256 of the OLD content forever. Sampling the
223
+ // first/last few KB — not hashing the whole multi-GB file, which would
224
+ // defeat the point of caching — is enough to catch almost any real
225
+ // replacement while staying cheap on every call (including cache hits).
226
+ const SAMPLE_BYTES = 4096;
227
+
228
+ /**
229
+ * Cheap content fingerprint: sha256 of the first + last SAMPLE_BYTES of the
230
+ * file (or the whole file if smaller). Synchronous, bounded I/O — a few KB
231
+ * regardless of file size, unlike sha256File's full-content streaming read.
232
+ */
233
+ function sampleFingerprint(path, size) {
234
+ const fd = openSync(path, 'r');
235
+ try {
236
+ const hash = createHash('sha256');
237
+ const headLen = Math.min(SAMPLE_BYTES, size);
238
+ if (headLen > 0) {
239
+ const head = Buffer.allocUnsafe(headLen);
240
+ readSync(fd, head, 0, headLen, 0);
241
+ hash.update(head);
242
+ }
243
+ if (size > SAMPLE_BYTES) {
244
+ const tailLen = Math.min(SAMPLE_BYTES, size - headLen);
245
+ const tail = Buffer.allocUnsafe(tailLen);
246
+ readSync(fd, tail, 0, tailLen, size - tailLen);
247
+ hash.update(tail);
248
+ }
249
+ return hash.digest('hex').slice(0, 16);
250
+ } finally {
251
+ closeSync(fd);
252
+ }
253
+ }
254
+
255
+ function withLicenceFields(base, name, { licence = null, modelLicences = null } = {}) {
256
+ const lic = resolveModelLicence(name, licence, modelLicences);
257
+ return {
258
+ ...base,
259
+ licence_id: lic.licence_id,
260
+ licence_url: lic.licence_url,
261
+ date_read: lic.date_read,
262
+ licence: lic,
263
+ };
264
+ }
265
+
186
266
  /**
187
267
  * Best-effort content identity for one model/LoRA file: {name, size_bytes, sha256}
188
268
  * (+ a short `hash_note` when sha256 can't be produced). size_bytes is always
189
269
  * filled when the file resolves (a cheap stat); sha256 is gated behind
190
- * `doHash` (the --hash-models opt-in) and cached by (name, size, mtime) so
191
- * multi-GB checkpoints aren't re-hashed every wave.
270
+ * `doHash` (the --hash-models opt-in) and cached by (name, size, mtime,
271
+ * partial-content sample) so multi-GB checkpoints aren't re-hashed every
272
+ * wave (SDL-M9: the sample is what keeps a same-size in-place replacement
273
+ * from reusing a stale cache entry).
192
274
  *
193
275
  * @param {string} name — model filename as it appears in the graph
194
276
  * @param {Object} [opts]
@@ -197,10 +279,10 @@ export function sha256File(path) {
197
279
  * @param {boolean} [opts.doHash=false] — compute sha256 (else record name+size only)
198
280
  * @param {{map: Map, cacheFile?: string, dirty: boolean}} [opts.cache]
199
281
  */
200
- export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHash = false, cache } = {}) {
201
- if (!name) return { name: name ?? null, size_bytes: null, sha256: null, hash_note: 'no filename' };
282
+ export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHash = false, cache, licence = null, modelLicences = null } = {}) {
283
+ if (!name) return withLicenceFields({ name: name ?? null, size_bytes: null, sha256: null, hash_note: 'no filename' }, name, { licence, modelLicences });
202
284
  const resolved = resolveModelPath(name, kind, modelsDir);
203
- if (!resolved) return { name, size_bytes: null, sha256: null, hash_note: `unresolved under ${modelsDir}` };
285
+ if (!resolved) return withLicenceFields({ name, size_bytes: null, sha256: null, hash_note: `unresolved under ${modelsDir}` }, name, { licence, modelLicences });
204
286
  let size_bytes = null;
205
287
  let mtimeMs = 0;
206
288
  try {
@@ -208,22 +290,30 @@ export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHa
208
290
  size_bytes = st.size;
209
291
  mtimeMs = st.mtimeMs;
210
292
  } catch {
211
- return { name, size_bytes: null, sha256: null, hash_note: 'stat failed' };
293
+ return withLicenceFields({ name, size_bytes: null, sha256: null, hash_note: 'stat failed' }, name, { licence, modelLicences });
294
+ }
295
+ if (!doHash) return withLicenceFields({ name, size_bytes, sha256: null, hash_note: HASH_SKIPPED_NOTE }, name, { licence, modelLicences });
296
+ let sample = '';
297
+ try {
298
+ sample = sampleFingerprint(resolved, size_bytes);
299
+ } catch {
300
+ // Best-effort — if sampling fails, fall back to the coarser
301
+ // name:size:mtime key; the sha256File() attempt below will surface any
302
+ // real read error anyway.
212
303
  }
213
- if (!doHash) return { name, size_bytes, sha256: null, hash_note: 'hash skipped (pass --hash-models)' };
214
- const key = `${name}:${size_bytes}:${Math.round(mtimeMs)}`;
215
- if (cache?.map?.has(key)) return { name, size_bytes, sha256: cache.map.get(key) };
304
+ const key = `${name}:${size_bytes}:${Math.round(mtimeMs)}:${sample}`;
305
+ if (cache?.map?.has(key)) return withLicenceFields({ name, size_bytes, sha256: cache.map.get(key) }, name, { licence, modelLicences });
216
306
  let sha256;
217
307
  try {
218
308
  sha256 = sha256File(resolved);
219
309
  } catch {
220
- return { name, size_bytes, sha256: null, hash_note: 'hash failed (read error)' };
310
+ return withLicenceFields({ name, size_bytes, sha256: null, hash_note: 'hash failed (read error)' }, name, { licence, modelLicences });
221
311
  }
222
312
  if (cache?.map) {
223
313
  cache.map.set(key, sha256);
224
314
  cache.dirty = true;
225
315
  }
226
- return { name, size_bytes, sha256 };
316
+ return withLicenceFields({ name, size_bytes, sha256 }, name, { licence, modelLicences });
227
317
  }
228
318
 
229
319
  /**
@@ -291,18 +381,62 @@ export function flushHashCache(cache) {
291
381
  * @param {boolean} [args.hashModels=false] — opt-in sha256 of model/LoRA files
292
382
  * @param {string} [args.modelsDir]
293
383
  * @param {{map: Map, cacheFile?: string, dirty: boolean}} [args.cache]
384
+ * @param {Object} [args.modelLicences]
385
+ * @param {boolean} [args.enforceLicence=false]
386
+ * @param {boolean} [args.allowUnlicensedResearch=false]
294
387
  * @returns {Object} pinning block
295
388
  */
296
- export function buildPinning({ graph, models = {}, loras = [], seedPolicy = null, hashModels = false, modelsDir = defaultModelsDir(), cache } = {}) {
297
- const opts = { modelsDir, doHash: hashModels, cache };
389
+ export function buildPinning({
390
+ graph,
391
+ models = {},
392
+ loras = [],
393
+ seedPolicy = null,
394
+ hashModels = false,
395
+ modelsDir = defaultModelsDir(),
396
+ cache,
397
+ modelLicences = null,
398
+ enforceLicence = false,
399
+ allowUnlicensedResearch = false,
400
+ } = {}) {
401
+ const opts = { modelsDir, doHash: hashModels, cache, modelLicences };
402
+ const modelHashes = buildModelHashes(models, opts);
403
+ const loraHashes = (loras || []).map((l) => ({
404
+ ...hashModelFile(l.name, { ...opts, kind: 'lora', licence: l.licence || null }),
405
+ weight: l.weight ?? 1.0,
406
+ }));
407
+
408
+ if (enforceLicence) {
409
+ assertClearedPinningLicences(modelHashes, loraHashes, { allowUnlicensedResearch });
410
+ }
411
+
412
+ // B07: hash_note silently encoded a hashing FAILURE (unresolved file,
413
+ // stat failed, read error, no filename) with nothing reading it —
414
+ // buildPinning() never aggregated or logged, and this file imported no
415
+ // logger at all. A typo'd SDLAB_MODELS_DIR produced a wave where every
416
+ // model has sha256: null + a note, with zero console signal — PIN_PER_STEP's
417
+ // whole purpose (byte-for-byte replayability) lost silently. Warn ONCE per
418
+ // wave with a count, but never on HASH_SKIPPED_NOTE — that note means
419
+ // "hashing wasn't requested" (the default, --hash-models omitted), which
420
+ // is expected behavior, not a failure, and would make every ordinary run
421
+ // noisy if it warned too.
422
+ const allEntries = [...Object.values(modelHashes), ...loraHashes];
423
+ const failed = allEntries.filter((e) => e && e.hash_note != null && e.hash_note !== HASH_SKIPPED_NOTE);
424
+ if (failed.length > 0) {
425
+ const detail = failed.map((e) => `${e.name ?? '(unnamed)'} (${e.hash_note})`).join(', ');
426
+ warn(`buildPinning: ${failed.length}/${allEntries.length} model(s)/lora(s) failed to hash: ${detail}`);
427
+ }
428
+
429
+ const ccBy4Eligible =
430
+ !allowUnlicensedResearch &&
431
+ allEntries.length > 0 &&
432
+ allEntries.every((entry) => isClearedLicenceEntry(entry));
433
+
298
434
  return {
299
435
  pinning_version: PINNING_VERSION,
300
436
  comfy_workflow_sha: comfyWorkflowSha(graph),
301
437
  seed_policy: seedPolicy,
302
- models: buildModelHashes(models, opts),
303
- loras: (loras || []).map((l) => ({
304
- ...hashModelFile(l.name, { ...opts, kind: 'lora' }),
305
- weight: l.weight ?? 1.0,
306
- })),
438
+ cc_by_4_eligible: ccBy4Eligible,
439
+ models: modelHashes,
440
+ loras: loraHashes,
307
441
  };
308
442
  }
@@ -12,8 +12,11 @@ import { join } from 'node:path';
12
12
  * Build a summary object from a manifest.
13
13
  */
14
14
  export function buildRunSummary(manifest) {
15
- const ok = manifest.outputs.filter(o => o.status === 'ok');
16
- const failed = manifest.outputs.filter(o => o.status === 'error');
15
+ // B10: defensive default — manifest.errors two lines below already had
16
+ // one; a legacy/truncated manifest with no `outputs` field crashed here
17
+ // with a bare TypeError instead of degrading to an empty results view.
18
+ const ok = (manifest.outputs || []).filter(o => o.status === 'ok');
19
+ const failed = (manifest.outputs || []).filter(o => o.status === 'error');
17
20
  const totalElapsed = ok.reduce((sum, o) => sum + (o.elapsed_ms || 0), 0);
18
21
 
19
22
  return {
@@ -54,7 +57,7 @@ export function renderRunMarkdown(manifest) {
54
57
  lines.push('');
55
58
  lines.push(`| # | Seed | Status | File | Time |`);
56
59
  lines.push(`|---|------|--------|------|------|`);
57
- for (const out of manifest.outputs) {
60
+ for (const out of (manifest.outputs || [])) {
58
61
  const file = out.filename || '—';
59
62
  const time = out.elapsed_ms ? `${(out.elapsed_ms / 1000).toFixed(1)}s` : '—';
60
63
  const status = out.status === 'ok' ? '✓' : out.status === 'dry_run' ? '○' : '✗';
@@ -98,8 +101,11 @@ export function renderRunMarkdown(manifest) {
98
101
  * Render a manifest to terminal text.
99
102
  */
100
103
  export function renderRunText(manifest) {
101
- const ok = manifest.outputs.filter(o => o.status === 'ok' || o.status === 'dry_run');
102
- const failed = manifest.outputs.filter(o => o.status === 'error');
104
+ // B10: same defensive default as buildRunSummary above — not explicitly
105
+ // named by line number in the finding, but the identical defect in the
106
+ // same file (this is the function `sdlab run show` calls in text mode).
107
+ const ok = (manifest.outputs || []).filter(o => o.status === 'ok' || o.status === 'dry_run');
108
+ const failed = (manifest.outputs || []).filter(o => o.status === 'error');
103
109
  const lines = [];
104
110
 
105
111
  lines.push(`Run: ${manifest.run_id}`);
@@ -111,7 +117,7 @@ export function renderRunText(manifest) {
111
117
  if (failed.length > 0) lines.push(`Errors: ${failed.length}`);
112
118
  lines.push('');
113
119
 
114
- for (const out of manifest.outputs) {
120
+ for (const out of (manifest.outputs || [])) {
115
121
  const status = out.status === 'ok' ? '✓' : out.status === 'dry_run' ? '○' : '✗';
116
122
  const time = out.elapsed_ms ? ` (${(out.elapsed_ms / 1000).toFixed(1)}s)` : '';
117
123
  const file = out.filename ? ` → ${out.filename}` : '';
@@ -11,7 +11,7 @@ import { existsSync, readdirSync, readFileSync } from 'node:fs';
11
11
  import { randomInt } from 'node:crypto';
12
12
  import { getRunsDir, getBriefsDir } from './paths.js';
13
13
  import { inputError } from './errors.js';
14
- import { verbose } from './log.js';
14
+ import { verbose, warn } from './log.js';
15
15
 
16
16
  // ─── Run ID generation ───────────────────────────────────────────
17
17
 
@@ -165,10 +165,28 @@ export async function checkpointRunManifest(runDir, manifest) {
165
165
 
166
166
  // ─── Load / List ─────────────────────────────────────────────────
167
167
 
168
+ // SDL-H7: the only shape a run id can ever legitimately have, mirroring the
169
+ // prefix/date/seq scheme generateRunId() builds above. Validated BEFORE the
170
+ // id is joined into a path, so `--resume ../../../../some/dir/leftover`
171
+ // (or any other traversal payload) never reaches existsSync/readFile. A
172
+ // mismatch is reported as the SAME RUN_NOT_FOUND (not a distinct code)
173
+ // because no real run can ever have an id outside this shape — the two
174
+ // cases are indistinguishable to the caller, and this avoids leaking
175
+ // validation internals to whoever is probing.
176
+ const RUN_ID_RE = /^run_\d{4}-\d{2}-\d{2}_\d{3}$/;
177
+
168
178
  /**
169
179
  * Load a run manifest by ID.
170
180
  */
171
181
  export async function loadRun(projectRoot, runId) {
182
+ if (typeof runId !== 'string' || !RUN_ID_RE.test(runId)) {
183
+ throw inputError(
184
+ 'RUN_NOT_FOUND',
185
+ `Run "${runId}" not found`,
186
+ `Check the run ID or run: sdlab run generate --brief <id>`
187
+ );
188
+ }
189
+
172
190
  const runDir = join(getRunsDir(projectRoot), runId);
173
191
  const manifestPath = join(runDir, 'manifest.json');
174
192
 
@@ -185,6 +203,15 @@ export async function loadRun(projectRoot, runId) {
185
203
 
186
204
  /**
187
205
  * List all runs in a project.
206
+ *
207
+ * B01: this used to `continue` past a directory with no readable
208
+ * manifest.json with zero signal — the exact shape a crashed/killed run
209
+ * leaves behind (see checkpointRunManifest() above). A run that ate 90
210
+ * minutes of GPU time before dying became invisible: no id, no `run show`,
211
+ * no resume, and no indication anything was ever skipped. Skipped/unreadable
212
+ * directories are now reported: a one-line warn() with a count always
213
+ * (so a human running `sdlab run list` cannot miss it), and the per-directory
214
+ * detail under --verbose.
188
215
  */
189
216
  export function listRuns(projectRoot) {
190
217
  const runsDir = getRunsDir(projectRoot);
@@ -195,9 +222,13 @@ export function listRuns(projectRoot) {
195
222
  .sort((a, b) => a.name.localeCompare(b.name));
196
223
 
197
224
  const runs = [];
225
+ const skipped = [];
198
226
  for (const entry of entries) {
199
227
  const manifestPath = join(runsDir, entry.name, 'manifest.json');
200
- if (!existsSync(manifestPath)) continue;
228
+ if (!existsSync(manifestPath)) {
229
+ skipped.push({ name: entry.name, reason: 'no manifest.json' });
230
+ continue;
231
+ }
201
232
 
202
233
  try {
203
234
  const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
@@ -209,8 +240,18 @@ export function listRuns(projectRoot) {
209
240
  created_at: manifest.created_at,
210
241
  adapter_target: manifest.adapter_target,
211
242
  });
212
- } catch {
213
- // Skip malformed
243
+ } catch (err) {
244
+ skipped.push({ name: entry.name, reason: `unreadable manifest.json (${err.message})` });
245
+ }
246
+ }
247
+
248
+ if (skipped.length > 0) {
249
+ warn(
250
+ `listRuns: skipped ${skipped.length} run director${skipped.length === 1 ? 'y' : 'ies'} ` +
251
+ `with no readable manifest.json (run with --verbose for details)`
252
+ );
253
+ for (const s of skipped) {
254
+ verbose(` skipped run dir "${s.name}": ${s.reason}`);
214
255
  }
215
256
  }
216
257