@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.es.md +117 -55
- package/README.fr.md +125 -65
- package/README.hi.md +117 -68
- package/README.it.md +119 -58
- package/README.ja.md +124 -74
- package/README.md +73 -7
- package/README.pt-BR.md +128 -63
- package/README.zh.md +116 -65
- package/bin/sdlab.js +100 -4
- package/lib/adapters/ai-toolkit.js +224 -35
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +94 -11
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/asset-ingest.js +575 -0
- package/lib/asset-masks.js +323 -0
- package/lib/asset-source.js +1176 -0
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-author.js +350 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/canon-review-gate.js +205 -0
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/colab-launcher-guards.js +285 -0
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +16 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +194 -6
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/lane1-render.js +668 -0
- package/lib/measure.js +422 -0
- package/lib/memory-store.js +663 -0
- package/lib/model-licences.js +305 -0
- package/lib/npy-meta.js +130 -0
- package/lib/paths.js +196 -7
- package/lib/png-meta.js +512 -0
- package/lib/records.js +54 -8
- package/lib/reingest-selected.js +113 -17
- package/lib/reingest.js +27 -3
- package/lib/run-manifest.js +159 -25
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/train-runs.js +214 -0
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +3 -0
- package/lib/verify-chain.js +367 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +4 -3
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/asset-ingest.js +160 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-draft.js +82 -0
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +31 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
- package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
- package/scripts/colab/colab_window.sh +587 -0
- package/scripts/colab/lane1_render.py +970 -0
- package/scripts/colab/launch_guards.py +564 -0
- package/scripts/colab/launch_lane1_stub.sh +309 -0
- package/scripts/colab/launch_lane3_stub.sh +366 -0
- package/scripts/colab/remote_runner.py +59 -0
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +182 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +86 -17
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/verify.js +67 -0
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
package/lib/reingest-selected.js
CHANGED
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* canon-binding flow as everything else. No bypass.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { writeFile, mkdir, copyFile } from 'node:fs/promises';
|
|
9
|
+
import { writeFile, mkdir, copyFile, rename, unlink } from 'node:fs/promises';
|
|
10
10
|
import { join, extname } from 'node:path';
|
|
11
|
-
import { existsSync, readFileSync,
|
|
11
|
+
import { existsSync, readFileSync, statSync } from 'node:fs';
|
|
12
12
|
import { getRunsDir } from './paths.js';
|
|
13
13
|
import { getBatchesDir } from './batch-runs.js';
|
|
14
14
|
import { getSelectionsDir, loadSelection } from './selections.js';
|
|
@@ -16,6 +16,35 @@ import { buildGeneratedProvenance } from './generated-provenance.js';
|
|
|
16
16
|
import { buildBaseRecord } from './records.js';
|
|
17
17
|
import { inputError } from './errors.js';
|
|
18
18
|
|
|
19
|
+
// SDL-H7: the only shape a selection id can ever legitimately have,
|
|
20
|
+
// mirroring generateSelectionId() in lib/selections.js (selection_<date>_<seq>,
|
|
21
|
+
// same prefix/date/seq scheme as run/batch ids). Validated BEFORE the id is
|
|
22
|
+
// used anywhere — including before loadSelection(), which does its own
|
|
23
|
+
// unguarded join — so a traversal payload never reaches the filesystem, and
|
|
24
|
+
// before the join at `selectionDir` below, which this module writes
|
|
25
|
+
// provenance.jsonl into. A mismatch reuses loadSelection's own
|
|
26
|
+
// SELECTION_NOT_FOUND shape (not a distinct code) because no real selection
|
|
27
|
+
// can ever have an id outside this shape.
|
|
28
|
+
const SELECTION_ID_RE = /^selection_\d{4}-\d{2}-\d{2}_\d{3}$/;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* B03: atomic write helper — write temp + rename, mirroring the same
|
|
32
|
+
* temp-file-then-rename checkpoint pattern lib/batch-runs.js uses for its
|
|
33
|
+
* per-slot manifest checkpoint. Rename is atomic on a single filesystem, so
|
|
34
|
+
* a crash mid-write can never leave a half-written provenance.jsonl in
|
|
35
|
+
* place of a good one.
|
|
36
|
+
*/
|
|
37
|
+
async function atomicWriteText(targetPath, text) {
|
|
38
|
+
const tmpPath = `${targetPath}.tmp-${process.pid}-${Date.now()}`;
|
|
39
|
+
try {
|
|
40
|
+
await writeFile(tmpPath, text);
|
|
41
|
+
await rename(tmpPath, targetPath);
|
|
42
|
+
} catch (err) {
|
|
43
|
+
try { await unlink(tmpPath); } catch { /* ignore */ }
|
|
44
|
+
throw err;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
19
48
|
/**
|
|
20
49
|
* Re-ingest a selection — creates candidate records from chosen outputs.
|
|
21
50
|
*
|
|
@@ -27,6 +56,14 @@ import { inputError } from './errors.js';
|
|
|
27
56
|
* @returns {Promise<{created: string[], skipped: string[]}>}
|
|
28
57
|
*/
|
|
29
58
|
export async function reingestSelection({ projectRoot, projectId, selectionId, dryRun }) {
|
|
59
|
+
if (typeof selectionId !== 'string' || !SELECTION_ID_RE.test(selectionId)) {
|
|
60
|
+
throw inputError(
|
|
61
|
+
'SELECTION_NOT_FOUND',
|
|
62
|
+
`Selection "${selectionId}" not found`,
|
|
63
|
+
'Check the selection ID or run: sdlab select --run <id> --approve <files>',
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
30
67
|
const manifest = loadSelection(projectRoot, selectionId);
|
|
31
68
|
|
|
32
69
|
if (!manifest.reingest_ready) {
|
|
@@ -69,7 +106,46 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
|
|
|
69
106
|
|
|
70
107
|
const created = [];
|
|
71
108
|
const skipped = [];
|
|
109
|
+
|
|
110
|
+
// B03: provenance.jsonl is now built incrementally — checkpointed (atomic
|
|
111
|
+
// full-rewrite, temp+rename) after every item — instead of accumulated in
|
|
112
|
+
// memory and overwritten once at the end. `provenanceLines` seeds from
|
|
113
|
+
// whatever is ALREADY on disk (e.g. left by an interrupted prior run of
|
|
114
|
+
// this same selection) so: (a) a rerun never duplicates a line for a
|
|
115
|
+
// record it already logged, and (b) a partial rerun can never ERASE lines
|
|
116
|
+
// a previous call already committed — the old end-of-run `writeFile`
|
|
117
|
+
// overwrote with only the CURRENT call's accumulator, so a second call
|
|
118
|
+
// touching fewer items than the first could silently wipe out provenance
|
|
119
|
+
// the first call had already written to disk.
|
|
120
|
+
const provenancePath = join(selectionDir, 'provenance.jsonl');
|
|
72
121
|
const provenanceLines = [];
|
|
122
|
+
const seenProvenanceIds = new Set();
|
|
123
|
+
if (existsSync(provenancePath)) {
|
|
124
|
+
for (const line of readFileSync(provenancePath, 'utf-8').split('\n')) {
|
|
125
|
+
if (!line.trim()) continue;
|
|
126
|
+
provenanceLines.push(line);
|
|
127
|
+
try {
|
|
128
|
+
const parsed = JSON.parse(line);
|
|
129
|
+
if (parsed?.record_id) seenProvenanceIds.add(parsed.record_id);
|
|
130
|
+
} catch {
|
|
131
|
+
// Tolerate a corrupt/partial trailing line from a prior crash —
|
|
132
|
+
// keep it verbatim rather than dropping it, just don't index it.
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Checkpoint one provenance line: skips dryRun, and skips a record_id
|
|
139
|
+
* that's already logged (idempotent — safe to call on both the fresh-write
|
|
140
|
+
* path and the skip/backfill path below).
|
|
141
|
+
*/
|
|
142
|
+
async function checkpointProvenance(recordId, prov) {
|
|
143
|
+
if (dryRun) return;
|
|
144
|
+
if (seenProvenanceIds.has(recordId)) return;
|
|
145
|
+
provenanceLines.push(JSON.stringify({ record_id: recordId, ...prov }));
|
|
146
|
+
seenProvenanceIds.add(recordId);
|
|
147
|
+
await atomicWriteText(provenancePath, provenanceLines.join('\n') + '\n');
|
|
148
|
+
}
|
|
73
149
|
|
|
74
150
|
// D-010: use entries() — not indexOf() — to avoid O(n^2) behavior and
|
|
75
151
|
// silent duplicate-ID bugs when manifest.items contains structurally
|
|
@@ -88,12 +164,11 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
|
|
|
88
164
|
seenIds.add(recordId);
|
|
89
165
|
const recordPath = join(recordsDir, `${recordId}.json`);
|
|
90
166
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
// Resolve the run manifest for this item
|
|
167
|
+
// Resolve the run manifest for this item + build its provenance BEFORE
|
|
168
|
+
// the existsSync check below — B03: a skipped (already-created) item
|
|
169
|
+
// still needs its provenance backfilled if an earlier interrupted run
|
|
170
|
+
// stopped short of appending it, so this can't be gated behind "the
|
|
171
|
+
// record is new".
|
|
97
172
|
let runManifest;
|
|
98
173
|
if (manifest.source_type === 'run') {
|
|
99
174
|
runManifest = getRunManifest(manifest.source_id);
|
|
@@ -124,8 +199,19 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
|
|
|
124
199
|
batchManifest,
|
|
125
200
|
item,
|
|
126
201
|
projectRoot,
|
|
202
|
+
canonSha: manifest.canon_sha ?? null,
|
|
127
203
|
});
|
|
128
204
|
|
|
205
|
+
if (existsSync(recordPath)) {
|
|
206
|
+
// B03: the record already landed on disk (this item was completed by
|
|
207
|
+
// a prior, possibly-interrupted run of this same re-ingest). Backfill
|
|
208
|
+
// its provenance line if that prior run never got to append one —
|
|
209
|
+
// never let a record exist with no corresponding provenance entry.
|
|
210
|
+
await checkpointProvenance(recordId, prov);
|
|
211
|
+
skipped.push(recordId);
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
|
|
129
215
|
// Image source in selection/chosen/
|
|
130
216
|
const srcImage = join(selectionDir, 'chosen', item.filename);
|
|
131
217
|
const ext = extname(item.filename) || '.png';
|
|
@@ -139,6 +225,15 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
|
|
|
139
225
|
fileBytes = statSync(srcImage).size;
|
|
140
226
|
}
|
|
141
227
|
|
|
228
|
+
const canonBlock =
|
|
229
|
+
prov.canon_sha || prov.entry_id || prov.entry_sha256
|
|
230
|
+
? {
|
|
231
|
+
canon_sha: prov.canon_sha ?? null,
|
|
232
|
+
entry_id: prov.entry_id ?? null,
|
|
233
|
+
entry_sha256: prov.entry_sha256 ?? null,
|
|
234
|
+
}
|
|
235
|
+
: null;
|
|
236
|
+
|
|
142
237
|
// Build record
|
|
143
238
|
const record = buildBaseRecord(recordId, assetPath, {
|
|
144
239
|
width: null,
|
|
@@ -148,9 +243,13 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
|
|
|
148
243
|
source: 'generated',
|
|
149
244
|
selection_id: selectionId,
|
|
150
245
|
generation_provenance: prov,
|
|
246
|
+
}, {
|
|
247
|
+
canon: canonBlock,
|
|
151
248
|
});
|
|
152
249
|
|
|
153
250
|
record.schema_version = '2.2.0';
|
|
251
|
+
record.generation_provenance = prov;
|
|
252
|
+
if (projectId) record.project_id = projectId;
|
|
154
253
|
|
|
155
254
|
// Tags
|
|
156
255
|
const recordTags = ['generated', 'selected'];
|
|
@@ -166,18 +265,15 @@ export async function reingestSelection({ projectRoot, projectId, selectionId, d
|
|
|
166
265
|
}
|
|
167
266
|
// Write record
|
|
168
267
|
await writeFile(recordPath, JSON.stringify(record, null, 2) + '\n');
|
|
268
|
+
// B03: checkpoint immediately after the record write (per-item,
|
|
269
|
+
// matching lib/batch-runs.js's per-slot checkpoint pattern) — NOT
|
|
270
|
+
// batched to the end of the run. A crash on item 120 of 200 must
|
|
271
|
+
// leave provenance for 1-119 durably on disk, not lose it because
|
|
272
|
+
// item 200 never finished.
|
|
273
|
+
await checkpointProvenance(recordId, prov);
|
|
169
274
|
}
|
|
170
275
|
|
|
171
276
|
created.push(recordId);
|
|
172
|
-
provenanceLines.push(JSON.stringify({ record_id: recordId, ...prov }));
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
// Write provenance log
|
|
176
|
-
if (!dryRun && provenanceLines.length > 0) {
|
|
177
|
-
await writeFile(
|
|
178
|
-
join(selectionDir, 'provenance.jsonl'),
|
|
179
|
-
provenanceLines.join('\n') + '\n',
|
|
180
|
-
);
|
|
181
277
|
}
|
|
182
278
|
|
|
183
279
|
return { created, skipped };
|
package/lib/reingest.js
CHANGED
|
@@ -12,6 +12,8 @@ import { join, basename, extname } from 'node:path';
|
|
|
12
12
|
import { existsSync } from 'node:fs';
|
|
13
13
|
import { loadTrainingManifest } from './training-manifests.js';
|
|
14
14
|
import { buildBaseRecord } from './records.js';
|
|
15
|
+
import { resolveCanonLink } from './generated-provenance.js';
|
|
16
|
+
import { resolveModelLicence } from './model-licences.js';
|
|
15
17
|
|
|
16
18
|
const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
|
|
17
19
|
|
|
@@ -22,12 +24,12 @@ const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
|
|
|
22
24
|
* - provenance.source = "generated"
|
|
23
25
|
* - provenance.training_manifest_id = <manifest-id>
|
|
24
26
|
* - judgment = null (must be reviewed)
|
|
25
|
-
* - canon =
|
|
27
|
+
* - canon = resolved canon link when canon-build is present, else null
|
|
26
28
|
*
|
|
27
29
|
* @param {string} projectRoot
|
|
28
30
|
* @param {string} sourcePath — directory containing generated images
|
|
29
31
|
* @param {string} manifestId — training manifest that produced them
|
|
30
|
-
* @param {Object} options — { dryRun? }
|
|
32
|
+
* @param {Object} options — { dryRun?, canonSha?, entryId? }
|
|
31
33
|
* @returns {Promise<{created: number, skipped: number, records: string[]}>}
|
|
32
34
|
*/
|
|
33
35
|
export async function reingestGenerated(projectRoot, sourcePath, manifestId, options = {}) {
|
|
@@ -66,12 +68,32 @@ export async function reingestGenerated(projectRoot, sourcePath, manifestId, opt
|
|
|
66
68
|
// Relative asset path from project root
|
|
67
69
|
const assetPath = `outputs/candidates/${file}`;
|
|
68
70
|
|
|
71
|
+
const canonLink = resolveCanonLink(projectRoot, {
|
|
72
|
+
canonSha: options.canonSha || manifest.canon_sha || null,
|
|
73
|
+
entryId: options.entryId || stem,
|
|
74
|
+
});
|
|
75
|
+
const baseLic = manifest.base_model ? resolveModelLicence(manifest.base_model) : null;
|
|
76
|
+
|
|
69
77
|
const provenance = {
|
|
70
78
|
source: 'generated',
|
|
71
79
|
training_manifest_id: manifestId,
|
|
72
80
|
training_profile_id: manifest.training_profile_id,
|
|
73
81
|
adapter_target: manifest.adapter_target,
|
|
74
82
|
base_model: manifest.base_model || null,
|
|
83
|
+
...(baseLic
|
|
84
|
+
? {
|
|
85
|
+
licence_id: baseLic.licence_id,
|
|
86
|
+
licence_url: baseLic.licence_url,
|
|
87
|
+
date_read: baseLic.date_read,
|
|
88
|
+
}
|
|
89
|
+
: {}),
|
|
90
|
+
...(canonLink
|
|
91
|
+
? {
|
|
92
|
+
canon_sha: canonLink.canon_sha,
|
|
93
|
+
entry_id: canonLink.entry_id,
|
|
94
|
+
entry_sha256: canonLink.entry_sha256,
|
|
95
|
+
}
|
|
96
|
+
: {}),
|
|
75
97
|
source_export_id: manifest.source_export_id,
|
|
76
98
|
generated_at: new Date().toISOString(),
|
|
77
99
|
original_path: filePath,
|
|
@@ -81,7 +103,9 @@ export async function reingestGenerated(projectRoot, sourcePath, manifestId, opt
|
|
|
81
103
|
width: null,
|
|
82
104
|
height: null,
|
|
83
105
|
bytes: fileStat.size,
|
|
84
|
-
}, provenance
|
|
106
|
+
}, provenance, {
|
|
107
|
+
canon: canonLink,
|
|
108
|
+
});
|
|
85
109
|
|
|
86
110
|
record.schema_version = '2.1.0';
|
|
87
111
|
|
package/lib/run-manifest.js
CHANGED
|
@@ -25,6 +25,20 @@
|
|
|
25
25
|
import { createHash } from 'node:crypto';
|
|
26
26
|
import { openSync, readSync, closeSync, statSync, existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
27
27
|
import { join } from 'node:path';
|
|
28
|
+
import { warn } from './log.js';
|
|
29
|
+
import {
|
|
30
|
+
CLEARED_SPDX_IDS,
|
|
31
|
+
resolveModelLicence,
|
|
32
|
+
isClearedLicenceEntry,
|
|
33
|
+
assertClearedPinningLicences,
|
|
34
|
+
} from './model-licences.js';
|
|
35
|
+
|
|
36
|
+
export {
|
|
37
|
+
CLEARED_SPDX_IDS,
|
|
38
|
+
resolveModelLicence,
|
|
39
|
+
isClearedLicenceEntry,
|
|
40
|
+
assertClearedPinningLicences,
|
|
41
|
+
};
|
|
28
42
|
|
|
29
43
|
/**
|
|
30
44
|
* Contract version for the `pinning` block itself. Bump when the block's shape
|
|
@@ -35,6 +49,16 @@ import { join } from 'node:path';
|
|
|
35
49
|
*/
|
|
36
50
|
export const PINNING_VERSION = '1.0.0';
|
|
37
51
|
|
|
52
|
+
/**
|
|
53
|
+
* The ONE hash_note value that means "hashing was never attempted because
|
|
54
|
+
* --hash-models wasn't passed" — NOT a failure. buildPinning() (B07) needs
|
|
55
|
+
* to tell this apart from every other hash_note (unresolved/stat
|
|
56
|
+
* failed/read error/no filename), all of which ARE failures worth warning
|
|
57
|
+
* about. Shared as a constant so hashModelFile's own note and buildPinning's
|
|
58
|
+
* filter can never drift apart.
|
|
59
|
+
*/
|
|
60
|
+
const HASH_SKIPPED_NOTE = 'hash skipped (pass --hash-models)';
|
|
61
|
+
|
|
38
62
|
/**
|
|
39
63
|
* Seed policies — the INTENT behind seed selection, recorded alongside the
|
|
40
64
|
* concrete per-item seed values so a reader knows how to regenerate.
|
|
@@ -144,17 +168,24 @@ const KIND_SUBDIRS = {
|
|
|
144
168
|
|
|
145
169
|
/**
|
|
146
170
|
* Resolve a model filename to an absolute path under the models dir. Searches
|
|
147
|
-
* the kind-appropriate subdirs
|
|
148
|
-
*
|
|
171
|
+
* ONLY the kind-appropriate subdirs (plus the modelsDir root itself, which is
|
|
172
|
+
* kind-agnostic, not another kind's territory) — mirroring ComfyUI's own
|
|
173
|
+
* loaders, which never search across kinds (a UNETLoader never looks in
|
|
174
|
+
* checkpoints/, a VAELoader never looks in loras/, etc).
|
|
175
|
+
*
|
|
176
|
+
* SDL-M8: this used to also fall back to every OTHER kind's subdirs, so a
|
|
177
|
+
* missing unet could silently resolve to a same-named file under
|
|
178
|
+
* checkpoints/ and get pinned as though it were the unet ComfyUI actually
|
|
179
|
+
* loaded — a clean-looking wrong identity. An honest `null` (unresolved →
|
|
180
|
+
* sha256 null + a note, see hashModelFile) is the correct outcome when the
|
|
181
|
+
* kind-specific subdirs don't have the file.
|
|
182
|
+
*
|
|
183
|
+
* Returns null when nothing matches.
|
|
149
184
|
*/
|
|
150
185
|
export function resolveModelPath(name, kind, modelsDir = defaultModelsDir()) {
|
|
151
186
|
if (!name || !modelsDir) return null;
|
|
187
|
+
const subdirs = [...(KIND_SUBDIRS[kind] || []), '.'];
|
|
152
188
|
const tried = new Set();
|
|
153
|
-
const subdirs = [
|
|
154
|
-
...(KIND_SUBDIRS[kind] || []),
|
|
155
|
-
...Object.values(KIND_SUBDIRS).flat(),
|
|
156
|
-
'.',
|
|
157
|
-
];
|
|
158
189
|
for (const sub of subdirs) {
|
|
159
190
|
const candidate = join(modelsDir, sub, name);
|
|
160
191
|
if (tried.has(candidate)) continue;
|
|
@@ -183,12 +214,63 @@ export function sha256File(path) {
|
|
|
183
214
|
return 'sha256:' + hash.digest('hex');
|
|
184
215
|
}
|
|
185
216
|
|
|
217
|
+
// SDL-M9: a cheap partial-content fingerprint folded into the cache key
|
|
218
|
+
// alongside name/size/mtime. The key used to be bare name:size:round(mtime)
|
|
219
|
+
// with no content check — on a coarse-mtime mount (network shares, some
|
|
220
|
+
// FAT/exFAT mounts), a same-size in-place replacement (e.g. swapping a
|
|
221
|
+
// LoRA) can round to the SAME mtime as the file it replaced, so the cache
|
|
222
|
+
// would silently serve the sha256 of the OLD content forever. Sampling the
|
|
223
|
+
// first/last few KB — not hashing the whole multi-GB file, which would
|
|
224
|
+
// defeat the point of caching — is enough to catch almost any real
|
|
225
|
+
// replacement while staying cheap on every call (including cache hits).
|
|
226
|
+
const SAMPLE_BYTES = 4096;
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Cheap content fingerprint: sha256 of the first + last SAMPLE_BYTES of the
|
|
230
|
+
* file (or the whole file if smaller). Synchronous, bounded I/O — a few KB
|
|
231
|
+
* regardless of file size, unlike sha256File's full-content streaming read.
|
|
232
|
+
*/
|
|
233
|
+
function sampleFingerprint(path, size) {
|
|
234
|
+
const fd = openSync(path, 'r');
|
|
235
|
+
try {
|
|
236
|
+
const hash = createHash('sha256');
|
|
237
|
+
const headLen = Math.min(SAMPLE_BYTES, size);
|
|
238
|
+
if (headLen > 0) {
|
|
239
|
+
const head = Buffer.allocUnsafe(headLen);
|
|
240
|
+
readSync(fd, head, 0, headLen, 0);
|
|
241
|
+
hash.update(head);
|
|
242
|
+
}
|
|
243
|
+
if (size > SAMPLE_BYTES) {
|
|
244
|
+
const tailLen = Math.min(SAMPLE_BYTES, size - headLen);
|
|
245
|
+
const tail = Buffer.allocUnsafe(tailLen);
|
|
246
|
+
readSync(fd, tail, 0, tailLen, size - tailLen);
|
|
247
|
+
hash.update(tail);
|
|
248
|
+
}
|
|
249
|
+
return hash.digest('hex').slice(0, 16);
|
|
250
|
+
} finally {
|
|
251
|
+
closeSync(fd);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function withLicenceFields(base, name, { licence = null, modelLicences = null } = {}) {
|
|
256
|
+
const lic = resolveModelLicence(name, licence, modelLicences);
|
|
257
|
+
return {
|
|
258
|
+
...base,
|
|
259
|
+
licence_id: lic.licence_id,
|
|
260
|
+
licence_url: lic.licence_url,
|
|
261
|
+
date_read: lic.date_read,
|
|
262
|
+
licence: lic,
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
186
266
|
/**
|
|
187
267
|
* Best-effort content identity for one model/LoRA file: {name, size_bytes, sha256}
|
|
188
268
|
* (+ a short `hash_note` when sha256 can't be produced). size_bytes is always
|
|
189
269
|
* filled when the file resolves (a cheap stat); sha256 is gated behind
|
|
190
|
-
* `doHash` (the --hash-models opt-in) and cached by (name, size, mtime
|
|
191
|
-
* multi-GB checkpoints aren't re-hashed every
|
|
270
|
+
* `doHash` (the --hash-models opt-in) and cached by (name, size, mtime,
|
|
271
|
+
* partial-content sample) so multi-GB checkpoints aren't re-hashed every
|
|
272
|
+
* wave (SDL-M9: the sample is what keeps a same-size in-place replacement
|
|
273
|
+
* from reusing a stale cache entry).
|
|
192
274
|
*
|
|
193
275
|
* @param {string} name — model filename as it appears in the graph
|
|
194
276
|
* @param {Object} [opts]
|
|
@@ -197,10 +279,10 @@ export function sha256File(path) {
|
|
|
197
279
|
* @param {boolean} [opts.doHash=false] — compute sha256 (else record name+size only)
|
|
198
280
|
* @param {{map: Map, cacheFile?: string, dirty: boolean}} [opts.cache]
|
|
199
281
|
*/
|
|
200
|
-
export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHash = false, cache } = {}) {
|
|
201
|
-
if (!name) return { name: name ?? null, size_bytes: null, sha256: null, hash_note: 'no filename' };
|
|
282
|
+
export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHash = false, cache, licence = null, modelLicences = null } = {}) {
|
|
283
|
+
if (!name) return withLicenceFields({ name: name ?? null, size_bytes: null, sha256: null, hash_note: 'no filename' }, name, { licence, modelLicences });
|
|
202
284
|
const resolved = resolveModelPath(name, kind, modelsDir);
|
|
203
|
-
if (!resolved) return { name, size_bytes: null, sha256: null, hash_note: `unresolved under ${modelsDir}` };
|
|
285
|
+
if (!resolved) return withLicenceFields({ name, size_bytes: null, sha256: null, hash_note: `unresolved under ${modelsDir}` }, name, { licence, modelLicences });
|
|
204
286
|
let size_bytes = null;
|
|
205
287
|
let mtimeMs = 0;
|
|
206
288
|
try {
|
|
@@ -208,22 +290,30 @@ export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHa
|
|
|
208
290
|
size_bytes = st.size;
|
|
209
291
|
mtimeMs = st.mtimeMs;
|
|
210
292
|
} catch {
|
|
211
|
-
return { name, size_bytes: null, sha256: null, hash_note: 'stat failed' };
|
|
293
|
+
return withLicenceFields({ name, size_bytes: null, sha256: null, hash_note: 'stat failed' }, name, { licence, modelLicences });
|
|
294
|
+
}
|
|
295
|
+
if (!doHash) return withLicenceFields({ name, size_bytes, sha256: null, hash_note: HASH_SKIPPED_NOTE }, name, { licence, modelLicences });
|
|
296
|
+
let sample = '';
|
|
297
|
+
try {
|
|
298
|
+
sample = sampleFingerprint(resolved, size_bytes);
|
|
299
|
+
} catch {
|
|
300
|
+
// Best-effort — if sampling fails, fall back to the coarser
|
|
301
|
+
// name:size:mtime key; the sha256File() attempt below will surface any
|
|
302
|
+
// real read error anyway.
|
|
212
303
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
if (cache?.map?.has(key)) return { name, size_bytes, sha256: cache.map.get(key) };
|
|
304
|
+
const key = `${name}:${size_bytes}:${Math.round(mtimeMs)}:${sample}`;
|
|
305
|
+
if (cache?.map?.has(key)) return withLicenceFields({ name, size_bytes, sha256: cache.map.get(key) }, name, { licence, modelLicences });
|
|
216
306
|
let sha256;
|
|
217
307
|
try {
|
|
218
308
|
sha256 = sha256File(resolved);
|
|
219
309
|
} catch {
|
|
220
|
-
return { name, size_bytes, sha256: null, hash_note: 'hash failed (read error)' };
|
|
310
|
+
return withLicenceFields({ name, size_bytes, sha256: null, hash_note: 'hash failed (read error)' }, name, { licence, modelLicences });
|
|
221
311
|
}
|
|
222
312
|
if (cache?.map) {
|
|
223
313
|
cache.map.set(key, sha256);
|
|
224
314
|
cache.dirty = true;
|
|
225
315
|
}
|
|
226
|
-
return { name, size_bytes, sha256 };
|
|
316
|
+
return withLicenceFields({ name, size_bytes, sha256 }, name, { licence, modelLicences });
|
|
227
317
|
}
|
|
228
318
|
|
|
229
319
|
/**
|
|
@@ -291,18 +381,62 @@ export function flushHashCache(cache) {
|
|
|
291
381
|
* @param {boolean} [args.hashModels=false] — opt-in sha256 of model/LoRA files
|
|
292
382
|
* @param {string} [args.modelsDir]
|
|
293
383
|
* @param {{map: Map, cacheFile?: string, dirty: boolean}} [args.cache]
|
|
384
|
+
* @param {Object} [args.modelLicences]
|
|
385
|
+
* @param {boolean} [args.enforceLicence=false]
|
|
386
|
+
* @param {boolean} [args.allowUnlicensedResearch=false]
|
|
294
387
|
* @returns {Object} pinning block
|
|
295
388
|
*/
|
|
296
|
-
export function buildPinning({
|
|
297
|
-
|
|
389
|
+
export function buildPinning({
|
|
390
|
+
graph,
|
|
391
|
+
models = {},
|
|
392
|
+
loras = [],
|
|
393
|
+
seedPolicy = null,
|
|
394
|
+
hashModels = false,
|
|
395
|
+
modelsDir = defaultModelsDir(),
|
|
396
|
+
cache,
|
|
397
|
+
modelLicences = null,
|
|
398
|
+
enforceLicence = false,
|
|
399
|
+
allowUnlicensedResearch = false,
|
|
400
|
+
} = {}) {
|
|
401
|
+
const opts = { modelsDir, doHash: hashModels, cache, modelLicences };
|
|
402
|
+
const modelHashes = buildModelHashes(models, opts);
|
|
403
|
+
const loraHashes = (loras || []).map((l) => ({
|
|
404
|
+
...hashModelFile(l.name, { ...opts, kind: 'lora', licence: l.licence || null }),
|
|
405
|
+
weight: l.weight ?? 1.0,
|
|
406
|
+
}));
|
|
407
|
+
|
|
408
|
+
if (enforceLicence) {
|
|
409
|
+
assertClearedPinningLicences(modelHashes, loraHashes, { allowUnlicensedResearch });
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
// B07: hash_note silently encoded a hashing FAILURE (unresolved file,
|
|
413
|
+
// stat failed, read error, no filename) with nothing reading it —
|
|
414
|
+
// buildPinning() never aggregated or logged, and this file imported no
|
|
415
|
+
// logger at all. A typo'd SDLAB_MODELS_DIR produced a wave where every
|
|
416
|
+
// model has sha256: null + a note, with zero console signal — PIN_PER_STEP's
|
|
417
|
+
// whole purpose (byte-for-byte replayability) lost silently. Warn ONCE per
|
|
418
|
+
// wave with a count, but never on HASH_SKIPPED_NOTE — that note means
|
|
419
|
+
// "hashing wasn't requested" (the default, --hash-models omitted), which
|
|
420
|
+
// is expected behavior, not a failure, and would make every ordinary run
|
|
421
|
+
// noisy if it warned too.
|
|
422
|
+
const allEntries = [...Object.values(modelHashes), ...loraHashes];
|
|
423
|
+
const failed = allEntries.filter((e) => e && e.hash_note != null && e.hash_note !== HASH_SKIPPED_NOTE);
|
|
424
|
+
if (failed.length > 0) {
|
|
425
|
+
const detail = failed.map((e) => `${e.name ?? '(unnamed)'} (${e.hash_note})`).join(', ');
|
|
426
|
+
warn(`buildPinning: ${failed.length}/${allEntries.length} model(s)/lora(s) failed to hash: ${detail}`);
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
const ccBy4Eligible =
|
|
430
|
+
!allowUnlicensedResearch &&
|
|
431
|
+
allEntries.length > 0 &&
|
|
432
|
+
allEntries.every((entry) => isClearedLicenceEntry(entry));
|
|
433
|
+
|
|
298
434
|
return {
|
|
299
435
|
pinning_version: PINNING_VERSION,
|
|
300
436
|
comfy_workflow_sha: comfyWorkflowSha(graph),
|
|
301
437
|
seed_policy: seedPolicy,
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
weight: l.weight ?? 1.0,
|
|
306
|
-
})),
|
|
438
|
+
cc_by_4_eligible: ccBy4Eligible,
|
|
439
|
+
models: modelHashes,
|
|
440
|
+
loras: loraHashes,
|
|
307
441
|
};
|
|
308
442
|
}
|
package/lib/run-summary.js
CHANGED
|
@@ -12,8 +12,11 @@ import { join } from 'node:path';
|
|
|
12
12
|
* Build a summary object from a manifest.
|
|
13
13
|
*/
|
|
14
14
|
export function buildRunSummary(manifest) {
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
// B10: defensive default — manifest.errors two lines below already had
|
|
16
|
+
// one; a legacy/truncated manifest with no `outputs` field crashed here
|
|
17
|
+
// with a bare TypeError instead of degrading to an empty results view.
|
|
18
|
+
const ok = (manifest.outputs || []).filter(o => o.status === 'ok');
|
|
19
|
+
const failed = (manifest.outputs || []).filter(o => o.status === 'error');
|
|
17
20
|
const totalElapsed = ok.reduce((sum, o) => sum + (o.elapsed_ms || 0), 0);
|
|
18
21
|
|
|
19
22
|
return {
|
|
@@ -54,7 +57,7 @@ export function renderRunMarkdown(manifest) {
|
|
|
54
57
|
lines.push('');
|
|
55
58
|
lines.push(`| # | Seed | Status | File | Time |`);
|
|
56
59
|
lines.push(`|---|------|--------|------|------|`);
|
|
57
|
-
for (const out of manifest.outputs) {
|
|
60
|
+
for (const out of (manifest.outputs || [])) {
|
|
58
61
|
const file = out.filename || '—';
|
|
59
62
|
const time = out.elapsed_ms ? `${(out.elapsed_ms / 1000).toFixed(1)}s` : '—';
|
|
60
63
|
const status = out.status === 'ok' ? '✓' : out.status === 'dry_run' ? '○' : '✗';
|
|
@@ -98,8 +101,11 @@ export function renderRunMarkdown(manifest) {
|
|
|
98
101
|
* Render a manifest to terminal text.
|
|
99
102
|
*/
|
|
100
103
|
export function renderRunText(manifest) {
|
|
101
|
-
|
|
102
|
-
|
|
104
|
+
// B10: same defensive default as buildRunSummary above — not explicitly
|
|
105
|
+
// named by line number in the finding, but the identical defect in the
|
|
106
|
+
// same file (this is the function `sdlab run show` calls in text mode).
|
|
107
|
+
const ok = (manifest.outputs || []).filter(o => o.status === 'ok' || o.status === 'dry_run');
|
|
108
|
+
const failed = (manifest.outputs || []).filter(o => o.status === 'error');
|
|
103
109
|
const lines = [];
|
|
104
110
|
|
|
105
111
|
lines.push(`Run: ${manifest.run_id}`);
|
|
@@ -111,7 +117,7 @@ export function renderRunText(manifest) {
|
|
|
111
117
|
if (failed.length > 0) lines.push(`Errors: ${failed.length}`);
|
|
112
118
|
lines.push('');
|
|
113
119
|
|
|
114
|
-
for (const out of manifest.outputs) {
|
|
120
|
+
for (const out of (manifest.outputs || [])) {
|
|
115
121
|
const status = out.status === 'ok' ? '✓' : out.status === 'dry_run' ? '○' : '✗';
|
|
116
122
|
const time = out.elapsed_ms ? ` (${(out.elapsed_ms / 1000).toFixed(1)}s)` : '';
|
|
117
123
|
const file = out.filename ? ` → ${out.filename}` : '';
|
package/lib/runtime-runs.js
CHANGED
|
@@ -11,7 +11,7 @@ import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
|
|
11
11
|
import { randomInt } from 'node:crypto';
|
|
12
12
|
import { getRunsDir, getBriefsDir } from './paths.js';
|
|
13
13
|
import { inputError } from './errors.js';
|
|
14
|
-
import { verbose } from './log.js';
|
|
14
|
+
import { verbose, warn } from './log.js';
|
|
15
15
|
|
|
16
16
|
// ─── Run ID generation ───────────────────────────────────────────
|
|
17
17
|
|
|
@@ -165,10 +165,28 @@ export async function checkpointRunManifest(runDir, manifest) {
|
|
|
165
165
|
|
|
166
166
|
// ─── Load / List ─────────────────────────────────────────────────
|
|
167
167
|
|
|
168
|
+
// SDL-H7: the only shape a run id can ever legitimately have, mirroring the
|
|
169
|
+
// prefix/date/seq scheme generateRunId() builds above. Validated BEFORE the
|
|
170
|
+
// id is joined into a path, so `--resume ../../../../some/dir/leftover`
|
|
171
|
+
// (or any other traversal payload) never reaches existsSync/readFile. A
|
|
172
|
+
// mismatch is reported as the SAME RUN_NOT_FOUND (not a distinct code)
|
|
173
|
+
// because no real run can ever have an id outside this shape — the two
|
|
174
|
+
// cases are indistinguishable to the caller, and this avoids leaking
|
|
175
|
+
// validation internals to whoever is probing.
|
|
176
|
+
const RUN_ID_RE = /^run_\d{4}-\d{2}-\d{2}_\d{3}$/;
|
|
177
|
+
|
|
168
178
|
/**
|
|
169
179
|
* Load a run manifest by ID.
|
|
170
180
|
*/
|
|
171
181
|
export async function loadRun(projectRoot, runId) {
|
|
182
|
+
if (typeof runId !== 'string' || !RUN_ID_RE.test(runId)) {
|
|
183
|
+
throw inputError(
|
|
184
|
+
'RUN_NOT_FOUND',
|
|
185
|
+
`Run "${runId}" not found`,
|
|
186
|
+
`Check the run ID or run: sdlab run generate --brief <id>`
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
172
190
|
const runDir = join(getRunsDir(projectRoot), runId);
|
|
173
191
|
const manifestPath = join(runDir, 'manifest.json');
|
|
174
192
|
|
|
@@ -185,6 +203,15 @@ export async function loadRun(projectRoot, runId) {
|
|
|
185
203
|
|
|
186
204
|
/**
|
|
187
205
|
* List all runs in a project.
|
|
206
|
+
*
|
|
207
|
+
* B01: this used to `continue` past a directory with no readable
|
|
208
|
+
* manifest.json with zero signal — the exact shape a crashed/killed run
|
|
209
|
+
* leaves behind (see checkpointRunManifest() above). A run that ate 90
|
|
210
|
+
* minutes of GPU time before dying became invisible: no id, no `run show`,
|
|
211
|
+
* no resume, and no indication anything was ever skipped. Skipped/unreadable
|
|
212
|
+
* directories are now reported: a one-line warn() with a count always
|
|
213
|
+
* (so a human running `sdlab run list` cannot miss it), and the per-directory
|
|
214
|
+
* detail under --verbose.
|
|
188
215
|
*/
|
|
189
216
|
export function listRuns(projectRoot) {
|
|
190
217
|
const runsDir = getRunsDir(projectRoot);
|
|
@@ -195,9 +222,13 @@ export function listRuns(projectRoot) {
|
|
|
195
222
|
.sort((a, b) => a.name.localeCompare(b.name));
|
|
196
223
|
|
|
197
224
|
const runs = [];
|
|
225
|
+
const skipped = [];
|
|
198
226
|
for (const entry of entries) {
|
|
199
227
|
const manifestPath = join(runsDir, entry.name, 'manifest.json');
|
|
200
|
-
if (!existsSync(manifestPath))
|
|
228
|
+
if (!existsSync(manifestPath)) {
|
|
229
|
+
skipped.push({ name: entry.name, reason: 'no manifest.json' });
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
201
232
|
|
|
202
233
|
try {
|
|
203
234
|
const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
@@ -209,8 +240,18 @@ export function listRuns(projectRoot) {
|
|
|
209
240
|
created_at: manifest.created_at,
|
|
210
241
|
adapter_target: manifest.adapter_target,
|
|
211
242
|
});
|
|
212
|
-
} catch {
|
|
213
|
-
|
|
243
|
+
} catch (err) {
|
|
244
|
+
skipped.push({ name: entry.name, reason: `unreadable manifest.json (${err.message})` });
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
if (skipped.length > 0) {
|
|
249
|
+
warn(
|
|
250
|
+
`listRuns: skipped ${skipped.length} run director${skipped.length === 1 ? 'y' : 'ies'} ` +
|
|
251
|
+
`with no readable manifest.json (run with --verbose for details)`
|
|
252
|
+
);
|
|
253
|
+
for (const s of skipped) {
|
|
254
|
+
verbose(` skipped run dir "${s.name}": ${s.reason}`);
|
|
214
255
|
}
|
|
215
256
|
}
|
|
216
257
|
|