@mcptoolshop/style-dataset-lab 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -1
- package/README.es.md +16 -5
- package/README.fr.md +54 -43
- package/README.hi.md +47 -36
- package/README.it.md +14 -3
- package/README.ja.md +16 -5
- package/README.md +11 -0
- package/README.pt-BR.md +16 -5
- package/README.zh.md +49 -38
- package/lib/adapters/comfyui-runner.js +36 -2
- package/lib/critique-engine.js +46 -1
- package/lib/critique-render.js +45 -0
- package/lib/freeze-gate.js +56 -13
- package/lib/run-manifest.js +308 -0
- package/lib/snapshot.js +7 -1
- package/lib/verifier.js +67 -0
- package/package.json +1 -1
- package/scripts/critique.js +32 -7
- package/scripts/curate.js +21 -0
- package/scripts/generate.js +24 -0
- package/scripts/qwen_generate.py +203 -0
- package/scripts/run-generate.js +3 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run-manifest pinning contract (PIN_PER_STEP).
|
|
3
|
+
*
|
|
4
|
+
* ONE source of truth for the fields that make a generation wave byte-for-byte
|
|
5
|
+
* replayable. Three writers emit provenance today — `scripts/generate.js`
|
|
6
|
+
* (record `provenance`), `lib/adapters/comfyui-runner.js` (run `manifest.json`),
|
|
7
|
+
* and `scripts/qwen_generate.py` (the wave `generation.json` receipt) — and they
|
|
8
|
+
* MUST all pin the same way. This module is the field-name authority; the
|
|
9
|
+
* Python receipt mirrors the identical field contract (see qwen_generate.py).
|
|
10
|
+
*
|
|
11
|
+
* Design (DECOMPOSE_BY_SECRETS, Parnas 1972): the pinning field names + the
|
|
12
|
+
* best-effort model hasher live here so the writers can never drift apart
|
|
13
|
+
* silently. When the model changes, this secret-hiding boundary survives.
|
|
14
|
+
*
|
|
15
|
+
* The pinning block closes three PIN_PER_STEP gaps that the pre-3.3.0 manifests
|
|
16
|
+
* left open (base/sampler/scheduler/shift/seed/steps/cfg were already recorded):
|
|
17
|
+
* 1. comfy_workflow_sha — the exact ComfyUI graph that ran, as a hash.
|
|
18
|
+
* 2. model/LoRA content identity — filenames upgraded to {name,size_bytes,sha256}.
|
|
19
|
+
* 3. seed_policy — HOW seeds were chosen (intent), not just the values.
|
|
20
|
+
*
|
|
21
|
+
* Every field is optional and additive. Legacy manifests with no `pinning`
|
|
22
|
+
* block load fine; readers nullish-coalesce (`m.pinning?.comfy_workflow_sha ?? null`).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { createHash } from 'node:crypto';
|
|
26
|
+
import { openSync, readSync, closeSync, statSync, existsSync, readFileSync, writeFileSync } from 'node:fs';
|
|
27
|
+
import { join } from 'node:path';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Contract version for the `pinning` block itself. Bump when the block's shape
|
|
31
|
+
* changes (a new field, a renamed field). Independent of the manifest
|
|
32
|
+
* SCHEMA_VERSION (lib/snapshot.js) — that versions the snapshot/split/export
|
|
33
|
+
* manifest family; this versions the generation-provenance pin. Mirrored as a
|
|
34
|
+
* literal in scripts/qwen_generate.py — keep the two in lockstep.
|
|
35
|
+
*/
|
|
36
|
+
export const PINNING_VERSION = '1.0.0';
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Seed policies — the INTENT behind seed selection, recorded alongside the
|
|
40
|
+
* concrete per-item seed values so a reader knows how to regenerate.
|
|
41
|
+
*/
|
|
42
|
+
export const SEED_POLICIES = Object.freeze({
|
|
43
|
+
EXPLICIT_PER_ITEM: 'explicit-per-item', // each item's seed is recorded verbatim (qwen_generate.py wave receipt)
|
|
44
|
+
BASE_INCREMENT: 'base+increment', // seed = base_seed + job_index (generate.js / batch path)
|
|
45
|
+
FIXED: 'fixed', // every item shares one seed
|
|
46
|
+
RANDOM: 'random', // first seed fixed, rest sampled
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Map a runtime seed_mode ('increment' | 'fixed' | 'random') to a seed_policy.
|
|
51
|
+
* `increment` is the base+index scheme; unknown modes fall back to base+increment.
|
|
52
|
+
*/
|
|
53
|
+
export function seedPolicyFromMode(seedMode) {
|
|
54
|
+
switch (seedMode) {
|
|
55
|
+
case 'fixed': return SEED_POLICIES.FIXED;
|
|
56
|
+
case 'random': return SEED_POLICIES.RANDOM;
|
|
57
|
+
case 'increment':
|
|
58
|
+
default: return SEED_POLICIES.BASE_INCREMENT;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// ─── Stable serialization ────────────────────────────────────────
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Deterministic JSON: object keys sorted, arrays kept in order (arrays are
|
|
66
|
+
* meaningful sequences — a LoRA chain, a node's input wiring). Guarantees the
|
|
67
|
+
* same semantic graph hashes identically regardless of key insertion order.
|
|
68
|
+
*/
|
|
69
|
+
export function stableStringify(value) {
|
|
70
|
+
if (value === null || value === undefined) return 'null';
|
|
71
|
+
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'null';
|
|
72
|
+
if (typeof value === 'boolean') return value ? 'true' : 'false';
|
|
73
|
+
if (typeof value === 'string') return JSON.stringify(value);
|
|
74
|
+
if (Array.isArray(value)) return '[' + value.map(stableStringify).join(',') + ']';
|
|
75
|
+
if (typeof value === 'object') {
|
|
76
|
+
const keys = Object.keys(value).sort();
|
|
77
|
+
const parts = [];
|
|
78
|
+
for (const k of keys) {
|
|
79
|
+
if (value[k] === undefined) continue;
|
|
80
|
+
parts.push(JSON.stringify(k) + ':' + stableStringify(value[k]));
|
|
81
|
+
}
|
|
82
|
+
return '{' + parts.join(',') + '}';
|
|
83
|
+
}
|
|
84
|
+
return 'null';
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Return a copy of a ComfyUI graph with per-ITEM nondeterminism removed, so the
|
|
89
|
+
* whole wave shares ONE comfy_workflow_sha. Normalized out (recorded elsewhere,
|
|
90
|
+
* per item): KSampler seed / noise_seed, CLIPTextEncode prompt text, SaveImage
|
|
91
|
+
* filename_prefix. What SURVIVES the hash is the pipeline skeleton — node
|
|
92
|
+
* topology, model loaders, sampler/scheduler/shift, LoRA chain + weights,
|
|
93
|
+
* dims/steps/cfg — i.e. exactly the part that must match to replay the wave.
|
|
94
|
+
*/
|
|
95
|
+
export function normalizeGraphForHash(graph) {
|
|
96
|
+
const out = {};
|
|
97
|
+
for (const [id, node] of Object.entries(graph || {})) {
|
|
98
|
+
const inputs = { ...(node?.inputs || {}) };
|
|
99
|
+
const cls = node?.class_type;
|
|
100
|
+
if (cls === 'KSampler' || cls === 'KSamplerAdvanced') {
|
|
101
|
+
if ('seed' in inputs) inputs.seed = 0;
|
|
102
|
+
if ('noise_seed' in inputs) inputs.noise_seed = 0;
|
|
103
|
+
} else if (cls === 'CLIPTextEncode') {
|
|
104
|
+
if ('text' in inputs) inputs.text = '';
|
|
105
|
+
} else if (cls === 'SaveImage') {
|
|
106
|
+
if ('filename_prefix' in inputs) inputs.filename_prefix = '';
|
|
107
|
+
}
|
|
108
|
+
out[id] = { class_type: cls, inputs };
|
|
109
|
+
}
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* SHA-256 of the exact ComfyUI graph submitted for the wave (per-item seed +
|
|
115
|
+
* prompt normalized out — see normalizeGraphForHash). Returns a `sha256:`-prefixed
|
|
116
|
+
* digest, or null for an empty/absent graph.
|
|
117
|
+
*/
|
|
118
|
+
export function comfyWorkflowSha(graph) {
|
|
119
|
+
if (!graph || Object.keys(graph).length === 0) return null;
|
|
120
|
+
const normalized = normalizeGraphForHash(graph);
|
|
121
|
+
return 'sha256:' + createHash('sha256').update(stableStringify(normalized)).digest('hex');
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ─── Model file resolution + hashing (best-effort) ───────────────
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Default ComfyUI models directory on the rig. Override with SDLAB_MODELS_DIR.
|
|
128
|
+
* Hashing is opt-in and best-effort, so a wrong/absent path degrades cleanly to
|
|
129
|
+
* `sha256: null` — it never fails a generation.
|
|
130
|
+
*/
|
|
131
|
+
export function defaultModelsDir() {
|
|
132
|
+
return process.env.SDLAB_MODELS_DIR
|
|
133
|
+
|| 'E:/AI-Models/ComfyUI_windows_portable/ComfyUI/models';
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Kind → candidate subdirectories under the models dir, in search order.
|
|
137
|
+
const KIND_SUBDIRS = {
|
|
138
|
+
unet: ['diffusion_models', 'unet'],
|
|
139
|
+
clip: ['text_encoders', 'clip'],
|
|
140
|
+
vae: ['vae'],
|
|
141
|
+
checkpoint: ['checkpoints'],
|
|
142
|
+
lora: ['loras'],
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Resolve a model filename to an absolute path under the models dir. Searches
|
|
147
|
+
* the kind-appropriate subdirs first, then every known subdir, then the root.
|
|
148
|
+
* Returns null when nothing matches (unresolvable → sha256 null with a note).
|
|
149
|
+
*/
|
|
150
|
+
export function resolveModelPath(name, kind, modelsDir = defaultModelsDir()) {
|
|
151
|
+
if (!name || !modelsDir) return null;
|
|
152
|
+
const tried = new Set();
|
|
153
|
+
const subdirs = [
|
|
154
|
+
...(KIND_SUBDIRS[kind] || []),
|
|
155
|
+
...Object.values(KIND_SUBDIRS).flat(),
|
|
156
|
+
'.',
|
|
157
|
+
];
|
|
158
|
+
for (const sub of subdirs) {
|
|
159
|
+
const candidate = join(modelsDir, sub, name);
|
|
160
|
+
if (tried.has(candidate)) continue;
|
|
161
|
+
tried.add(candidate);
|
|
162
|
+
if (existsSync(candidate)) return candidate;
|
|
163
|
+
}
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Streaming SHA-256 of a file — 1 MiB chunks, synchronous, never loads the whole
|
|
169
|
+
* (multi-GB) file into memory. Returns a `sha256:`-prefixed digest.
|
|
170
|
+
*/
|
|
171
|
+
export function sha256File(path) {
|
|
172
|
+
const hash = createHash('sha256');
|
|
173
|
+
const fd = openSync(path, 'r');
|
|
174
|
+
try {
|
|
175
|
+
const buf = Buffer.allocUnsafe(1 << 20);
|
|
176
|
+
let bytes;
|
|
177
|
+
while ((bytes = readSync(fd, buf, 0, buf.length, null)) > 0) {
|
|
178
|
+
hash.update(bytes === buf.length ? buf : buf.subarray(0, bytes));
|
|
179
|
+
}
|
|
180
|
+
} finally {
|
|
181
|
+
closeSync(fd);
|
|
182
|
+
}
|
|
183
|
+
return 'sha256:' + hash.digest('hex');
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Best-effort content identity for one model/LoRA file: {name, size_bytes, sha256}
|
|
188
|
+
* (+ a short `hash_note` when sha256 can't be produced). size_bytes is always
|
|
189
|
+
* filled when the file resolves (a cheap stat); sha256 is gated behind
|
|
190
|
+
* `doHash` (the --hash-models opt-in) and cached by (name, size, mtime) so
|
|
191
|
+
* multi-GB checkpoints aren't re-hashed every wave.
|
|
192
|
+
*
|
|
193
|
+
* @param {string} name — model filename as it appears in the graph
|
|
194
|
+
* @param {Object} [opts]
|
|
195
|
+
* @param {'unet'|'clip'|'vae'|'checkpoint'|'lora'} [opts.kind]
|
|
196
|
+
* @param {string} [opts.modelsDir]
|
|
197
|
+
* @param {boolean} [opts.doHash=false] — compute sha256 (else record name+size only)
|
|
198
|
+
* @param {{map: Map, cacheFile?: string, dirty: boolean}} [opts.cache]
|
|
199
|
+
*/
|
|
200
|
+
export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHash = false, cache } = {}) {
|
|
201
|
+
if (!name) return { name: name ?? null, size_bytes: null, sha256: null, hash_note: 'no filename' };
|
|
202
|
+
const resolved = resolveModelPath(name, kind, modelsDir);
|
|
203
|
+
if (!resolved) return { name, size_bytes: null, sha256: null, hash_note: `unresolved under ${modelsDir}` };
|
|
204
|
+
let size_bytes = null;
|
|
205
|
+
let mtimeMs = 0;
|
|
206
|
+
try {
|
|
207
|
+
const st = statSync(resolved);
|
|
208
|
+
size_bytes = st.size;
|
|
209
|
+
mtimeMs = st.mtimeMs;
|
|
210
|
+
} catch {
|
|
211
|
+
return { name, size_bytes: null, sha256: null, hash_note: 'stat failed' };
|
|
212
|
+
}
|
|
213
|
+
if (!doHash) return { name, size_bytes, sha256: null, hash_note: 'hash skipped (pass --hash-models)' };
|
|
214
|
+
const key = `${name}:${size_bytes}:${Math.round(mtimeMs)}`;
|
|
215
|
+
if (cache?.map?.has(key)) return { name, size_bytes, sha256: cache.map.get(key) };
|
|
216
|
+
let sha256;
|
|
217
|
+
try {
|
|
218
|
+
sha256 = sha256File(resolved);
|
|
219
|
+
} catch {
|
|
220
|
+
return { name, size_bytes, sha256: null, hash_note: 'hash failed (read error)' };
|
|
221
|
+
}
|
|
222
|
+
if (cache?.map) {
|
|
223
|
+
cache.map.set(key, sha256);
|
|
224
|
+
cache.dirty = true;
|
|
225
|
+
}
|
|
226
|
+
return { name, size_bytes, sha256 };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Content-identity block for the model loaders of a wave. `modelSpec` maps a
|
|
231
|
+
* role to a filename — {unet, clip, vae} for the Qwen DiT path, {checkpoint}
|
|
232
|
+
* for the SDXL path. Only provided roles appear in the result.
|
|
233
|
+
*/
|
|
234
|
+
export function buildModelHashes(modelSpec = {}, opts = {}) {
|
|
235
|
+
const roleKind = { unet: 'unet', clip: 'clip', vae: 'vae', checkpoint: 'checkpoint' };
|
|
236
|
+
const out = {};
|
|
237
|
+
for (const [role, kind] of Object.entries(roleKind)) {
|
|
238
|
+
const name = modelSpec[role];
|
|
239
|
+
if (!name) continue;
|
|
240
|
+
out[role] = hashModelFile(name, { ...opts, kind });
|
|
241
|
+
}
|
|
242
|
+
return out;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// ─── Persistent hash cache (best-effort) ─────────────────────────
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Create a hash cache. Loads the on-disk JSON map (name:size:mtime → sha256) if
|
|
249
|
+
* present, so hashes survive across waves. `cacheFile` defaults to a dotfile in
|
|
250
|
+
* the models dir; override with SDLAB_HASH_CACHE. All I/O is best-effort — a
|
|
251
|
+
* missing/unwritable cache never fails a run.
|
|
252
|
+
*/
|
|
253
|
+
export function createHashCache(cacheFile) {
|
|
254
|
+
const file = cacheFile
|
|
255
|
+
|| process.env.SDLAB_HASH_CACHE
|
|
256
|
+
|| join(defaultModelsDir(), '.sdlab-hash-cache.json');
|
|
257
|
+
const map = new Map();
|
|
258
|
+
try {
|
|
259
|
+
if (existsSync(file)) {
|
|
260
|
+
const obj = JSON.parse(readFileSync(file, 'utf-8'));
|
|
261
|
+
for (const [k, v] of Object.entries(obj)) map.set(k, v);
|
|
262
|
+
}
|
|
263
|
+
} catch { /* best-effort: ignore a corrupt/unreadable cache */ }
|
|
264
|
+
return { map, cacheFile: file, dirty: false };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Persist a hash cache to disk if it grew. Best-effort — swallows write errors.
|
|
269
|
+
*/
|
|
270
|
+
export function flushHashCache(cache) {
|
|
271
|
+
if (!cache || !cache.dirty || !cache.cacheFile) return;
|
|
272
|
+
try {
|
|
273
|
+
const obj = Object.fromEntries(cache.map);
|
|
274
|
+
writeFileSync(cache.cacheFile, JSON.stringify(obj, null, 2));
|
|
275
|
+
cache.dirty = false;
|
|
276
|
+
} catch { /* best-effort: ignore */ }
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// ─── The pinning block ───────────────────────────────────────────
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Build the pinning block for a wave/item. Identical shape across all three
|
|
283
|
+
* writers (see module header). Every writer calls this so the contract is
|
|
284
|
+
* enforced in ONE place.
|
|
285
|
+
*
|
|
286
|
+
* @param {Object} args
|
|
287
|
+
* @param {Object} [args.graph] — the ComfyUI node graph submitted (for comfy_workflow_sha)
|
|
288
|
+
* @param {Object} [args.models] — {unet?, clip?, vae?, checkpoint?} filenames
|
|
289
|
+
* @param {Array<{name:string, weight?:number}>} [args.loras]
|
|
290
|
+
* @param {string} [args.seedPolicy] — one of SEED_POLICIES
|
|
291
|
+
* @param {boolean} [args.hashModels=false] — opt-in sha256 of model/LoRA files
|
|
292
|
+
* @param {string} [args.modelsDir]
|
|
293
|
+
* @param {{map: Map, cacheFile?: string, dirty: boolean}} [args.cache]
|
|
294
|
+
* @returns {Object} pinning block
|
|
295
|
+
*/
|
|
296
|
+
export function buildPinning({ graph, models = {}, loras = [], seedPolicy = null, hashModels = false, modelsDir = defaultModelsDir(), cache } = {}) {
|
|
297
|
+
const opts = { modelsDir, doHash: hashModels, cache };
|
|
298
|
+
return {
|
|
299
|
+
pinning_version: PINNING_VERSION,
|
|
300
|
+
comfy_workflow_sha: comfyWorkflowSha(graph),
|
|
301
|
+
seed_policy: seedPolicy,
|
|
302
|
+
models: buildModelHashes(models, opts),
|
|
303
|
+
loras: (loras || []).map((l) => ({
|
|
304
|
+
...hashModelFile(l.name, { ...opts, kind: 'lora' }),
|
|
305
|
+
weight: l.weight ?? 1.0,
|
|
306
|
+
})),
|
|
307
|
+
};
|
|
308
|
+
}
|
package/lib/snapshot.js
CHANGED
|
@@ -19,8 +19,14 @@ import { warn } from './log.js';
|
|
|
19
19
|
* snapshot/split/export/eval-pack/eval-run/training-manifest/training-package/
|
|
20
20
|
* impl-pack stamps this version; readers log a WARNING on version mismatch
|
|
21
21
|
* (best-effort compatibility).
|
|
22
|
+
*
|
|
23
|
+
* 2.3.0 (v3.3.0): PIN_PER_STEP — the ComfyUI run manifest now also stamps this
|
|
24
|
+
* version and carries a `pinning` block (comfy_workflow_sha + model/LoRA content
|
|
25
|
+
* hashes + seed_policy; see lib/run-manifest.js). The block is additive and
|
|
26
|
+
* optional, so a legacy manifest with no schema_version / no pinning still loads
|
|
27
|
+
* (checkManifestVersion only warns; it never throws).
|
|
22
28
|
*/
|
|
23
|
-
export const SCHEMA_VERSION = '2.
|
|
29
|
+
export const SCHEMA_VERSION = '2.3.0';
|
|
24
30
|
|
|
25
31
|
/**
|
|
26
32
|
* Warn if a loaded manifest's schema_version does not match the reader.
|
package/lib/verifier.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EXTERNAL_VERIFIER support (Kambhampati arXiv:2402.01817; Huang / DeepMind
|
|
3
|
+
* arXiv:2310.01798): a judgment records WHO judged it (`judged_by_model`) and
|
|
4
|
+
* WHAT produced the artifact (`generator_model`), and we WARN when they are the
|
|
5
|
+
* same — the self-verification failure mode a verifier must never fall into.
|
|
6
|
+
*
|
|
7
|
+
* Today the two judges are `human` (curate) and a rule-based engine
|
|
8
|
+
* (critique-engine) — both a different artifact class from the image generator,
|
|
9
|
+
* so the warning does not fire in practice and the standard's spirit is already
|
|
10
|
+
* met. Recording the fields + wiring the warning installs the muscle: the moment
|
|
11
|
+
* an LLM critique mode enters the loop sharing the generator's base, the WARN
|
|
12
|
+
* fires, and the standard's remediation flips it from a warning to a hard gate.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { warn } from './log.js';
|
|
16
|
+
|
|
17
|
+
/** The rule-based critique engine's judge id. Bump vN when the rule set changes. */
|
|
18
|
+
export const RULE_BASED_JUDGE = 'rule-based:sdlab-critique-v1';
|
|
19
|
+
|
|
20
|
+
/** The human curator's judge id (curate.js reviewer is "human:mike"). */
|
|
21
|
+
export const HUMAN_JUDGE = 'human';
|
|
22
|
+
|
|
23
|
+
const MODEL_EXT_RE = /\.(safetensors|ckpt|pt|pth|bin|gguf)$/i;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Derive a stable generator model id from a record `provenance` (or any object
|
|
27
|
+
* carrying {base, unet|checkpoint}): `<base>:<model-basename-without-ext>`, e.g.
|
|
28
|
+
* "qwen-image:qwen_image_fp8_e4m3fn" or "sdxl:dreamshaperXL". Returns null when
|
|
29
|
+
* neither a base nor a model file is known (legacy/hand-migrated records).
|
|
30
|
+
*/
|
|
31
|
+
export function deriveGeneratorModel(provenance) {
|
|
32
|
+
if (!provenance || typeof provenance !== 'object') return null;
|
|
33
|
+
const base = provenance.base || null;
|
|
34
|
+
const modelFile = provenance.unet || provenance.checkpoint || null;
|
|
35
|
+
const stripped = modelFile ? String(modelFile).replace(MODEL_EXT_RE, '') : null;
|
|
36
|
+
if (base && stripped) return `${base}:${stripped}`;
|
|
37
|
+
return base || stripped || null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Derive the generator model id from a run manifest (comfyui-runner shape).
|
|
42
|
+
* Prefers the pinning block's model identity, falling back to the top-level
|
|
43
|
+
* checkpoint. Qwen manifests carry pinning.models.unet; SDXL carry checkpoint.
|
|
44
|
+
*/
|
|
45
|
+
export function generatorModelFromManifest(manifest) {
|
|
46
|
+
if (!manifest || typeof manifest !== 'object') return null;
|
|
47
|
+
const unet = manifest.pinning?.models?.unet?.name;
|
|
48
|
+
const checkpoint = manifest.checkpoint || manifest.pinning?.models?.checkpoint?.name;
|
|
49
|
+
const base = unet ? 'qwen-image' : (checkpoint ? 'sdxl' : null);
|
|
50
|
+
return deriveGeneratorModel({ base, unet, checkpoint });
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Emit a WARN when the judge and the generator are the same model — the
|
|
55
|
+
* self-verification failure mode. No-op when either is null or they differ.
|
|
56
|
+
* Returns true when the warning fired, so callers/tests can assert on it without
|
|
57
|
+
* capturing stderr.
|
|
58
|
+
*/
|
|
59
|
+
export function warnIfSelfVerification(judgedByModel, generatorModel, context = '') {
|
|
60
|
+
if (!judgedByModel || !generatorModel) return false;
|
|
61
|
+
if (judgedByModel !== generatorModel) return false;
|
|
62
|
+
warn(
|
|
63
|
+
`EXTERNAL_VERIFIER: judged_by_model === generator_model ("${judgedByModel}")${context ? ` for ${context}` : ''} — ` +
|
|
64
|
+
'a model must not verify its own output. Use a different-family judge (ai-eyes-mcp or a cross-family LLM).',
|
|
65
|
+
);
|
|
66
|
+
return true;
|
|
67
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mcptoolshop/style-dataset-lab",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.0",
|
|
4
4
|
"description": "Canon-aligned dataset production and generation workbench — define visual rules, build versioned training data, compile production briefs, run local workflows, batch-produce, select winners, and re-ingest into your corpus",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/scripts/critique.js
CHANGED
|
@@ -8,10 +8,10 @@
|
|
|
8
8
|
* sdlab critique --project star-freight --run run_2026-04-15_001 --json
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
11
|
+
import { parseArgs, getProjectName, parseNumberFlag } from '../lib/args.js';
|
|
12
12
|
import { getProjectRoot } from '../lib/paths.js';
|
|
13
|
-
import { critiqueRun, saveCritique } from '../lib/critique-engine.js';
|
|
14
|
-
import { renderCritiqueMarkdown, renderCritiqueText } from '../lib/critique-render.js';
|
|
13
|
+
import { critiqueRun, saveCritique, triageCandidates, DEFAULT_DRIFT_THRESHOLD } from '../lib/critique-engine.js';
|
|
14
|
+
import { renderCritiqueMarkdown, renderCritiqueText, renderCritiqueTriage } from '../lib/critique-render.js';
|
|
15
15
|
import { info } from '../lib/log.js';
|
|
16
16
|
|
|
17
17
|
export async function run(argv = process.argv.slice(2)) {
|
|
@@ -20,16 +20,20 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
20
20
|
project: { type: 'string' },
|
|
21
21
|
run: { type: 'string' },
|
|
22
22
|
json: { type: 'boolean' },
|
|
23
|
+
triage: { type: 'boolean' },
|
|
24
|
+
'drift-threshold': { type: 'string' },
|
|
23
25
|
},
|
|
24
26
|
deprecated: { game: 'project' },
|
|
25
27
|
});
|
|
26
28
|
|
|
27
29
|
if (!flags.run) {
|
|
28
|
-
console.log('Usage: sdlab critique --run <id> [--project <name>]');
|
|
30
|
+
console.log('Usage: sdlab critique --run <id> [--project <name>] [--triage]');
|
|
29
31
|
console.log('');
|
|
30
32
|
console.log('Options:');
|
|
31
|
-
console.log(' --run <id>
|
|
32
|
-
console.log(' --
|
|
33
|
+
console.log(' --run <id> Run to critique (required)');
|
|
34
|
+
console.log(' --triage Show only candidates needing attention (off-model OR ≥ threshold drift issues)');
|
|
35
|
+
console.log(` --drift-threshold <n> Drift-issue count that flags a candidate under --triage (default ${DEFAULT_DRIFT_THRESHOLD})`);
|
|
36
|
+
console.log(' --json Output raw JSON');
|
|
33
37
|
return;
|
|
34
38
|
}
|
|
35
39
|
|
|
@@ -48,10 +52,31 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
48
52
|
runId: flags.run,
|
|
49
53
|
});
|
|
50
54
|
|
|
51
|
-
// Save
|
|
55
|
+
// Save the FULL report — triage is a view over it, never a mutation.
|
|
52
56
|
const md = renderCritiqueMarkdown(report);
|
|
53
57
|
await saveCritique(projectRoot, flags.run, report, md);
|
|
54
58
|
|
|
59
|
+
// UNCERTAINTY_GATED_HUMANS: --triage surfaces only the candidates that need a
|
|
60
|
+
// human — off-model OR >= threshold drift issues — so attention isn't spent on
|
|
61
|
+
// every item. The full critique.json still holds all candidates.
|
|
62
|
+
if (flags.triage) {
|
|
63
|
+
const driftThreshold = flags['drift-threshold']
|
|
64
|
+
? parseNumberFlag('drift-threshold', flags['drift-threshold'], { int: true, min: 0 })
|
|
65
|
+
: DEFAULT_DRIFT_THRESHOLD;
|
|
66
|
+
const triage = triageCandidates(report.candidates, { driftThreshold });
|
|
67
|
+
if (flags.json) {
|
|
68
|
+
console.log(JSON.stringify({
|
|
69
|
+
run_id: report.run_id,
|
|
70
|
+
triage: { drift_threshold: driftThreshold, flagged_count: triage.flagged.length, suppressed_count: triage.suppressed.length },
|
|
71
|
+
candidates: triage.flagged,
|
|
72
|
+
}, null, 2));
|
|
73
|
+
} else {
|
|
74
|
+
console.log(renderCritiqueTriage(report, triage));
|
|
75
|
+
console.log(`Full critique saved to: runs/${flags.run}/critique.json`);
|
|
76
|
+
}
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
|
|
55
80
|
if (flags.json) {
|
|
56
81
|
console.log(JSON.stringify(report, null, 2));
|
|
57
82
|
} else {
|
package/scripts/curate.js
CHANGED
|
@@ -18,6 +18,7 @@ import { readJsonFile } from "../lib/config.js";
|
|
|
18
18
|
import { inputError, runtimeError, handleCliError } from "../lib/errors.js";
|
|
19
19
|
import { result } from "../lib/log.js";
|
|
20
20
|
import { assertNotFrozenByAssetPath } from "../lib/freeze-gate.js";
|
|
21
|
+
import { deriveGeneratorModel, warnIfSelfVerification, HUMAN_JUDGE } from "../lib/verifier.js";
|
|
21
22
|
|
|
22
23
|
export async function run(argv = process.argv.slice(2)) {
|
|
23
24
|
const { flags, positionals } = parseArgs(argv, {
|
|
@@ -132,11 +133,19 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
132
133
|
);
|
|
133
134
|
}
|
|
134
135
|
|
|
136
|
+
// EXTERNAL_VERIFIER: record WHO judged (human) and WHAT generated the artifact,
|
|
137
|
+
// and warn if they ever coincide (they never do for human curation — the field
|
|
138
|
+
// + warning install the muscle for when an LLM judge enters the loop).
|
|
139
|
+
const generator_model = deriveGeneratorModel(record.provenance);
|
|
140
|
+
warnIfSelfVerification(HUMAN_JUDGE, generator_model, assetId);
|
|
141
|
+
|
|
135
142
|
// Now safely update the record. If writing fails, try to restore the image.
|
|
136
143
|
record.asset_path = newPath;
|
|
137
144
|
record.judgment = {
|
|
138
145
|
status,
|
|
139
146
|
reviewer: "human:mike",
|
|
147
|
+
judged_by_model: HUMAN_JUDGE,
|
|
148
|
+
generator_model,
|
|
140
149
|
reviewed_at: new Date().toISOString(),
|
|
141
150
|
explanation,
|
|
142
151
|
criteria_scores,
|
|
@@ -164,6 +173,18 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
164
173
|
result(` record: records/${assetId}.json`);
|
|
165
174
|
console.log(` ${explanation}`);
|
|
166
175
|
if (failure_modes.length) console.log(` Failures: ${failure_modes.join(", ")}`);
|
|
176
|
+
|
|
177
|
+
// UNCERTAINTY_GATED_HUMANS: contrastive advisory on borderline (Horvitz CHI
|
|
178
|
+
// 1999). State the default (HOLD — borderline is not training-eligible) and
|
|
179
|
+
// what a later promotion must justify, so the promoter argues against a stated
|
|
180
|
+
// default rather than nudging a status upward unexamined.
|
|
181
|
+
if (status === 'borderline') {
|
|
182
|
+
const drift = failure_modes.length ? failure_modes.join(', ') : 'the drift you noted';
|
|
183
|
+
console.log(
|
|
184
|
+
` \x1b[2mDefault: HOLD — 'borderline' is not training-eligible. ` +
|
|
185
|
+
`To later promote to 'approved', justify why ${drift} is acceptable for canon (re-run curate approved with that reason).\x1b[0m`,
|
|
186
|
+
);
|
|
187
|
+
}
|
|
167
188
|
}
|
|
168
189
|
|
|
169
190
|
async function listUncurated(GAME_ROOT) {
|
package/scripts/generate.js
CHANGED
|
@@ -20,6 +20,7 @@ import { runtimeError, inputError, handleCliError } from "../lib/errors.js";
|
|
|
20
20
|
import { comfyHealth, submitAndWait, downloadImage } from "../lib/comfyui.js";
|
|
21
21
|
import { pickOutputImage } from "../lib/comfyui-output.js";
|
|
22
22
|
import { buildQwenGraph, QWEN_DEFAULTS } from "../lib/adapters/comfyui-workflows.js";
|
|
23
|
+
import { buildPinning, SEED_POLICIES, createHashCache, flushHashCache } from "../lib/run-manifest.js";
|
|
23
24
|
import { info, result } from "../lib/log.js";
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -272,12 +273,18 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
272
273
|
game: { type: 'string' },
|
|
273
274
|
'dry-run': { type: 'boolean' },
|
|
274
275
|
resume: { type: 'boolean' },
|
|
276
|
+
'hash-models': { type: 'boolean' },
|
|
275
277
|
},
|
|
276
278
|
deprecated: { game: 'project' },
|
|
277
279
|
});
|
|
278
280
|
const packPath = parsed.positionals[0] || "inputs/prompts/rpg-icons-lane1.json";
|
|
279
281
|
const dryRun = parsed.flags['dry-run'] === true;
|
|
280
282
|
const resume = parsed.flags['resume'] === true;
|
|
283
|
+
// PIN_PER_STEP: --hash-models opts into content hashing of the checkpoint/
|
|
284
|
+
// unet/clip/vae/LoRA files (best-effort, cached). Without it the pinning block
|
|
285
|
+
// still records comfy_workflow_sha + model name/size + seed_policy.
|
|
286
|
+
const hashModels = parsed.flags['hash-models'] === true;
|
|
287
|
+
const hashCache = hashModels ? createHashCache() : undefined;
|
|
281
288
|
|
|
282
289
|
const fullPackPath = resolveSafeProjectPath(GAME_ROOT, packPath, { flagName: 'pack-path' });
|
|
283
290
|
const raw = await readJsonFile(fullPackPath);
|
|
@@ -432,6 +439,20 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
432
439
|
provenance.loras = loras;
|
|
433
440
|
}
|
|
434
441
|
|
|
442
|
+
// PIN_PER_STEP: pin the exact graph + model/LoRA content identity + seed
|
|
443
|
+
// policy via the shared contract (lib/run-manifest.js). Model hashing is
|
|
444
|
+
// wave-constant; the shared cache means it runs at most once per wave.
|
|
445
|
+
provenance.pinning = buildPinning({
|
|
446
|
+
graph: nodes,
|
|
447
|
+
models: isQwen
|
|
448
|
+
? { unet: provenance.unet, clip: provenance.clip, vae: provenance.vae }
|
|
449
|
+
: { checkpoint: d.checkpoint },
|
|
450
|
+
loras: isQwen ? (d.loras || []) : loras,
|
|
451
|
+
seedPolicy: SEED_POLICIES.BASE_INCREMENT,
|
|
452
|
+
hashModels,
|
|
453
|
+
cache: hashCache,
|
|
454
|
+
});
|
|
455
|
+
|
|
435
456
|
const record = {
|
|
436
457
|
id: assetId,
|
|
437
458
|
schema_version: "1.0.0",
|
|
@@ -467,6 +488,9 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
467
488
|
}
|
|
468
489
|
}
|
|
469
490
|
|
|
491
|
+
// Persist any newly computed model hashes so the next wave reuses them.
|
|
492
|
+
flushHashCache(hashCache);
|
|
493
|
+
|
|
470
494
|
console.log("");
|
|
471
495
|
const succeeded = generated - errors - skipped;
|
|
472
496
|
const totalElapsedMs = Date.now() - runStartMs;
|