@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.es.md +117 -55
- package/README.fr.md +125 -65
- package/README.hi.md +117 -68
- package/README.it.md +119 -58
- package/README.ja.md +124 -74
- package/README.md +73 -7
- package/README.pt-BR.md +128 -63
- package/README.zh.md +116 -65
- package/bin/sdlab.js +100 -4
- package/lib/adapters/ai-toolkit.js +224 -35
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +94 -11
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/asset-ingest.js +575 -0
- package/lib/asset-masks.js +323 -0
- package/lib/asset-source.js +1176 -0
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-author.js +350 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/canon-review-gate.js +205 -0
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/colab-launcher-guards.js +285 -0
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +16 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +194 -6
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/lane1-render.js +668 -0
- package/lib/measure.js +422 -0
- package/lib/memory-store.js +663 -0
- package/lib/model-licences.js +305 -0
- package/lib/npy-meta.js +130 -0
- package/lib/paths.js +196 -7
- package/lib/png-meta.js +512 -0
- package/lib/records.js +54 -8
- package/lib/reingest-selected.js +113 -17
- package/lib/reingest.js +27 -3
- package/lib/run-manifest.js +159 -25
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/train-runs.js +214 -0
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +3 -0
- package/lib/verify-chain.js +367 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +4 -3
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/asset-ingest.js +160 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-draft.js +82 -0
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +31 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
- package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
- package/scripts/colab/colab_window.sh +587 -0
- package/scripts/colab/lane1_render.py +970 -0
- package/scripts/colab/launch_guards.py +564 -0
- package/scripts/colab/launch_lane1_stub.sh +309 -0
- package/scripts/colab/launch_lane3_stub.sh +366 -0
- package/scripts/colab/remote_runner.py +59 -0
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +182 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +86 -17
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/verify.js +67 -0
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* End-to-end offline provenance chain verifier (G9, §2.2–§2.3).
|
|
3
|
+
*
|
|
4
|
+
* Walks every link required for CC-BY-4.0 release eligibility:
|
|
5
|
+
* 1. owner_and_licence (G5): owner === "mcp-tool-shop", licence === "CC-BY-4.0", ai_assisted_disclosure === true
|
|
6
|
+
* 2. canon_link (G6): canon_sha, entry_id, entry_sha256 match canon-build/<canon_sha>/manifest.json per_entity_hashes
|
|
7
|
+
* 3. run_manifest_link (§2.2.1): run_id, seed, prompt/prompt_sha256, comfy_workflow_sha match runs/<run_id>/manifest.json, and image exists
|
|
8
|
+
* 4. model_licences (G4): all models and LoRAs in pinning have cleared SPDX licence_id, non-empty licence_url, and date_read
|
|
9
|
+
* 5. quarantine_guard (§2.2.5): no reference to held Portlight data (`plsvessel`, `nano-banana`)
|
|
10
|
+
* 6. verdict_separation (§2.2.6): approved records cite a reviewer/judge distinct from the generator model
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
14
|
+
import { join, resolve, relative, isAbsolute } from 'node:path';
|
|
15
|
+
import { STUDIO_OWNER, STUDIO_LICENCE } from './records.js';
|
|
16
|
+
import {
|
|
17
|
+
isClearedLicenceEntry,
|
|
18
|
+
isQuarantinedName,
|
|
19
|
+
hasModelSha256,
|
|
20
|
+
normalizeModelFamilyKey,
|
|
21
|
+
} from './model-licences.js';
|
|
22
|
+
|
|
23
|
+
function isSafeRelativeAssetPath(projectRoot, assetPath) {
|
|
24
|
+
if (!assetPath || typeof assetPath !== 'string') return false;
|
|
25
|
+
const absRoot = resolve(projectRoot);
|
|
26
|
+
const absAsset = resolve(projectRoot, assetPath);
|
|
27
|
+
const rel = relative(absRoot, absAsset);
|
|
28
|
+
return Boolean(rel && !rel.startsWith('..') && !isAbsolute(rel));
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Verify all provenance links for a single record object against on-disk manifests.
|
|
33
|
+
*/
|
|
34
|
+
export function verifyRecordProvenance(projectRoot, record) {
|
|
35
|
+
const recordId = record?.id || record?.record_id || '<unknown>';
|
|
36
|
+
const failures = [];
|
|
37
|
+
const links = {
|
|
38
|
+
owner_and_licence: 'pass',
|
|
39
|
+
canon_link: 'pass',
|
|
40
|
+
run_manifest_link: 'pass',
|
|
41
|
+
model_licences: 'pass',
|
|
42
|
+
quarantine_guard: 'pass',
|
|
43
|
+
verdict_separation: 'pass',
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
function fail(link, code, message) {
|
|
47
|
+
links[link] = 'fail';
|
|
48
|
+
failures.push({ record_id: recordId, link, code, message });
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// 1. G5: Owner and CC-BY-4.0 licence stamp
|
|
52
|
+
const owner = record?.owner ?? record?.rights?.owner ?? null;
|
|
53
|
+
const licence = record?.licence ?? record?.rights?.licence ?? null;
|
|
54
|
+
const disclosure = record?.rights?.ai_assisted_disclosure === true;
|
|
55
|
+
if (owner !== STUDIO_OWNER || licence !== STUDIO_LICENCE || !disclosure) {
|
|
56
|
+
fail(
|
|
57
|
+
'owner_and_licence',
|
|
58
|
+
'MISSING_OWNER_OR_LICENCE',
|
|
59
|
+
`Record "${recordId}" must carry owner="${STUDIO_OWNER}", licence="${STUDIO_LICENCE}", and rights.ai_assisted_disclosure=true (got owner=${JSON.stringify(owner)}, licence=${JSON.stringify(licence)}, disclosure=${JSON.stringify(disclosure)})`,
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const genProv =
|
|
64
|
+
record?.generation_provenance ||
|
|
65
|
+
record?.provenance?.generation_provenance ||
|
|
66
|
+
record?.provenance ||
|
|
67
|
+
{};
|
|
68
|
+
|
|
69
|
+
// 2. G6: Canon link (canon_sha, entry_id, entry_sha256 -> canon-build/<canon_sha>/manifest.json)
|
|
70
|
+
const canonSha = record?.canon?.canon_sha ?? genProv?.canon_sha ?? null;
|
|
71
|
+
const entryId = record?.canon?.entry_id ?? genProv?.entry_id ?? null;
|
|
72
|
+
const entrySha256 = record?.canon?.entry_sha256 ?? genProv?.entry_sha256 ?? null;
|
|
73
|
+
|
|
74
|
+
if (!canonSha || !entryId || !entrySha256) {
|
|
75
|
+
fail(
|
|
76
|
+
'canon_link',
|
|
77
|
+
'BROKEN_CANON_LINK',
|
|
78
|
+
`Record "${recordId}" is missing canon link fields (canon_sha=${canonSha}, entry_id=${entryId}, entry_sha256=${entrySha256})`,
|
|
79
|
+
);
|
|
80
|
+
} else {
|
|
81
|
+
const dirName = String(canonSha).replace(/:/g, '-');
|
|
82
|
+
const canonManifestPath = existsSync(join(projectRoot, 'canon-build', String(canonSha), 'manifest.json'))
|
|
83
|
+
? join(projectRoot, 'canon-build', String(canonSha), 'manifest.json')
|
|
84
|
+
: join(projectRoot, 'canon-build', dirName, 'manifest.json');
|
|
85
|
+
if (!existsSync(canonManifestPath)) {
|
|
86
|
+
fail(
|
|
87
|
+
'canon_link',
|
|
88
|
+
'BROKEN_CANON_LINK',
|
|
89
|
+
`Canon build manifest not found at canon-build/${canonSha}/manifest.json for record "${recordId}"`,
|
|
90
|
+
);
|
|
91
|
+
} else {
|
|
92
|
+
try {
|
|
93
|
+
const canonManifest = JSON.parse(readFileSync(canonManifestPath, 'utf-8'));
|
|
94
|
+
const hashes = canonManifest?.per_entity_hashes || {};
|
|
95
|
+
const bareId = String(entryId).includes('/') ? String(entryId).split('/').pop() : null;
|
|
96
|
+
const expectedSha = hashes[entryId] ?? (bareId ? hashes[bareId] : undefined);
|
|
97
|
+
if (!expectedSha || expectedSha !== entrySha256) {
|
|
98
|
+
fail(
|
|
99
|
+
'canon_link',
|
|
100
|
+
'BROKEN_CANON_LINK',
|
|
101
|
+
`Record "${recordId}" entry_sha256 (${entrySha256}) does not match canon-build/${canonSha}/manifest.json per_entity_hashes["${entryId}"] (${expectedSha || 'missing'})`,
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
} catch (err) {
|
|
105
|
+
fail(
|
|
106
|
+
'canon_link',
|
|
107
|
+
'BROKEN_CANON_LINK',
|
|
108
|
+
`Canon build manifest canon-build/${canonSha}/manifest.json could not be parsed: ${err.message}`,
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// 3. §2.2.1: Run manifest link + workflow sha + image existence
|
|
115
|
+
const runId = genProv?.run_id ?? null;
|
|
116
|
+
const recWorkflowSha = genProv?.comfy_workflow_sha ?? record?.provenance?.pinning?.comfy_workflow_sha ?? null;
|
|
117
|
+
const hasSeed = Number.isFinite(genProv?.seed);
|
|
118
|
+
const hasPrompt =
|
|
119
|
+
(typeof genProv?.prompt === 'string' && genProv.prompt.length > 0) ||
|
|
120
|
+
(typeof genProv?.prompt_sha256 === 'string' && genProv.prompt_sha256.length > 0);
|
|
121
|
+
|
|
122
|
+
let runManifest = null;
|
|
123
|
+
if (!runId || !recWorkflowSha || !hasSeed || !hasPrompt) {
|
|
124
|
+
fail(
|
|
125
|
+
'run_manifest_link',
|
|
126
|
+
'BROKEN_RUN_MANIFEST_LINK',
|
|
127
|
+
`Record "${recordId}" provenance is missing run_id (${runId}), comfy_workflow_sha (${recWorkflowSha}), seed (${genProv?.seed}), or prompt`,
|
|
128
|
+
);
|
|
129
|
+
} else {
|
|
130
|
+
const runManifestPath = join(projectRoot, 'runs', String(runId), 'manifest.json');
|
|
131
|
+
if (!existsSync(runManifestPath)) {
|
|
132
|
+
fail(
|
|
133
|
+
'run_manifest_link',
|
|
134
|
+
'BROKEN_RUN_MANIFEST_LINK',
|
|
135
|
+
`Run manifest runs/${runId}/manifest.json not found for record "${recordId}"`,
|
|
136
|
+
);
|
|
137
|
+
} else {
|
|
138
|
+
try {
|
|
139
|
+
runManifest = JSON.parse(readFileSync(runManifestPath, 'utf-8'));
|
|
140
|
+
const manifestWfSha = runManifest?.pinning?.comfy_workflow_sha ?? null;
|
|
141
|
+
if (!manifestWfSha || manifestWfSha !== recWorkflowSha) {
|
|
142
|
+
fail(
|
|
143
|
+
'run_manifest_link',
|
|
144
|
+
'BROKEN_RUN_MANIFEST_LINK',
|
|
145
|
+
`Record "${recordId}" comfy_workflow_sha (${recWorkflowSha}) does not match runs/${runId}/manifest.json (${manifestWfSha})`,
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
} catch (err) {
|
|
149
|
+
fail(
|
|
150
|
+
'run_manifest_link',
|
|
151
|
+
'BROKEN_RUN_MANIFEST_LINK',
|
|
152
|
+
`Run manifest runs/${runId}/manifest.json could not be parsed: ${err.message}`,
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const assetPath = record?.asset_path ?? record?.image_path ?? null;
|
|
159
|
+
if (!isSafeRelativeAssetPath(projectRoot, assetPath) || !existsSync(join(projectRoot, assetPath))) {
|
|
160
|
+
fail(
|
|
161
|
+
'run_manifest_link',
|
|
162
|
+
'BROKEN_RUN_MANIFEST_LINK',
|
|
163
|
+
`Record "${recordId}" asset_path (${assetPath}) does not exist inside the project`,
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// 4. G4: Model and LoRA licences and sha256 digests
|
|
168
|
+
const pinning = runManifest?.pinning ?? record?.provenance?.pinning ?? null;
|
|
169
|
+
const modelEntries = Object.entries(pinning?.models || {}).map(([role, entry]) => ({ role, ...entry }));
|
|
170
|
+
const loraEntries = (pinning?.loras || []).map((entry, idx) => ({ role: `lora[${idx}]`, ...entry }));
|
|
171
|
+
const allWeightEntries = [...modelEntries, ...loraEntries];
|
|
172
|
+
|
|
173
|
+
const recModelLicences =
|
|
174
|
+
record?.generation_provenance?.model_licences ?? genProv?.model_licences ?? null;
|
|
175
|
+
const recModelEntries = recModelLicences
|
|
176
|
+
? Object.entries(recModelLicences.models || {}).map(([role, entry]) => ({
|
|
177
|
+
role: `generation_provenance.model_licences.models.${role}`,
|
|
178
|
+
...entry,
|
|
179
|
+
}))
|
|
180
|
+
: [];
|
|
181
|
+
const recLoraEntries = recModelLicences
|
|
182
|
+
? (recModelLicences.loras || []).map((entry, idx) => ({
|
|
183
|
+
role: `generation_provenance.model_licences.loras[${idx}]`,
|
|
184
|
+
...entry,
|
|
185
|
+
}))
|
|
186
|
+
: [];
|
|
187
|
+
const allCheckedEntries = [...allWeightEntries, ...recModelEntries, ...recLoraEntries];
|
|
188
|
+
|
|
189
|
+
if (!pinning || allWeightEntries.length === 0) {
|
|
190
|
+
fail(
|
|
191
|
+
'model_licences',
|
|
192
|
+
'UNLICENSED_OR_EXCLUDED_MODEL',
|
|
193
|
+
`Record "${recordId}" has no pinned model weights to verify licences against`,
|
|
194
|
+
);
|
|
195
|
+
} else {
|
|
196
|
+
const missingShaWeights = allCheckedEntries.filter((entry) => !hasModelSha256(entry));
|
|
197
|
+
if (
|
|
198
|
+
record?.generation_provenance &&
|
|
199
|
+
(!recModelLicences || recModelEntries.length < modelEntries.length || recLoraEntries.length < loraEntries.length)
|
|
200
|
+
) {
|
|
201
|
+
missingShaWeights.push({
|
|
202
|
+
role: 'generation_provenance.model_licences',
|
|
203
|
+
name: '<missing>',
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
if (missingShaWeights.length > 0) {
|
|
207
|
+
const shaDesc = missingShaWeights
|
|
208
|
+
.map((w) => `${w.role}:${w.name || '<unnamed>'}`)
|
|
209
|
+
.join(', ');
|
|
210
|
+
fail(
|
|
211
|
+
'model_licences',
|
|
212
|
+
'MISSING_MODEL_SHA256',
|
|
213
|
+
`Record "${recordId}" is missing a non-empty sha256 for model/LoRA weights: ${shaDesc}`,
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const invalidLicenceWeights = allCheckedEntries.filter(
|
|
218
|
+
(entry) => !isClearedLicenceEntry(entry, { requireSha256: false }),
|
|
219
|
+
);
|
|
220
|
+
if (
|
|
221
|
+
invalidLicenceWeights.length > 0 ||
|
|
222
|
+
(missingShaWeights.length === 0 &&
|
|
223
|
+
(pinning.cc_by_4_eligible === false || recModelLicences?.cc_by_4_eligible === false))
|
|
224
|
+
) {
|
|
225
|
+
const desc = invalidLicenceWeights
|
|
226
|
+
.map((w) => `${w.role}:${w.name || '<unnamed>'}(licence=${w.licence_id || 'missing'})`)
|
|
227
|
+
.join(', ');
|
|
228
|
+
fail(
|
|
229
|
+
'model_licences',
|
|
230
|
+
'UNLICENSED_OR_EXCLUDED_MODEL',
|
|
231
|
+
`Record "${recordId}" uses unlicensed or excluded model/LoRA weights: ${desc || 'cc_by_4_eligible=false'}`,
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// 5. §2.2.5: Quarantine guard (no held Portlight data / plsvessel / nano-banana)
|
|
237
|
+
const serializedChain = JSON.stringify({ record, runManifest });
|
|
238
|
+
if (isQuarantinedName(serializedChain)) {
|
|
239
|
+
fail(
|
|
240
|
+
'quarantine_guard',
|
|
241
|
+
'QUARANTINED_SOURCE_REFERENCED',
|
|
242
|
+
`Record "${recordId}" or its run manifest references quarantined/held data (plsvessel / nano-banana)`,
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// 6. §2.2.6 / §12.5: Verdict separation (judge family != generator family on approved records)
|
|
247
|
+
if (record?.judgment && record.judgment.status === 'approved') {
|
|
248
|
+
const judge = record.judgment.judged_by_model || record.judgment.reviewed_by || null;
|
|
249
|
+
const generator =
|
|
250
|
+
record.judgment.generator_model ||
|
|
251
|
+
(pinning?.models?.unet?.name ? `qwen-image:${pinning.models.unet.name}` : null);
|
|
252
|
+
const judgeFamily = normalizeModelFamilyKey(judge);
|
|
253
|
+
const generatorFamily = normalizeModelFamilyKey(generator);
|
|
254
|
+
const sameFamily =
|
|
255
|
+
Boolean(judgeFamily && generatorFamily) &&
|
|
256
|
+
judgeFamily !== 'human' &&
|
|
257
|
+
judgeFamily !== 'rule-based' &&
|
|
258
|
+
judgeFamily === generatorFamily;
|
|
259
|
+
if (!judge || (generator && (judge === generator || sameFamily))) {
|
|
260
|
+
fail(
|
|
261
|
+
'verdict_separation',
|
|
262
|
+
'INVALID_VERDICT_PROVENANCE',
|
|
263
|
+
`Record "${recordId}" has an invalid approval verdict (judge=${judge} [${judgeFamily || 'none'}], generator=${generator} [${generatorFamily || 'none'}])`,
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
return {
|
|
269
|
+
record_id: recordId,
|
|
270
|
+
ok: failures.length === 0,
|
|
271
|
+
links,
|
|
272
|
+
failures,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Verify the provenance chain across a project (or a single record / snapshot).
|
|
278
|
+
*/
|
|
279
|
+
export async function verifyProvenanceChain(
|
|
280
|
+
projectRoot,
|
|
281
|
+
{ recordId = null, snapshotId = null, allowEmpty = false } = {},
|
|
282
|
+
) {
|
|
283
|
+
const recordsDir = join(projectRoot, 'records');
|
|
284
|
+
const recordsToCheck = [];
|
|
285
|
+
const globalFailures = [];
|
|
286
|
+
|
|
287
|
+
if (recordId) {
|
|
288
|
+
const recPath = join(recordsDir, `${recordId}.json`);
|
|
289
|
+
if (!existsSync(recPath)) {
|
|
290
|
+
globalFailures.push({
|
|
291
|
+
record_id: recordId,
|
|
292
|
+
link: 'scope',
|
|
293
|
+
code: 'RECORD_NOT_FOUND',
|
|
294
|
+
message: `Record "${recordId}" not found at records/${recordId}.json`,
|
|
295
|
+
});
|
|
296
|
+
} else {
|
|
297
|
+
recordsToCheck.push(JSON.parse(readFileSync(recPath, 'utf-8')));
|
|
298
|
+
}
|
|
299
|
+
} else if (snapshotId) {
|
|
300
|
+
const incPath = join(projectRoot, 'snapshots', snapshotId, 'included.jsonl');
|
|
301
|
+
if (!existsSync(incPath)) {
|
|
302
|
+
globalFailures.push({
|
|
303
|
+
record_id: null,
|
|
304
|
+
link: 'scope',
|
|
305
|
+
code: 'SNAPSHOT_NOT_FOUND',
|
|
306
|
+
message: `Snapshot "${snapshotId}" included.jsonl not found`,
|
|
307
|
+
});
|
|
308
|
+
} else {
|
|
309
|
+
const lines = readFileSync(incPath, 'utf-8')
|
|
310
|
+
.split('\n')
|
|
311
|
+
.filter((l) => l.trim())
|
|
312
|
+
.map((l) => JSON.parse(l));
|
|
313
|
+
for (const item of lines) {
|
|
314
|
+
const rid = item.record_id || item.id;
|
|
315
|
+
const recPath = join(recordsDir, `${rid}.json`);
|
|
316
|
+
if (!existsSync(recPath)) {
|
|
317
|
+
globalFailures.push({
|
|
318
|
+
record_id: rid,
|
|
319
|
+
link: 'scope',
|
|
320
|
+
code: 'RECORD_NOT_FOUND',
|
|
321
|
+
message: `Snapshot record "${rid}" not found at records/${rid}.json`,
|
|
322
|
+
});
|
|
323
|
+
} else {
|
|
324
|
+
recordsToCheck.push(JSON.parse(readFileSync(recPath, 'utf-8')));
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
} else if (existsSync(recordsDir)) {
|
|
329
|
+
const files = readdirSync(recordsDir)
|
|
330
|
+
.filter((f) => f.endsWith('.json'))
|
|
331
|
+
.sort();
|
|
332
|
+
for (const file of files) {
|
|
333
|
+
recordsToCheck.push(JSON.parse(readFileSync(join(recordsDir, file), 'utf-8')));
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (recordsToCheck.length === 0 && globalFailures.length === 0 && !allowEmpty) {
|
|
338
|
+
globalFailures.push({
|
|
339
|
+
record_id: null,
|
|
340
|
+
link: 'scope',
|
|
341
|
+
code: 'NO_RECORDS_IN_SCOPE',
|
|
342
|
+
message: 'No records found in scope to verify',
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const results = [];
|
|
347
|
+
const failures = [...globalFailures];
|
|
348
|
+
|
|
349
|
+
for (const rec of recordsToCheck) {
|
|
350
|
+
const res = verifyRecordProvenance(projectRoot, rec);
|
|
351
|
+
results.push(res);
|
|
352
|
+
failures.push(...res.failures);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
const passedCount = results.filter((r) => r.ok).length;
|
|
356
|
+
const failedCount = results.length - passedCount;
|
|
357
|
+
|
|
358
|
+
return {
|
|
359
|
+
ok: failures.length === 0,
|
|
360
|
+
project_root: projectRoot,
|
|
361
|
+
records_checked: results.length,
|
|
362
|
+
passed_count: passedCount,
|
|
363
|
+
failed_count: failedCount,
|
|
364
|
+
results,
|
|
365
|
+
failures,
|
|
366
|
+
};
|
|
367
|
+
}
|
package/lib/workflow-profiles.js
CHANGED
|
@@ -12,6 +12,7 @@ import { readFile, readdir, writeFile, mkdir } from 'node:fs/promises';
|
|
|
12
12
|
import { join } from 'node:path';
|
|
13
13
|
import { existsSync } from 'node:fs';
|
|
14
14
|
import { getWorkflowProfilesDir } from './paths.js';
|
|
15
|
+
import { inputError } from './errors.js';
|
|
15
16
|
|
|
16
17
|
const REQUIRED_FIELDS = [
|
|
17
18
|
'workflow_id', 'label', 'lane_id', 'output_mode', 'output_count',
|
|
@@ -105,12 +106,32 @@ export async function getWorkflowProfile(projectRoot, workflowId) {
|
|
|
105
106
|
const dir = getWorkflowProfilesDir(projectRoot);
|
|
106
107
|
const path = join(dir, `${workflowId}.json`);
|
|
107
108
|
if (!existsSync(path)) {
|
|
108
|
-
throw
|
|
109
|
+
throw inputError(
|
|
110
|
+
'WORKFLOW_PROFILE_NOT_FOUND',
|
|
111
|
+
`Workflow profile "${workflowId}" not found at ${path}`,
|
|
112
|
+
'Check the workflow id, or run: sdlab workflow list'
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
// B06: JSON.parse used to run unguarded here, so a malformed profile file
|
|
116
|
+
// threw a raw, uncaught SyntaxError naming nothing. Wrapped in the same
|
|
117
|
+
// structured-error pattern lib/config.js's loadJsonFile already uses.
|
|
118
|
+
let data;
|
|
119
|
+
try {
|
|
120
|
+
data = JSON.parse(await readFile(path, 'utf-8'));
|
|
121
|
+
} catch (err) {
|
|
122
|
+
throw inputError(
|
|
123
|
+
'INPUT_INVALID_JSON',
|
|
124
|
+
`Failed to parse ${path}: ${err.message}`,
|
|
125
|
+
'Check the file for JSON syntax errors (trailing commas, unquoted keys, etc).'
|
|
126
|
+
);
|
|
109
127
|
}
|
|
110
|
-
const data = JSON.parse(await readFile(path, 'utf-8'));
|
|
111
128
|
const { valid, errors } = validateWorkflowProfile(data);
|
|
112
129
|
if (!valid) {
|
|
113
|
-
throw
|
|
130
|
+
throw inputError(
|
|
131
|
+
'WORKFLOW_PROFILE_INVALID',
|
|
132
|
+
`Workflow profile "${workflowId}" is invalid:\n ${errors.join('\n ')}`,
|
|
133
|
+
'Fix the listed field(s) and try again.'
|
|
134
|
+
);
|
|
114
135
|
}
|
|
115
136
|
return data;
|
|
116
137
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mcptoolshop/style-dataset-lab",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.5.0",
|
|
4
4
|
"description": "Canon-aligned dataset production and generation workbench — define visual rules, build versioned training data, compile production briefs, run local workflows, batch-produce, select winners, and re-ingest into your corpus",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -40,10 +40,11 @@
|
|
|
40
40
|
"files": [
|
|
41
41
|
"bin/",
|
|
42
42
|
"lib/",
|
|
43
|
+
"runtime/",
|
|
43
44
|
"scripts/",
|
|
44
45
|
"!scripts/bulk-curate-*.js",
|
|
45
46
|
"!scripts/curate-wave*.js",
|
|
46
|
-
"!scripts/
|
|
47
|
+
"!scripts/__pycache__/**",
|
|
47
48
|
"templates/",
|
|
48
49
|
"README.md",
|
|
49
50
|
"LICENSE",
|
|
@@ -81,7 +82,7 @@
|
|
|
81
82
|
"author": "mcp-tool-shop",
|
|
82
83
|
"license": "MIT",
|
|
83
84
|
"devDependencies": {
|
|
84
|
-
"c8": "^
|
|
85
|
+
"c8": "^12.0.0"
|
|
85
86
|
},
|
|
86
87
|
"dependencies": {
|
|
87
88
|
"yaml": "^2.8.3"
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"template_id": "txt2img-qwen",
|
|
3
|
+
"label": "Text-to-Image — Qwen-Image (non-anime studio base)",
|
|
4
|
+
"description": "Qwen-Image txt2img pipeline: UNETLoader → CLIPLoader(qwen_image) + VAELoader → ModelSamplingAuraFlow(shift) → EmptySD3LatentImage → KSampler(euler/simple) → VAEDecode → SaveImage. The studio's verified NON-ANIME base (Apache-2.0). Selected by project/pack defaults.base = \"qwen-image\".",
|
|
5
|
+
"adapter_target": "comfyui",
|
|
6
|
+
"base": "qwen-image",
|
|
7
|
+
"compatible_modes": ["portrait_set", "expression_sheet", "variant_pack", "moodboard", "silhouette_sheet", "turnaround", "concept_art"],
|
|
8
|
+
"placeholders": {
|
|
9
|
+
"PROMPT": "Positive prompt text",
|
|
10
|
+
"NEGATIVE_PROMPT": "Negative prompt text (always include anime/cartoon/cel-shading terms — non-anime is non-negotiable)",
|
|
11
|
+
"UNET": "Diffusion model filename (default qwen_image_fp8_e4m3fn.safetensors)",
|
|
12
|
+
"CLIP": "Text encoder filename (default qwen_2.5_vl_7b_fp8_scaled.safetensors, loaded with type=qwen_image)",
|
|
13
|
+
"VAE": "VAE filename (default qwen_image_vae.safetensors)",
|
|
14
|
+
"SHIFT": "ModelSamplingAuraFlow shift (default 3.1)",
|
|
15
|
+
"WIDTH": "Image width (default 1024)",
|
|
16
|
+
"HEIGHT": "Image height (default 1024)",
|
|
17
|
+
"STEPS": "Sampling steps (default 22)",
|
|
18
|
+
"CFG": "Classifier-free guidance scale (default 3.5)",
|
|
19
|
+
"SAMPLER": "Sampler name (default euler)",
|
|
20
|
+
"SCHEDULER": "Scheduler name (default simple)",
|
|
21
|
+
"SEED": "Random seed"
|
|
22
|
+
},
|
|
23
|
+
"defaults": {
|
|
24
|
+
"unet": "qwen_image_fp8_e4m3fn.safetensors",
|
|
25
|
+
"clip": "qwen_2.5_vl_7b_fp8_scaled.safetensors",
|
|
26
|
+
"clip_type": "qwen_image",
|
|
27
|
+
"vae": "qwen_image_vae.safetensors",
|
|
28
|
+
"weight_dtype": "default",
|
|
29
|
+
"shift": 3.1,
|
|
30
|
+
"sampler": "euler",
|
|
31
|
+
"scheduler": "simple",
|
|
32
|
+
"steps": 22,
|
|
33
|
+
"cfg": 3.5,
|
|
34
|
+
"width": 1024,
|
|
35
|
+
"height": 1024
|
|
36
|
+
},
|
|
37
|
+
"notes": "Built programmatically — the JSON above is a reference. lib/adapters/comfyui-workflows.js buildQwenGraph() constructs the live graph; buildWorkflowGraph() and scripts/generate.js both route to it when base === 'qwen-image'. Node ids/wiring mirror scripts/qwen_generate.py for cross-runner reproducibility. NEVER substitute an anime checkpoint (Illustrious/Pony/Animagine/NoobAI) — see memory/feedback_no_anime.md."
|
|
38
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"template_id": "txt2img-standard",
|
|
3
|
+
"label": "Text-to-Image Standard",
|
|
4
|
+
"description": "Standard txt2img pipeline: checkpoint → LoRA chain → KSampler → VAEDecode → SaveImage.",
|
|
5
|
+
"adapter_target": "comfyui",
|
|
6
|
+
"compatible_modes": ["portrait_set", "expression_sheet", "variant_pack", "moodboard", "silhouette_sheet", "turnaround"],
|
|
7
|
+
"placeholders": {
|
|
8
|
+
"PROMPT": "Positive prompt text",
|
|
9
|
+
"NEGATIVE_PROMPT": "Negative prompt text",
|
|
10
|
+
"CHECKPOINT": "Checkpoint filename",
|
|
11
|
+
"WIDTH": "Image width",
|
|
12
|
+
"HEIGHT": "Image height",
|
|
13
|
+
"STEPS": "Sampling steps",
|
|
14
|
+
"CFG": "Classifier-free guidance scale",
|
|
15
|
+
"SAMPLER": "Sampler name",
|
|
16
|
+
"SCHEDULER": "Scheduler name",
|
|
17
|
+
"SEED": "Random seed",
|
|
18
|
+
"LORAS": "Array of {name, weight, clip_weight?} objects"
|
|
19
|
+
},
|
|
20
|
+
"notes": "This template is built programmatically — the JSON below is a reference. comfyui-workflows.js uses buildWorkflowGraph() to construct the graph dynamically with LoRA chain support."
|
|
21
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* _resume.js — internal helper module (NOT a CLI command; underscore prefix
|
|
3
|
+
* matches the existing scripts/_batch_cut.py / _seam_cutout.py convention
|
|
4
|
+
* for "helper, not dispatched by bin/sdlab.js").
|
|
5
|
+
*
|
|
6
|
+
* H5: generate.js was the only one of the four ComfyUI generator scripts
|
|
7
|
+
* that implemented `--resume` — skip an asset whose provenance record AND
|
|
8
|
+
* output image already exist on disk, WITHOUT skipping its seed slot, so a
|
|
9
|
+
* resumed run stays bit-identical to an uninterrupted one. generate-identity
|
|
10
|
+
* / generate-controlnet / generate-ipadapter had no --resume at all, which
|
|
11
|
+
* is expensive specifically for generate-identity (the priciest per-image
|
|
12
|
+
* generator in the pipeline): interrupting a 12-subject run after subject 9
|
|
13
|
+
* meant regenerating all 12 from scratch.
|
|
14
|
+
*
|
|
15
|
+
* Extracted here (once) so all four generators — and any future one — get
|
|
16
|
+
* the same discipline by importing it, instead of by copy-pasting the check
|
|
17
|
+
* next to generate.js's inline version. `lib/` is outside this fix's file
|
|
18
|
+
* ownership, so this lives under scripts/** instead; see the dispatch
|
|
19
|
+
* report for the full rationale.
|
|
20
|
+
*
|
|
21
|
+
* `requireRecord` exists because not every generator writes a provenance
|
|
22
|
+
* record: generate.js and generate-identity.js both write
|
|
23
|
+
* records/<id>.json, but generate-controlnet.js and generate-ipadapter.js
|
|
24
|
+
* are lighter-weight discovery scripts that only ever write the output PNG
|
|
25
|
+
* — they have no records/ write path at all. Requiring a record that a
|
|
26
|
+
* script structurally never writes would make --resume a permanent no-op
|
|
27
|
+
* for those two, which is worse than not having the flag. Pass
|
|
28
|
+
* `{ requireRecord: false }` for generators with no records/ write path.
|
|
29
|
+
*/
|
|
30
|
+
import { existsSync } from 'node:fs';
|
|
31
|
+
import { join } from 'node:path';
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @param {string} projectRoot — GAME_ROOT
|
|
35
|
+
* @param {string} assetId — the record id / output filename stem (no extension)
|
|
36
|
+
* @param {Object} [opts]
|
|
37
|
+
* @param {boolean} [opts.requireRecord=true] — also require records/<id>.json
|
|
38
|
+
* @param {string} [opts.recordsSubdir='records']
|
|
39
|
+
* @param {string} [opts.candidatesSubdir='outputs/candidates']
|
|
40
|
+
* @returns {boolean} true when the asset is safe to skip on --resume
|
|
41
|
+
*/
|
|
42
|
+
export function isResumable(projectRoot, assetId, opts = {}) {
|
|
43
|
+
const requireRecord = opts.requireRecord !== false;
|
|
44
|
+
const candidatesSubdir = opts.candidatesSubdir || 'outputs/candidates';
|
|
45
|
+
const imagePath = join(projectRoot, candidatesSubdir, `${assetId}.png`);
|
|
46
|
+
if (!existsSync(imagePath)) return false;
|
|
47
|
+
|
|
48
|
+
if (!requireRecord) return true;
|
|
49
|
+
|
|
50
|
+
const recordsSubdir = opts.recordsSubdir || 'records';
|
|
51
|
+
const recordPath = join(projectRoot, recordsSubdir, `${assetId}.json`);
|
|
52
|
+
return existsSync(recordPath);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Print the standard "(resumed — skipped)" progress line — matches
|
|
57
|
+
* generate.js's console.log verbatim so all four generators read
|
|
58
|
+
* identically under --resume.
|
|
59
|
+
*/
|
|
60
|
+
export function logResumedSkip(index, total, assetId) {
|
|
61
|
+
console.log(` [${index}/${total}] ${assetId} \x1b[2m(resumed — skipped)\x1b[0m`);
|
|
62
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* _shared-args.js — internal helper module (NOT a CLI command; underscore
|
|
3
|
+
* prefix matches the existing scripts/_batch_cut.py / _seam_cutout.py
|
|
4
|
+
* convention for "helper, not dispatched by bin/sdlab.js").
|
|
5
|
+
*
|
|
6
|
+
* H4 (second half): the five canon-*.js scripts (canon-build.js,
|
|
7
|
+
* canon-drift.js, canon-freeze.js, canon-freeze-status.js,
|
|
8
|
+
* canon-unfreeze.js) all use Node's built-in `util.parseArgs({strict:
|
|
9
|
+
* false})`. strict:false is there deliberately, so a bare positional
|
|
10
|
+
* argument (the entity id) isn't rejected — but it has a side effect: ANY
|
|
11
|
+
* unrecognized `--flag` is silently accepted too. A misspelled OPTIONAL
|
|
12
|
+
* flag (`--staus follow` instead of `--status follow`, `--jsn` instead of
|
|
13
|
+
* `--json`) produces no error and no warning — the option you meant to set
|
|
14
|
+
* just quietly keeps its default, and Node's parser stores the typo under
|
|
15
|
+
* ITS OWN key in `parsed.values`, never touching the one you meant.
|
|
16
|
+
*
|
|
17
|
+
* assertKnownOptions() closes that gap without giving up strict:false's
|
|
18
|
+
* positional leniency: after parsing, walk `parsed.values`'s keys — since
|
|
19
|
+
* every DECLARED option (default or explicit) and every UNDECLARED flag
|
|
20
|
+
* alike lands there — and reject any key outside the known option set.
|
|
21
|
+
* Mirrors lib/args.js's own UNKNOWN_FLAG shape (same code, same
|
|
22
|
+
* "Did you mean" hint via findClosest) so a typo on a canon-*.js flag reads
|
|
23
|
+
* identically to a typo anywhere else in the CLI.
|
|
24
|
+
*/
|
|
25
|
+
import { inputError } from '../lib/errors.js';
|
|
26
|
+
import { findClosest } from '../lib/args.js';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @param {Object} parsedValues — the `.values` object returned by
|
|
30
|
+
* node:util's parseArgs({strict:false, ...})
|
|
31
|
+
* @param {Object} optionsSpec — the SAME `options` object passed to parseArgs
|
|
32
|
+
* @param {{ command?: string }} [opts] — command label for the hint text
|
|
33
|
+
*/
|
|
34
|
+
export function assertKnownOptions(parsedValues, optionsSpec, opts = {}) {
|
|
35
|
+
const known = new Set(Object.keys(optionsSpec));
|
|
36
|
+
for (const key of Object.keys(parsedValues || {})) {
|
|
37
|
+
if (known.has(key)) continue;
|
|
38
|
+
const suggestion = findClosest(key, [...known]);
|
|
39
|
+
const hint = suggestion
|
|
40
|
+
? `Did you mean --${suggestion}? (Run "sdlab ${opts.command ? opts.command + ' ' : ''}--help" to see all supported flags.)`
|
|
41
|
+
: `Run "sdlab ${opts.command ? opts.command + ' ' : ''}--help" to see supported flags.`;
|
|
42
|
+
throw inputError('UNKNOWN_FLAG', `Unknown flag: --${key}`, hint);
|
|
43
|
+
}
|
|
44
|
+
}
|