@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.es.md +117 -55
- package/README.fr.md +125 -65
- package/README.hi.md +117 -68
- package/README.it.md +119 -58
- package/README.ja.md +124 -74
- package/README.md +73 -7
- package/README.pt-BR.md +128 -63
- package/README.zh.md +116 -65
- package/bin/sdlab.js +100 -4
- package/lib/adapters/ai-toolkit.js +224 -35
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +94 -11
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/asset-ingest.js +575 -0
- package/lib/asset-masks.js +323 -0
- package/lib/asset-source.js +1176 -0
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-author.js +350 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/canon-review-gate.js +205 -0
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/colab-launcher-guards.js +285 -0
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +16 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +194 -6
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/lane1-render.js +668 -0
- package/lib/measure.js +422 -0
- package/lib/memory-store.js +663 -0
- package/lib/model-licences.js +305 -0
- package/lib/npy-meta.js +130 -0
- package/lib/paths.js +196 -7
- package/lib/png-meta.js +512 -0
- package/lib/records.js +54 -8
- package/lib/reingest-selected.js +113 -17
- package/lib/reingest.js +27 -3
- package/lib/run-manifest.js +159 -25
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/train-runs.js +214 -0
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +3 -0
- package/lib/verify-chain.js +367 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +4 -3
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/asset-ingest.js +160 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-draft.js +82 -0
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +31 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
- package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
- package/scripts/colab/colab_window.sh +587 -0
- package/scripts/colab/lane1_render.py +970 -0
- package/scripts/colab/launch_guards.py +564 -0
- package/scripts/colab/launch_lane1_stub.sh +309 -0
- package/scripts/colab/launch_lane3_stub.sh +366 -0
- package/scripts/colab/remote_runner.py +59 -0
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +182 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +86 -17
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/verify.js +67 -0
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model and LoRA licence catalog and verification helpers (G4, §5.2–§5.3).
|
|
3
|
+
*
|
|
4
|
+
* Cleared SPDX licences for CC-BY-4.0 dataset/LoRA release:
|
|
5
|
+
* Apache-2.0, MIT, BSD-2-Clause, BSD-3-Clause, CC0-1.0, CC-BY-4.0
|
|
6
|
+
*
|
|
7
|
+
* Excluded / held models (FLUX.1-dev, FLUX.2-dev, FLUX.2-klein-9B, SD 3.5,
|
|
8
|
+
* Qwen-Image 2.1, community SDXL merges, and the held Portlight `plsvessel`
|
|
9
|
+
* LoRA trained on 336 Nano Banana-derived pairs) are explicitly flagged
|
|
10
|
+
* `cc_by_4_eligible: false` and `excluded: true`.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { inputError } from './errors.js';
|
|
14
|
+
|
|
15
|
+
export const CLEARED_SPDX_IDS = Object.freeze([
|
|
16
|
+
'Apache-2.0',
|
|
17
|
+
'MIT',
|
|
18
|
+
'BSD-2-Clause',
|
|
19
|
+
'BSD-3-Clause',
|
|
20
|
+
'CC0-1.0',
|
|
21
|
+
'CC-BY-4.0',
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
export const QUARANTINED_PATTERNS = Object.freeze([
|
|
25
|
+
/plsvessel/i,
|
|
26
|
+
/nano[_-\s]?banana/i,
|
|
27
|
+
/held[_-\s]?portlight/i,
|
|
28
|
+
]);
|
|
29
|
+
|
|
30
|
+
const KNOWN_CATALOG = Object.freeze([
|
|
31
|
+
// Quarantined / held Portlight distillation artifacts (§5.3, §8.1)
|
|
32
|
+
{
|
|
33
|
+
pattern: /^(?:plsvessel|.*nano[_-\s]?banana.*)/i,
|
|
34
|
+
spdx_id: 'PROPRIETARY-HELD-DISTILL',
|
|
35
|
+
url: 'https://ai.google.dev/gemini-api/terms',
|
|
36
|
+
verified_at: '2026-10-25',
|
|
37
|
+
cc_by_4_eligible: false,
|
|
38
|
+
excluded: true,
|
|
39
|
+
quarantined: true,
|
|
40
|
+
status: 'held',
|
|
41
|
+
reason: 'Held Portlight LoRA trained on 336 Nano Banana-derived pairs; not eligible for CC-BY-4.0 release or training',
|
|
42
|
+
},
|
|
43
|
+
// Excluded Qwen-Image 2.1 proprietary API (§5.3)
|
|
44
|
+
{
|
|
45
|
+
pattern: /^qwen[_-]image[_-]2\.1/i,
|
|
46
|
+
spdx_id: 'PROPRIETARY-API',
|
|
47
|
+
url: 'https://www.alibabacloud.com/help/en/model-studio/',
|
|
48
|
+
verified_at: '2026-10-25',
|
|
49
|
+
cc_by_4_eligible: false,
|
|
50
|
+
excluded: true,
|
|
51
|
+
status: 'excluded',
|
|
52
|
+
reason: 'Proprietary API-only model; no open weights',
|
|
53
|
+
},
|
|
54
|
+
// Cleared Qwen-Image 20B and VAE (§5.2)
|
|
55
|
+
{
|
|
56
|
+
pattern: /^qwen[_-]image/i,
|
|
57
|
+
spdx_id: 'Apache-2.0',
|
|
58
|
+
url: 'https://github.com/QwenLM/Qwen-Image/blob/main/LICENSE',
|
|
59
|
+
verified_at: '2026-10-25',
|
|
60
|
+
cc_by_4_eligible: true,
|
|
61
|
+
excluded: false,
|
|
62
|
+
status: 'cleared',
|
|
63
|
+
},
|
|
64
|
+
// Cleared Qwen 2.5 VL 7B text encoder (§5.2)
|
|
65
|
+
{
|
|
66
|
+
pattern: /^qwen[_-]?2\.5[_-]vl[_-]7b/i,
|
|
67
|
+
spdx_id: 'Apache-2.0',
|
|
68
|
+
url: 'https://github.com/QwenLM/Qwen2.5-VL/blob/main/LICENSE',
|
|
69
|
+
verified_at: '2026-10-28',
|
|
70
|
+
cc_by_4_eligible: true,
|
|
71
|
+
excluded: false,
|
|
72
|
+
status: 'cleared',
|
|
73
|
+
},
|
|
74
|
+
// Cleared FLUX.2 [klein] 4B and FLUX.2 VAE (§5.2)
|
|
75
|
+
{
|
|
76
|
+
pattern: /^(?:flux[_-]?2[_-]klein[_-](?:base[_-])?4b|flux2[_-]vae|ae\.safetensors)/i,
|
|
77
|
+
spdx_id: 'Apache-2.0',
|
|
78
|
+
url: 'https://huggingface.co/black-forest-labs/FLUX.2-klein-4B/blob/main/LICENSE.md',
|
|
79
|
+
verified_at: '2026-10-25',
|
|
80
|
+
cc_by_4_eligible: true,
|
|
81
|
+
excluded: false,
|
|
82
|
+
status: 'cleared',
|
|
83
|
+
},
|
|
84
|
+
// Cleared Muse Glimmer 30B / Qwen3-VL-30B-A3B-Instruct / Qwen2.5-VL-32B (§5.2)
|
|
85
|
+
{
|
|
86
|
+
pattern: /^qwen(?:3[_-]vl[_-]30b[_-]a3b|2\.5[_-]vl[_-]32b|2\.5[:_-]14b|3[:_-]14b)/i,
|
|
87
|
+
spdx_id: 'Apache-2.0',
|
|
88
|
+
url: 'https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct/blob/main/LICENSE',
|
|
89
|
+
verified_at: '2026-10-25',
|
|
90
|
+
cc_by_4_eligible: true,
|
|
91
|
+
excluded: false,
|
|
92
|
+
status: 'cleared',
|
|
93
|
+
},
|
|
94
|
+
// Excluded FLUX.1 [dev] (§5.3)
|
|
95
|
+
{
|
|
96
|
+
pattern: /^flux[_-]?1[_-]dev/i,
|
|
97
|
+
spdx_id: 'FLUX-1-DEV-NC',
|
|
98
|
+
url: 'https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md',
|
|
99
|
+
verified_at: '2026-10-25',
|
|
100
|
+
cc_by_4_eligible: false,
|
|
101
|
+
excluded: true,
|
|
102
|
+
status: 'excluded',
|
|
103
|
+
reason: 'FLUX.1 [dev] Non-Commercial License restricts model and derivatives',
|
|
104
|
+
},
|
|
105
|
+
// Excluded FLUX.2 [dev] and FLUX.2 [klein] 9B (§5.3)
|
|
106
|
+
{
|
|
107
|
+
pattern: /^flux[_-]?2[_-](?:dev|klein[_-](?:base[_-])?9b)/i,
|
|
108
|
+
spdx_id: 'FLUX-2-NC',
|
|
109
|
+
url: 'https://huggingface.co/black-forest-labs/FLUX.2-dev/blob/main/LICENSE.md',
|
|
110
|
+
verified_at: '2026-10-25',
|
|
111
|
+
cc_by_4_eligible: false,
|
|
112
|
+
excluded: true,
|
|
113
|
+
status: 'excluded',
|
|
114
|
+
reason: 'FLUX.2 Non-Commercial License',
|
|
115
|
+
},
|
|
116
|
+
// Excluded Stable Diffusion 3.5 family (§5.3)
|
|
117
|
+
{
|
|
118
|
+
pattern: /^(?:sd3\.?5|stable[_-]diffusion[_-]3\.?5)/i,
|
|
119
|
+
spdx_id: 'STABILITY-COMMUNITY',
|
|
120
|
+
url: 'https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md',
|
|
121
|
+
verified_at: '2026-10-25',
|
|
122
|
+
cc_by_4_eligible: false,
|
|
123
|
+
excluded: true,
|
|
124
|
+
status: 'excluded',
|
|
125
|
+
reason: 'Stability AI Community License revenue cap and derivative terms',
|
|
126
|
+
},
|
|
127
|
+
// Excluded community SDXL merges (§5.3)
|
|
128
|
+
{
|
|
129
|
+
pattern: /^(?:dreamshaper|juggernaut|pony|illustrious|animagine)/i,
|
|
130
|
+
spdx_id: 'UNCLEARED-COMMUNITY-MERGE',
|
|
131
|
+
url: null,
|
|
132
|
+
verified_at: '2026-10-25',
|
|
133
|
+
cc_by_4_eligible: false,
|
|
134
|
+
excluded: true,
|
|
135
|
+
status: 'excluded',
|
|
136
|
+
reason: 'Community SDXL merge without audited upstream weight lineage',
|
|
137
|
+
},
|
|
138
|
+
]);
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Check whether a string references a quarantined/held artifact (`plsvessel`, `nano-banana`).
|
|
142
|
+
*/
|
|
143
|
+
export function isQuarantinedName(value) {
|
|
144
|
+
if (!value || typeof value !== 'string') return false;
|
|
145
|
+
return QUARANTINED_PATTERNS.some((re) => re.test(value));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Normalize a licence descriptor into the canonical `{ spdx_id, licence_id, url, licence_url, verified_at, date_read, cc_by_4_eligible, excluded, status }` shape.
|
|
150
|
+
*/
|
|
151
|
+
function normalizeLicenceObject(raw) {
|
|
152
|
+
const spdxId = raw.spdx_id || raw.licence_id || null;
|
|
153
|
+
const url = raw.url || raw.licence_url || null;
|
|
154
|
+
const verifiedAt = raw.verified_at || raw.date_read || null;
|
|
155
|
+
const excluded = Boolean(raw.excluded);
|
|
156
|
+
const quarantined = Boolean(raw.quarantined);
|
|
157
|
+
const eligible =
|
|
158
|
+
!excluded &&
|
|
159
|
+
!quarantined &&
|
|
160
|
+
raw.cc_by_4_eligible !== false &&
|
|
161
|
+
Boolean(spdxId && CLEARED_SPDX_IDS.includes(spdxId)) &&
|
|
162
|
+
typeof url === 'string' &&
|
|
163
|
+
url.trim().length > 0 &&
|
|
164
|
+
typeof verifiedAt === 'string' &&
|
|
165
|
+
verifiedAt.trim().length > 0;
|
|
166
|
+
return {
|
|
167
|
+
spdx_id: spdxId,
|
|
168
|
+
licence_id: spdxId,
|
|
169
|
+
url,
|
|
170
|
+
licence_url: url,
|
|
171
|
+
verified_at: verifiedAt,
|
|
172
|
+
date_read: verifiedAt,
|
|
173
|
+
cc_by_4_eligible: eligible,
|
|
174
|
+
excluded,
|
|
175
|
+
...(quarantined ? { quarantined: true } : {}),
|
|
176
|
+
status: raw.status || (eligible ? 'cleared' : excluded ? 'excluded' : 'unknown'),
|
|
177
|
+
...(raw.reason ? { reason: raw.reason } : {}),
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Resolve licence metadata for a model or LoRA filename.
|
|
183
|
+
*
|
|
184
|
+
* @param {string} name - Model or LoRA filename
|
|
185
|
+
* @param {Object|null} [explicitLicence] - Explicit override `{ licence_id, licence_url, date_read }`
|
|
186
|
+
* @param {Object|null} [modelLicences] - Map of `filename -> licence` overrides
|
|
187
|
+
*/
|
|
188
|
+
export function resolveModelLicence(name, explicitLicence = null, modelLicences = null) {
|
|
189
|
+
if (!name || typeof name !== 'string') {
|
|
190
|
+
return normalizeLicenceObject({ status: 'unknown', cc_by_4_eligible: false });
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Always check catalog first for quarantined/excluded models so an override cannot mask a held model.
|
|
194
|
+
const baseName = name.split(/[\\/]/).pop();
|
|
195
|
+
const catalogMatch = KNOWN_CATALOG.find((entry) => entry.pattern.test(baseName));
|
|
196
|
+
if (catalogMatch && catalogMatch.excluded) {
|
|
197
|
+
return normalizeLicenceObject(catalogMatch);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const override = explicitLicence || (modelLicences && (modelLicences[name] || modelLicences[baseName]));
|
|
201
|
+
if (override && typeof override === 'object') {
|
|
202
|
+
return normalizeLicenceObject(override);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (catalogMatch) {
|
|
206
|
+
return normalizeLicenceObject(catalogMatch);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
return normalizeLicenceObject({ status: 'unknown', cc_by_4_eligible: false });
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Check whether a model/LoRA entry carries a non-empty sha256 digest.
|
|
214
|
+
*/
|
|
215
|
+
export function hasModelSha256(entry) {
|
|
216
|
+
if (!entry || typeof entry !== 'object') return false;
|
|
217
|
+
return typeof entry.sha256 === 'string' && entry.sha256.trim().length > 0;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Normalize a model identifier to its canonical family key (DESIGN-v3 §12.5).
|
|
222
|
+
*/
|
|
223
|
+
export function normalizeModelFamilyKey(modelId) {
|
|
224
|
+
if (!modelId || typeof modelId !== 'string') return '';
|
|
225
|
+
const raw = modelId.trim().toLowerCase();
|
|
226
|
+
if (!raw) return '';
|
|
227
|
+
if (raw.startsWith('human:') || raw === 'human') return 'human';
|
|
228
|
+
if (raw.startsWith('rule-based:')) return 'rule-based';
|
|
229
|
+
const stripped = raw
|
|
230
|
+
.replace(/^model:/, '')
|
|
231
|
+
.replace(/@sha256:[0-9a-f]{64}$/, '')
|
|
232
|
+
.replace(/\.(safetensors|ckpt|pt|pth|bin|gguf)$/, '');
|
|
233
|
+
if (stripped.includes('glimmer') || stripped.includes('muse-glimmer') || stripped.includes('muse_glimmer')) {
|
|
234
|
+
return 'family:muse-glimmer-30b';
|
|
235
|
+
}
|
|
236
|
+
if (stripped.includes('qwen')) {
|
|
237
|
+
return 'family:qwen';
|
|
238
|
+
}
|
|
239
|
+
if (stripped.includes('gemma')) {
|
|
240
|
+
return 'family:gemma';
|
|
241
|
+
}
|
|
242
|
+
if (stripped.includes('internvl')) {
|
|
243
|
+
return 'family:internvl';
|
|
244
|
+
}
|
|
245
|
+
if (stripped.includes('siglip') || stripped.includes('ai-eyes')) {
|
|
246
|
+
return 'family:siglip2';
|
|
247
|
+
}
|
|
248
|
+
if (stripped.includes('flux') || stripped === 'ae') {
|
|
249
|
+
return 'family:flux';
|
|
250
|
+
}
|
|
251
|
+
if (/^(?:sd3\.?5|stable[_-]diffusion[_-]3\.?5)/.test(stripped)) {
|
|
252
|
+
return 'family:sd3';
|
|
253
|
+
}
|
|
254
|
+
if (
|
|
255
|
+
stripped.startsWith('sdxl') ||
|
|
256
|
+
stripped.includes('stable-diffusion-xl') ||
|
|
257
|
+
stripped.includes('stable_diffusion_xl') ||
|
|
258
|
+
/^(?:dreamshaper|juggernaut|pony|illustrious|animagine)/.test(stripped)
|
|
259
|
+
) {
|
|
260
|
+
return 'family:sdxl';
|
|
261
|
+
}
|
|
262
|
+
return stripped;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Check whether a model/LoRA pinning entry has a valid cleared SPDX licence,
|
|
267
|
+
* non-empty URL, non-empty date_read, and (by default) a non-empty sha256.
|
|
268
|
+
*/
|
|
269
|
+
export function isClearedLicenceEntry(entry, { requireSha256 = true } = {}) {
|
|
270
|
+
if (!entry || typeof entry !== 'object') return false;
|
|
271
|
+
if (entry.name && isQuarantinedName(entry.name)) return false;
|
|
272
|
+
if (requireSha256 && !hasModelSha256(entry)) return false;
|
|
273
|
+
const lic = entry.licence || {};
|
|
274
|
+
const spdxId = entry.licence_id || lic.spdx_id || lic.licence_id;
|
|
275
|
+
const url = entry.licence_url || lic.url || lic.licence_url;
|
|
276
|
+
const dateRead = entry.date_read || lic.verified_at || lic.date_read;
|
|
277
|
+
if (lic.excluded === true || lic.cc_by_4_eligible === false) return false;
|
|
278
|
+
if (!spdxId || !CLEARED_SPDX_IDS.includes(spdxId)) return false;
|
|
279
|
+
if (typeof url !== 'string' || !url.trim()) return false;
|
|
280
|
+
if (typeof dateRead !== 'string' || !dateRead.trim()) return false;
|
|
281
|
+
return true;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Enforce that every model and LoRA in a pinning block has a cleared licence
|
|
286
|
+
* unless `allowUnlicensedResearch` is explicitly set.
|
|
287
|
+
*/
|
|
288
|
+
export function assertClearedPinningLicences(models, loras, { allowUnlicensedResearch = false, requireSha256 = false } = {}) {
|
|
289
|
+
if (allowUnlicensedResearch) return;
|
|
290
|
+
const entries = [
|
|
291
|
+
...Object.entries(models || {}).map(([role, entry]) => ({ role, ...entry })),
|
|
292
|
+
...(loras || []).map((entry, idx) => ({ role: `lora[${idx}]`, ...entry })),
|
|
293
|
+
];
|
|
294
|
+
const bad = entries.filter((e) => !isClearedLicenceEntry(e, { requireSha256 }));
|
|
295
|
+
if (bad.length > 0) {
|
|
296
|
+
const summary = bad
|
|
297
|
+
.map((b) => `${b.role}:${b.name || '<unnamed>'} (${b.licence_id || 'unlicensed'})`)
|
|
298
|
+
.join(', ');
|
|
299
|
+
throw inputError(
|
|
300
|
+
'MODEL_LICENCE_REJECTED',
|
|
301
|
+
`Model/LoRA licence check failed for: ${summary}`,
|
|
302
|
+
'Only cleared SPDX weights (Apache-2.0, MIT, BSD-*, CC0-1.0, CC-BY-4.0) with a verified licence_url are eligible for CC-BY-4.0 release.',
|
|
303
|
+
);
|
|
304
|
+
}
|
|
305
|
+
}
|
package/lib/npy-meta.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NPY header reader — structural validation for .npy channel files in the
|
|
3
|
+
* asset-lane ingest contract (docs/asset-lane-kickoff.md).
|
|
4
|
+
*
|
|
5
|
+
* The asset contract lets a manifest declare a channel stored as a NumPy
|
|
6
|
+
* array (facet's standing sidecars, e.g. a styled/unstyled mask or a future
|
|
7
|
+
* per-texel view-owner map). sdlab never interprets the array's MEANING —
|
|
8
|
+
* it proves the file's declared dtype and shape match its actual header,
|
|
9
|
+
* the same way lib/png-meta.js proves a PNG's declared encoding matches
|
|
10
|
+
* its IHDR. Header only; the payload is hashed, not parsed.
|
|
11
|
+
*
|
|
12
|
+
* Format reference: NumPy NEP 1 (the .npy format, versions 1.0/2.0/3.0):
|
|
13
|
+
* magic "\x93NUMPY", one version byte pair, a little-endian header length
|
|
14
|
+
* (2 bytes for v1, 4 bytes for v2+), then a Python dict literal with keys
|
|
15
|
+
* 'descr', 'fortran_order', 'shape'.
|
|
16
|
+
*
|
|
17
|
+
* Schema 1.1.0 adds `readNpyValues` — still no MEANING, but a categorical
|
|
18
|
+
* npy channel declares its classes, and "declared ⊆ actual" is provable only
|
|
19
|
+
* by looking at the values. Kept to single-byte dtypes: those are what a
|
|
20
|
+
* per-pixel class/owner map is, and a reader that cannot honestly decode a
|
|
21
|
+
* dtype says so rather than skipping the proof silently.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { inputError } from './errors.js';
|
|
25
|
+
|
|
26
|
+
const MAGIC = Buffer.from('\x93NUMPY', 'latin1');
|
|
27
|
+
|
|
28
|
+
function npyError(message, hint) {
|
|
29
|
+
return inputError('NPY_INVALID', message, hint);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Parse a .npy file's header.
|
|
34
|
+
*
|
|
35
|
+
* @param {Buffer} buf — at least the first ~64KB of the file (the whole
|
|
36
|
+
* file is fine; only the header is read)
|
|
37
|
+
* @param {string} [label] — filename for error messages
|
|
38
|
+
* @returns {{dtype:string, fortranOrder:boolean, shape:number[],
|
|
39
|
+
* version:string, headerLength:number, dataOffset:number}}
|
|
40
|
+
*/
|
|
41
|
+
export function parseNpyHeader(buf, label = 'npy') {
|
|
42
|
+
if (buf.length < 10 || !buf.subarray(0, 6).equals(MAGIC)) {
|
|
43
|
+
throw npyError(`${label}: not a .npy file (bad magic).`);
|
|
44
|
+
}
|
|
45
|
+
const major = buf[6];
|
|
46
|
+
const minor = buf[7];
|
|
47
|
+
let headerLength;
|
|
48
|
+
let headerStart;
|
|
49
|
+
if (major === 1) {
|
|
50
|
+
headerLength = buf.readUInt16LE(8);
|
|
51
|
+
headerStart = 10;
|
|
52
|
+
} else if (major === 2 || major === 3) {
|
|
53
|
+
if (buf.length < 12) throw npyError(`${label}: truncated v${major} header.`);
|
|
54
|
+
headerLength = buf.readUInt32LE(8);
|
|
55
|
+
headerStart = 12;
|
|
56
|
+
} else {
|
|
57
|
+
throw npyError(`${label}: unsupported .npy format version ${major}.${minor}.`);
|
|
58
|
+
}
|
|
59
|
+
if (buf.length < headerStart + headerLength) {
|
|
60
|
+
throw npyError(`${label}: truncated header (declared ${headerLength} bytes).`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const header = buf.toString('latin1', headerStart, headerStart + headerLength);
|
|
64
|
+
|
|
65
|
+
const descrMatch = header.match(/'descr'\s*:\s*'([^']+)'/);
|
|
66
|
+
const fortranMatch = header.match(/'fortran_order'\s*:\s*(True|False)/);
|
|
67
|
+
const shapeMatch = header.match(/'shape'\s*:\s*\(([^)]*)\)/);
|
|
68
|
+
if (!descrMatch || !fortranMatch || !shapeMatch) {
|
|
69
|
+
throw npyError(`${label}: header dict is missing descr/fortran_order/shape.`);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const shape = shapeMatch[1]
|
|
73
|
+
.split(',')
|
|
74
|
+
.map((s) => s.trim())
|
|
75
|
+
.filter((s) => s.length > 0)
|
|
76
|
+
.map((s) => {
|
|
77
|
+
const n = Number(s);
|
|
78
|
+
if (!Number.isInteger(n) || n < 0) {
|
|
79
|
+
throw npyError(`${label}: non-integer dimension "${s}" in shape.`);
|
|
80
|
+
}
|
|
81
|
+
return n;
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
dtype: descrMatch[1],
|
|
86
|
+
fortranOrder: fortranMatch[1] === 'True',
|
|
87
|
+
shape,
|
|
88
|
+
version: `${major}.${minor}`,
|
|
89
|
+
headerLength,
|
|
90
|
+
dataOffset: headerStart + headerLength,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Single-byte dtypes whose values this contract can read. A categorical npy
|
|
96
|
+
* channel must declare one of these — see readNpyValues.
|
|
97
|
+
*/
|
|
98
|
+
export const NPY_PROVABLE_DTYPES = new Set(['|i1', '|u1', '|b1']);
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Read an npy payload as a typed array. Single-byte dtypes only, so element
|
|
102
|
+
* order is byte order and `fortran_order` does not change the VALUE SET a
|
|
103
|
+
* categorical proof examines (it changes which index means which pixel —
|
|
104
|
+
* that is the consumer's business, and numpy reads the flag itself).
|
|
105
|
+
*
|
|
106
|
+
* Also catches a truncation the header-only parse cannot: a file whose
|
|
107
|
+
* payload is shorter than its declared shape.
|
|
108
|
+
*
|
|
109
|
+
* @param {Buffer} buf — the whole file
|
|
110
|
+
* @param {ReturnType<typeof parseNpyHeader>} header
|
|
111
|
+
* @param {string} [label]
|
|
112
|
+
* @returns {Int8Array|Uint8Array}
|
|
113
|
+
*/
|
|
114
|
+
export function readNpyValues(buf, header, label = 'npy') {
|
|
115
|
+
if (!NPY_PROVABLE_DTYPES.has(header.dtype)) {
|
|
116
|
+
throw npyError(
|
|
117
|
+
`${label}: dtype ${header.dtype} is not one this contract reads values from.`,
|
|
118
|
+
`Value proofs cover ${[...NPY_PROVABLE_DTYPES].join(', ')} — the single-byte dtypes a per-pixel class or owner map uses. Export the channel in one of those, or drop "categorical" and let it ride as an opaque declared array.`
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
const count = header.shape.reduce((a, b) => a * b, 1);
|
|
122
|
+
const available = buf.length - header.dataOffset;
|
|
123
|
+
if (available < count) {
|
|
124
|
+
throw npyError(
|
|
125
|
+
`${label}: truncated payload — shape ${JSON.stringify(header.shape)} needs ${count} byte(s), the file carries ${available} after the header.`
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
const Ctor = header.dtype === '|i1' ? Int8Array : Uint8Array;
|
|
129
|
+
return new Ctor(buf.buffer, buf.byteOffset + header.dataOffset, count);
|
|
130
|
+
}
|
package/lib/paths.js
CHANGED
|
@@ -10,12 +10,176 @@ import { inputError } from './errors.js';
|
|
|
10
10
|
const __filename = fileURLToPath(import.meta.url);
|
|
11
11
|
const __dirname = dirname(__filename);
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
// ─── Workspace root resolution (SDL-C1) ───────────────────────────
|
|
14
|
+
//
|
|
15
|
+
// `REPO_ROOT` used to be `join(__dirname, '..')` — always the directory
|
|
16
|
+
// this module happens to live in. That is correct for an in-repo checkout,
|
|
17
|
+
// but for an npm-installed consumer it resolves to
|
|
18
|
+
// `node_modules/@mcptoolshop/style-dataset-lab`, so `sdlab init demo`
|
|
19
|
+
// silently wrote the user's whole project tree into node_modules — gone on
|
|
20
|
+
// the next `npm ci`.
|
|
21
|
+
//
|
|
22
|
+
// The fix is a 4-step search, cheapest/most-explicit signal first:
|
|
23
|
+
// 1. SDLAB_ROOT env var, if set and it exists.
|
|
24
|
+
// 2. Walk up from process.cwd() looking for a directory containing
|
|
25
|
+
// `projects/`. First hit wins — this is how a user's own workspace
|
|
26
|
+
// (wherever it lives) gets found.
|
|
27
|
+
// 3. The module's own root, IF it already has a `projects/` dir — this is
|
|
28
|
+
// what preserves every existing in-repo dev/test workflow, where cwd
|
|
29
|
+
// already resolves via step 2 anyway, but is kept as an explicit
|
|
30
|
+
// fallback for callers with an unusual cwd.
|
|
31
|
+
// 4. process.cwd() itself — a fresh workspace where `init` will create
|
|
32
|
+
// `projects/`.
|
|
33
|
+
//
|
|
34
|
+
// Whatever step wins, the result is never allowed to sit inside a
|
|
35
|
+
// node_modules directory — that is the exact shape of the original bug, so
|
|
36
|
+
// it is a hard error instead of a silent write into someone's
|
|
37
|
+
// node_modules tree.
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* True if any path segment of `absPath` is literally `node_modules`.
|
|
41
|
+
*/
|
|
42
|
+
function isInsideNodeModules(absPath) {
|
|
43
|
+
return resolve(absPath)
|
|
44
|
+
.split(/[\\/]+/)
|
|
45
|
+
.some((seg) => seg.toLowerCase() === 'node_modules');
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function assertNotInNodeModules(absPath, label) {
|
|
49
|
+
if (isInsideNodeModules(absPath)) {
|
|
50
|
+
throw inputError(
|
|
51
|
+
'INPUT_ROOT_IN_NODE_MODULES',
|
|
52
|
+
`Resolved workspace root (${label}) sits inside a node_modules directory: ${absPath}`,
|
|
53
|
+
'Run sdlab from your own project directory, or set SDLAB_ROOT to your workspace path.'
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
return absPath;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Walk up from `startDir` looking for a directory that contains a
|
|
61
|
+
* `projects/` subdirectory. Returns the first hit, or null if the walk
|
|
62
|
+
* reaches the filesystem root without finding one.
|
|
63
|
+
*/
|
|
64
|
+
function walkUpForProjectsDir(startDir) {
|
|
65
|
+
let dir = resolve(startDir);
|
|
66
|
+
while (true) {
|
|
67
|
+
if (existsSync(join(dir, 'projects'))) return dir;
|
|
68
|
+
const parent = dirname(dir);
|
|
69
|
+
if (parent === dir) return null; // reached the filesystem root
|
|
70
|
+
dir = parent;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Pure resolution logic for the workspace root, taking every input
|
|
76
|
+
* explicitly (cwd, env, module root) rather than reading global process
|
|
77
|
+
* state directly. This is what makes it possible to unit-test all four
|
|
78
|
+
* branches deterministically without mutating the real process.cwd() or
|
|
79
|
+
* environment (which are shared, global, and unsafe to mutate across a
|
|
80
|
+
* concurrently-run test suite).
|
|
81
|
+
*
|
|
82
|
+
* Production callers should use `getWorkspaceRoot()` instead, which
|
|
83
|
+
* supplies the real process.cwd()/env and memoizes the result.
|
|
84
|
+
*
|
|
85
|
+
* @param {string} cwd
|
|
86
|
+
* @param {{env?: Object, moduleRoot?: string}} [opts]
|
|
87
|
+
* @returns {string} absolute workspace root
|
|
88
|
+
*/
|
|
89
|
+
export function resolveWorkspaceRootFrom(cwd, opts = {}) {
|
|
90
|
+
const env = opts.env || process.env;
|
|
91
|
+
const moduleRoot = opts.moduleRoot || join(__dirname, '..');
|
|
92
|
+
|
|
93
|
+
// 1. SDLAB_ROOT env var, if set and it exists.
|
|
94
|
+
const envRoot = env.SDLAB_ROOT;
|
|
95
|
+
if (envRoot && existsSync(envRoot)) {
|
|
96
|
+
return assertNotInNodeModules(resolve(envRoot), 'from SDLAB_ROOT');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// 2. Walk up from cwd looking for projects/.
|
|
100
|
+
const fromCwd = walkUpForProjectsDir(cwd);
|
|
101
|
+
if (fromCwd) {
|
|
102
|
+
return assertNotInNodeModules(fromCwd, 'found by walking up from the current directory');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// 3. The module root itself is the repo checkout.
|
|
106
|
+
if (existsSync(join(moduleRoot, 'projects'))) {
|
|
107
|
+
return assertNotInNodeModules(resolve(moduleRoot), 'the style-dataset-lab install directory');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// 4. Fresh workspace — init will create projects/ here.
|
|
111
|
+
return assertNotInNodeModules(resolve(cwd), 'the current directory');
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let _workspaceRootCache = null;
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Resolve (and memoize) the workspace root using the real process.cwd()
|
|
118
|
+
* and process.env. Safe to call repeatedly — it only walks the filesystem
|
|
119
|
+
* once per process, unless reset.
|
|
120
|
+
* @returns {string}
|
|
121
|
+
*/
|
|
122
|
+
export function getWorkspaceRoot() {
|
|
123
|
+
if (_workspaceRootCache === null) {
|
|
124
|
+
_workspaceRootCache = resolveWorkspaceRootFrom(process.cwd());
|
|
125
|
+
}
|
|
126
|
+
return _workspaceRootCache;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Clear the memoized workspace root so the next getWorkspaceRoot() call
|
|
131
|
+
* re-resolves from scratch. Exists for tests that need to exercise the
|
|
132
|
+
* resolver under different SDLAB_ROOT/cwd conditions within one process.
|
|
133
|
+
*/
|
|
134
|
+
export function resetWorkspaceRootCache() {
|
|
135
|
+
_workspaceRootCache = null;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Absolute path to the workspace root — the directory that contains (or
|
|
140
|
+
* will contain) `projects/`. Kept as a plain exported constant for
|
|
141
|
+
* backward compatibility with the ~30 existing call sites that do
|
|
142
|
+
* `join(REPO_ROOT, ...)`, but its VALUE now comes from the resolver above
|
|
143
|
+
* rather than being hardcoded to this module's own location.
|
|
144
|
+
*/
|
|
145
|
+
export const REPO_ROOT = getWorkspaceRoot();
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Absolute path to the INSTALLED PACKAGE's own root — always module-relative,
|
|
149
|
+
* never the user's workspace.
|
|
150
|
+
*
|
|
151
|
+
* There are two distinct roots and conflating them is a shipping bug:
|
|
152
|
+
*
|
|
153
|
+
* - **workspace root** (`getWorkspaceRoot()`) — where the user's `projects/`
|
|
154
|
+
* live. Correctly cwd-relative, so an npm-installed user's data lands in
|
|
155
|
+
* their own directory instead of inside `node_modules`.
|
|
156
|
+
* - **package root** (this) — where assets that SHIP IN THE TARBALL live:
|
|
157
|
+
* `templates/` (domain starter packs) and `runtime/` (ComfyUI workflow
|
|
158
|
+
* graphs). These belong to the install, not to the workspace.
|
|
159
|
+
*
|
|
160
|
+
* In a repo checkout the two coincide, which is exactly why the distinction
|
|
161
|
+
* is easy to lose and impossible to catch with in-repo tests. Measured in a
|
|
162
|
+
* clean-room tarball install: with `templates/` resolved against the
|
|
163
|
+
* workspace root, `sdlab init demo --domain character-design` failed with
|
|
164
|
+
* `INPUT_UNKNOWN_DOMAIN: Available: generic,` — the five domain packs were
|
|
165
|
+
* sitting in the package the whole time, being looked for in the user's cwd.
|
|
166
|
+
*/
|
|
167
|
+
export function getPackageRoot() {
|
|
168
|
+
return resolve(join(__dirname, '..'));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ─── Project name / path safety (SDL-C3) ──────────────────────────
|
|
172
|
+
|
|
173
|
+
// A project name must be a single path segment: no separators, no `..`.
|
|
174
|
+
// This is deliberately a blacklist (not the stricter creation-time
|
|
175
|
+
// `/^[a-z0-9][a-z0-9-]*[a-z0-9]$/` from scripts/init.js) so existing
|
|
176
|
+
// project directories keep resolving; it is paired with the
|
|
177
|
+
// resolveSafeProjectPath() containment check below as defense in depth.
|
|
178
|
+
const UNSAFE_PROJECT_NAME_RE = /\.\.|[\\/]/;
|
|
15
179
|
|
|
16
180
|
/**
|
|
17
181
|
* Resolve the root directory for a named project.
|
|
18
|
-
* @param {string} name —
|
|
182
|
+
* @param {string} name — a single path segment, e.g. 'my-project'
|
|
19
183
|
* @returns {string} absolute path to projects/<name>/
|
|
20
184
|
*/
|
|
21
185
|
export function getProjectRoot(name) {
|
|
@@ -26,12 +190,32 @@ export function getProjectRoot(name) {
|
|
|
26
190
|
'Pass --project <name>.'
|
|
27
191
|
);
|
|
28
192
|
}
|
|
29
|
-
|
|
193
|
+
if (UNSAFE_PROJECT_NAME_RE.test(name)) {
|
|
194
|
+
throw inputError(
|
|
195
|
+
'INPUT_UNSAFE_PROJECT_NAME',
|
|
196
|
+
`Project name "${name}" is not valid — it contains a path separator or "..".`,
|
|
197
|
+
`--project takes a name, not a path: it names a directory under projects/. ` +
|
|
198
|
+
`Try --project my-project. Run "sdlab init <name>" to create one.`
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
const projectsDir = join(getWorkspaceRoot(), 'projects');
|
|
202
|
+
// Defense in depth: even though the blacklist above already rejects the
|
|
203
|
+
// characters needed to escape projectsDir, reuse the same containment
|
|
204
|
+
// pattern as every other user-path entry point in this file so a future
|
|
205
|
+
// change to the blacklist can't silently reopen the traversal.
|
|
206
|
+
const root = resolveSafeProjectPath(projectsDir, name, { flagName: 'project' });
|
|
30
207
|
if (!existsSync(root)) {
|
|
31
208
|
throw inputError(
|
|
32
209
|
'INPUT_UNKNOWN_PROJECT',
|
|
33
210
|
`Project "${name}" not found at ${root}.`,
|
|
34
|
-
|
|
211
|
+
// Two different causes land here and they need opposite responses, so
|
|
212
|
+
// name both rather than assuming the project is missing. Since the
|
|
213
|
+
// workspace root is resolved from the current directory, running from
|
|
214
|
+
// the wrong place produces this same error — and blindly following
|
|
215
|
+
// "run init" would then scaffold a duplicate project somewhere else.
|
|
216
|
+
`Looking under ${projectsDir} (workspace resolved from the current directory).\n` +
|
|
217
|
+
` · Wrong directory? cd to your workspace, or set SDLAB_ROOT to it.\n` +
|
|
218
|
+
` · New project? sdlab init ${name} --domain character-design`
|
|
35
219
|
);
|
|
36
220
|
}
|
|
37
221
|
return root;
|
|
@@ -100,8 +284,13 @@ export function getRunsDir(projectRoot) {
|
|
|
100
284
|
}
|
|
101
285
|
|
|
102
286
|
/**
|
|
103
|
-
* Return the runtime templates directory
|
|
287
|
+
* Return the ComfyUI runtime workflow templates directory.
|
|
288
|
+
*
|
|
289
|
+
* PACKAGE-relative, deliberately — see `getPackageRoot()`. `runtime/` ships
|
|
290
|
+
* inside the npm tarball; it is not workspace content. Resolving it against
|
|
291
|
+
* the workspace root made `listWorkflowTemplates()` return `[]` for every
|
|
292
|
+
* installed user, which is the bug `getPackageRoot()` exists to prevent.
|
|
104
293
|
*/
|
|
105
294
|
export function getRuntimeDir() {
|
|
106
|
-
return join(
|
|
295
|
+
return join(getPackageRoot(), 'runtime');
|
|
107
296
|
}
|