@mcptoolshop/style-dataset-lab 1.2.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +138 -113
- package/bin/sdlab.js +126 -0
- package/lib/args.js +106 -0
- package/lib/card.js +145 -0
- package/lib/comfyui.js +109 -0
- package/lib/config.js +248 -0
- package/lib/deprecation.js +14 -0
- package/lib/eligibility.js +110 -0
- package/lib/eval-pack.js +283 -0
- package/lib/export.js +259 -0
- package/lib/paths.js +38 -0
- package/lib/records.js +69 -0
- package/lib/snapshot.js +211 -0
- package/lib/split.js +409 -0
- package/package.json +20 -9
- package/scripts/bulk-curate-wave2-5.js +1 -1
- package/scripts/bulk-curate-waves11-18.js +1 -1
- package/scripts/canon-bind.js +188 -338
- package/scripts/card.js +73 -0
- package/scripts/compare.js +40 -30
- package/scripts/curate-wave25.js +1 -1
- package/scripts/curate.js +34 -28
- package/scripts/doctor.js +324 -0
- package/scripts/eligibility.js +91 -0
- package/scripts/eval-pack.js +92 -0
- package/scripts/export.js +85 -0
- package/scripts/generate-controlnet.js +36 -121
- package/scripts/generate-identity.js +38 -106
- package/scripts/generate-ipadapter.js +42 -119
- package/scripts/generate.js +22 -81
- package/scripts/init.js +229 -0
- package/scripts/migrate-records.js +15 -21
- package/scripts/painterly-test.js +20 -47
- package/scripts/painterly.js +27 -106
- package/scripts/snapshot.js +132 -0
- package/scripts/split.js +154 -0
- package/templates/domains/architecture/constitution.json +97 -0
- package/templates/domains/architecture/lanes.json +41 -0
- package/templates/domains/architecture/project.json +20 -0
- package/templates/domains/architecture/rubric.json +40 -0
- package/templates/domains/architecture/terminology.json +19 -0
- package/templates/domains/character-design/constitution.json +88 -0
- package/templates/domains/character-design/lanes.json +35 -0
- package/templates/domains/character-design/project.json +17 -0
- package/templates/domains/character-design/rubric.json +38 -0
- package/templates/domains/character-design/terminology.json +21 -0
- package/templates/domains/creature-design/constitution.json +87 -0
- package/templates/domains/creature-design/lanes.json +41 -0
- package/templates/domains/creature-design/project.json +18 -0
- package/templates/domains/creature-design/rubric.json +39 -0
- package/templates/domains/creature-design/terminology.json +20 -0
- package/templates/domains/game-art/constitution.json +113 -0
- package/templates/domains/game-art/lanes.json +47 -0
- package/templates/domains/game-art/project.json +17 -0
- package/templates/domains/game-art/rubric.json +36 -0
- package/templates/domains/game-art/terminology.json +30 -0
- package/templates/domains/vehicle-mech/constitution.json +87 -0
- package/templates/domains/vehicle-mech/lanes.json +41 -0
- package/templates/domains/vehicle-mech/project.json +17 -0
- package/templates/domains/vehicle-mech/rubric.json +40 -0
- package/templates/domains/vehicle-mech/terminology.json +21 -0
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,79 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [2.1.0] - 2026-04-16
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **Dataset spine** -- complete snapshot-to-export pipeline with proof at every stage
|
|
10
|
+
- `sdlab snapshot create/list/show/diff` -- frozen, deterministic record selections with config fingerprinting
|
|
11
|
+
- `sdlab eligibility audit` -- training eligibility evaluation with explicit reason traces
|
|
12
|
+
- `sdlab split build/list/show/audit` -- subject-isolated, lane-balanced train/val/test splitting
|
|
13
|
+
- `sdlab card generate` -- dataset card generation (markdown + JSON twin)
|
|
14
|
+
- `sdlab export build/list` -- self-contained export packages with manifest, metadata, images, splits, checksums
|
|
15
|
+
- `sdlab eval-pack build/list/show` -- canon-aware eval packs (4 task types)
|
|
16
|
+
- **New library modules:**
|
|
17
|
+
- `lib/snapshot.js` -- snapshot creation, loading, diffing
|
|
18
|
+
- `lib/eligibility.js` -- eligibility evaluation with reason traces and exclusion categorization
|
|
19
|
+
- `lib/split.js` -- subject isolation (identity/lineage/suffix), mulberry32 PRNG, lane-balanced splitting, leakage audit
|
|
20
|
+
- `lib/card.js` -- dataset card rendering from snapshot + split + config
|
|
21
|
+
- `lib/export.js` -- export package builder with checksums and reproducibility manifest
|
|
22
|
+
- `lib/eval-pack.js` -- four eval task types (lane coverage, forbidden drift, anchor/gold, subject continuity)
|
|
23
|
+
- **Selection, split, and export profiles** in `lib/config.js` with sensible defaults
|
|
24
|
+
- **Detection functions** (`detectLane`, `detectGroup`) extracted from canon-bind into `lib/config.js` for reuse
|
|
25
|
+
|
|
26
|
+
### Key properties
|
|
27
|
+
|
|
28
|
+
- Snapshots are frozen -- once created, never silently changes
|
|
29
|
+
- Inclusion is explainable -- every record has a reason trace
|
|
30
|
+
- Splits preserve canon truth -- no subject family appears in multiple partitions
|
|
31
|
+
- Exports are reproducible -- rebuildable from snapshot ref + split ref + config fingerprint
|
|
32
|
+
|
|
33
|
+
## [2.0.0] - 2026-04-15
|
|
34
|
+
|
|
35
|
+
### Breaking changes
|
|
36
|
+
|
|
37
|
+
- **`games/` renamed to `projects/`** -- all project data now lives under `projects/<name>/`
|
|
38
|
+
- **`--game` renamed to `--project`** -- `--game` still works with a deprecation warning (will be removed in v3.0)
|
|
39
|
+
- **Canon-bind loads from config files** -- rules, lanes, rubric, and terminology are now per-project JSON, not hardcoded
|
|
40
|
+
|
|
41
|
+
### Added
|
|
42
|
+
|
|
43
|
+
- **`sdlab` CLI** -- unified command-line interface (`bin/sdlab.js`)
|
|
44
|
+
- `sdlab init <name> --domain <domain>` -- scaffold projects from domain templates
|
|
45
|
+
- `sdlab project doctor` -- validate project config and structure
|
|
46
|
+
- `sdlab bind`, `sdlab curate`, `sdlab generate`, etc. -- all pipeline commands
|
|
47
|
+
- **Shared library** (`lib/`) -- 6 modules replacing duplicated code across 13 scripts
|
|
48
|
+
- `lib/args.js` -- unified argument parser with `--project` and deprecated `--game` support
|
|
49
|
+
- `lib/paths.js` -- centralized path resolution
|
|
50
|
+
- `lib/comfyui.js` -- ComfyUI HTTP client (extracted from 5 scripts)
|
|
51
|
+
- `lib/config.js` -- per-project JSON config loader
|
|
52
|
+
- `lib/records.js` -- record I/O utilities
|
|
53
|
+
- `lib/deprecation.js` -- one-shot deprecation warnings
|
|
54
|
+
- **Per-project config files** -- 5 JSON files per project:
|
|
55
|
+
- `project.json` -- identity, domain, generation defaults
|
|
56
|
+
- `constitution.json` -- rules with rationale templates
|
|
57
|
+
- `lanes.json` -- subject lanes with regex detection patterns
|
|
58
|
+
- `rubric.json` -- scoring dimensions, thresholds, failure-to-rule mappings
|
|
59
|
+
- `terminology.json` -- group vocabulary with separate `id_detection_order` and `prompt_detection_order`
|
|
60
|
+
- **5 domain starter templates** -- game-art, character-design, creature-design, architecture, vehicle-mech
|
|
61
|
+
- **Training profile placeholders** in domain templates (eligible lanes, thresholds, export format)
|
|
62
|
+
|
|
63
|
+
### Changed
|
|
64
|
+
|
|
65
|
+
- Canon-bind is now fully config-driven -- loads rules, lanes, dimensions, thresholds, group vocabulary, and rationale templates from per-project JSON files
|
|
66
|
+
- Missing config files cause hard failure with actionable error messages (no silent fallback)
|
|
67
|
+
- All 10 CLI-exposed scripts refactored to export `run(argv)` for CLI dispatch
|
|
68
|
+
- Net code reduction: ~305 lines removed from deduplication
|
|
69
|
+
|
|
70
|
+
### Migration
|
|
71
|
+
|
|
72
|
+
1. Rename `games/` to `projects/`
|
|
73
|
+
2. Use `--project` instead of `--game` (or keep `--game` with deprecation warning)
|
|
74
|
+
3. For canon-bind: add 5 JSON config files to your project (use `sdlab init` as reference)
|
|
75
|
+
|
|
76
|
+
Star Freight config files are included as the canonical example.
|
|
77
|
+
|
|
5
78
|
## [1.2.0] - 2026-04-14
|
|
6
79
|
|
|
7
80
|
- Added `templates/` with blank constitution, review rubric, and example prompt pack
|
package/README.md
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
<p align="center">
|
|
2
|
-
<a href="README.ja.md">日本語</a> | <a href="README.zh.md">中文</a> | <a href="README.es.md">Español</a> | <a href="README.fr.md">Français</a> | <a href="README.hi.md">हिन्दी</a> | <a href="README.it.md">Italiano</a> | <a href="README.pt-BR.md">Português (BR)</a>
|
|
3
|
-
</p>
|
|
4
|
-
|
|
5
1
|
<p align="center">
|
|
6
2
|
<img src="https://raw.githubusercontent.com/mcp-tool-shop-org/brand/main/logos/style-dataset-lab/readme.png" width="400">
|
|
7
3
|
</p>
|
|
@@ -13,163 +9,192 @@
|
|
|
13
9
|
|
|
14
10
|
# style-dataset-lab
|
|
15
11
|
|
|
16
|
-
|
|
12
|
+
Turn approved visual work into versioned, review-backed datasets, splits, export packages, and eval packs.
|
|
17
13
|
|
|
18
14
|
## What this is
|
|
19
15
|
|
|
20
|
-
A **pipeline
|
|
21
|
-
|
|
22
|
-
The pipeline is game-agnostic. Each game gets its own data directory under `games/<name>/`; the 13 scripts and blank templates are shared. Every asset produced carries three things:
|
|
16
|
+
A **visual canon and dataset production pipeline**. Define what your project looks like. Curate against constitution rules. Produce reproducible dataset packages with leakage-safe splits. Generate eval packs for future model verification.
|
|
23
17
|
|
|
24
|
-
|
|
25
|
-
2. **Canon explanation** -- why this is on-style or off-style, grounded in a style constitution
|
|
26
|
-
3. **Quality judgment** -- approved/rejected with per-dimension scores and cited rules
|
|
18
|
+
The pipeline produces four artifacts:
|
|
27
19
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
**
|
|
20
|
+
| Artifact | What it is |
|
|
21
|
+
|----------|-----------|
|
|
22
|
+
| **Snapshot** | Frozen, fingerprinted selection of eligible records. Every inclusion has an explicit reason trace. |
|
|
23
|
+
| **Split** | Leakage-safe train/val/test partition. Records sharing a subject family always land in the same split. |
|
|
24
|
+
| **Export package** | Self-contained dataset: manifest, metadata, images, splits, dataset card, and checksums. |
|
|
25
|
+
| **Eval pack** | Canon-aware verification tasks: lane coverage, forbidden drift, anchor/gold, subject continuity. |
|
|
31
26
|
|
|
32
|
-
|
|
27
|
+
Every asset in the pipeline carries three things:
|
|
33
28
|
|
|
34
|
-
|
|
29
|
+
1. **Provenance** -- full generation history (checkpoint, LoRA, seed, sampler, cfg, timing)
|
|
30
|
+
2. **Canon binding** -- which constitution rules this asset passes, fails, or partially meets
|
|
31
|
+
3. **Quality judgment** -- approved/rejected/borderline with per-dimension scores
|
|
35
32
|
|
|
36
|
-
|
|
37
|
-
- **Blank templates** -- starter constitution, review rubric, and example prompt pack in `templates/`
|
|
33
|
+
Works for game art, character design, creature design, architecture, vehicle/mech concepts, and any domain where visual production needs to stay on-model.
|
|
38
34
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
## Install
|
|
35
|
+
## Quick start
|
|
42
36
|
|
|
43
37
|
```bash
|
|
44
|
-
#
|
|
45
|
-
npm install @mcptoolshop/style-dataset-lab
|
|
38
|
+
# Install the CLI + pipeline
|
|
39
|
+
npm install -g @mcptoolshop/style-dataset-lab
|
|
46
40
|
|
|
47
|
-
#
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
41
|
+
# Scaffold a new project
|
|
42
|
+
sdlab init my-project --domain character-design
|
|
43
|
+
|
|
44
|
+
# Validate the project structure
|
|
45
|
+
sdlab project doctor --project my-project
|
|
51
46
|
```
|
|
52
47
|
|
|
53
|
-
|
|
48
|
+
Available domains: `game-art`, `character-design`, `creature-design`, `architecture`, `vehicle-mech`, or `generic`.
|
|
49
|
+
|
|
50
|
+
## CLI
|
|
54
51
|
|
|
55
52
|
```bash
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
53
|
+
sdlab init <name> [--domain <domain>] # Scaffold a new project
|
|
54
|
+
sdlab project doctor [--project <name>] # Validate project config
|
|
55
|
+
|
|
56
|
+
sdlab generate <pack> [--project <name>] # Generate candidates via ComfyUI
|
|
57
|
+
sdlab generate:identity <packet> # Named-subject identity images
|
|
58
|
+
sdlab generate:controlnet # ControlNet-guided generation
|
|
59
|
+
sdlab generate:ipadapter # IP-Adapter reference-guided
|
|
60
|
+
|
|
61
|
+
sdlab curate <id> <status> <explanation> # Record review judgment
|
|
62
|
+
sdlab compare <a> <b> <winner> <reason> # Pairwise A-vs-B comparison
|
|
63
|
+
sdlab bind [--project <name>] # Bind records to constitution rules
|
|
64
|
+
sdlab painterly [--project <name>] # Post-processing style pass
|
|
65
|
+
|
|
66
|
+
sdlab snapshot create [--profile <name>] # Create frozen dataset snapshot
|
|
67
|
+
sdlab snapshot list # List all snapshots
|
|
68
|
+
sdlab snapshot diff <a> <b> # Compare two snapshots
|
|
69
|
+
sdlab eligibility audit # Audit record training eligibility
|
|
70
|
+
sdlab split build [--snapshot <id>] # Build train/val/test split
|
|
71
|
+
sdlab split audit <id> # Audit split for leakage + balance
|
|
72
|
+
sdlab card generate # Generate dataset card (md + JSON)
|
|
73
|
+
sdlab export build [--snapshot <id>] # Build versioned export package
|
|
74
|
+
sdlab eval-pack build # Build canon-aware eval pack
|
|
61
75
|
```
|
|
62
76
|
|
|
63
|
-
|
|
77
|
+
All commands accept `--project <name>` (default: `star-freight`).
|
|
78
|
+
|
|
79
|
+
## Project model
|
|
64
80
|
|
|
65
|
-
|
|
81
|
+
Each project is a self-contained directory under `projects/` with its own canon, config, and data:
|
|
66
82
|
|
|
67
83
|
```
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
84
|
+
projects/
|
|
85
|
+
my-project/
|
|
86
|
+
project.json Project identity + generation defaults
|
|
87
|
+
constitution.json Rules array with rationale templates
|
|
88
|
+
lanes.json Subject lanes with detection patterns
|
|
89
|
+
rubric.json Scoring dimensions + thresholds
|
|
90
|
+
terminology.json Group vocabulary + detection order
|
|
91
|
+
canon/ Style constitution (markdown)
|
|
92
|
+
records/ Per-asset JSON (provenance + judgment + canon)
|
|
93
|
+
inputs/prompts/ Prompt packs (JSON)
|
|
94
|
+
outputs/ Generated images (gitignored)
|
|
95
|
+
comparisons/ A-vs-B preference judgments
|
|
96
|
+
snapshots/ Frozen dataset snapshots
|
|
97
|
+
splits/ Train/val/test partitions
|
|
98
|
+
exports/ Versioned export packages
|
|
99
|
+
eval-packs/ Canon-aware eval instruments
|
|
82
100
|
```
|
|
83
101
|
|
|
84
|
-
## Pipeline
|
|
102
|
+
## Pipeline
|
|
85
103
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
104
|
+
```
|
|
105
|
+
canon → generate → curate → bind → snapshot → split → export → eval
|
|
106
|
+
| | | | | | | |
|
|
107
|
+
rules ComfyUI judgment rules frozen subject package verify
|
|
108
|
+
selection isolation
|
|
109
|
+
```
|
|
91
110
|
|
|
92
|
-
|
|
93
|
-
|
|
111
|
+
1. **Define canon** -- write your style constitution and review rubric
|
|
112
|
+
2. **Generate** -- ComfyUI produces candidates with full provenance
|
|
113
|
+
3. **Curate** -- approve/reject with per-dimension scores and failure modes
|
|
114
|
+
4. **Bind** -- link each asset to constitution rules with pass/fail/partial verdicts
|
|
115
|
+
5. **Snapshot** -- freeze eligible records into a deterministic, fingerprinted selection
|
|
116
|
+
6. **Split** -- partition into train/val/test with subject isolation and lane balance
|
|
117
|
+
7. **Export** -- build a self-contained package with manifest, metadata, images, and checksums
|
|
118
|
+
8. **Eval** -- generate canon-aware test instruments for model verification
|
|
94
119
|
|
|
95
|
-
|
|
96
|
-
npm run generate -- --game star-freight inputs/prompts/wave1.json
|
|
97
|
-
npm run generate -- --game star-freight inputs/prompts/wave1.json --dry-run
|
|
120
|
+
Downstream format conversion (TRL, LLaVA, Parquet, etc.) is handled by [`repo-dataset`](https://github.com/mcp-tool-shop-org/repo-dataset). `sdlab` owns dataset truth; `repo-dataset` renders it into specialized formats.
|
|
98
121
|
|
|
99
|
-
|
|
100
|
-
npm run generate:identity -- --game star-freight inputs/identity-packets/wave27a.json
|
|
122
|
+
## Domain templates
|
|
101
123
|
|
|
102
|
-
|
|
103
|
-
npm run curate -- --game star-freight <asset_id> approved "explanation"
|
|
104
|
-
npm run curate -- --game star-freight <asset_id> rejected "explanation" --failures "too_clean"
|
|
124
|
+
Each domain template ships with lane definitions, constitution rules, scoring rubrics, and terminology structures designed for that production context:
|
|
105
125
|
|
|
106
|
-
|
|
107
|
-
|
|
126
|
+
| Domain | Lanes | Key concerns |
|
|
127
|
+
|--------|-------|-------------|
|
|
128
|
+
| **game-art** | character, environment, prop, UI, ship, interior, equipment | Silhouette at gameplay scale, faction differentiation, wear/aging |
|
|
129
|
+
| **character-design** | portrait, full_body, turnaround, expression_sheet, action_pose | Proportion accuracy, costume logic, personality read, gesture clarity |
|
|
130
|
+
| **creature-design** | concept, orthographic, detail_study, action, scale_reference, habitat | Anatomical plausibility, evolutionary logic, silhouette distinction |
|
|
131
|
+
| **architecture** | exterior, interior, streetscape, structural_detail, ruin, landscape | Structural plausibility, material consistency, perspective, era coherence |
|
|
132
|
+
| **vehicle-mech** | exterior, cockpit, component, schematic, silhouette_sheet, damage_variant | Mechanical logic, functional design language, access points, damage narrative |
|
|
108
133
|
|
|
109
|
-
|
|
110
|
-
npm run canon-bind -- --game star-freight
|
|
134
|
+
## Dataset production
|
|
111
135
|
|
|
112
|
-
|
|
113
|
-
npm run compare -- --game star-freight <asset_a> <asset_b> a "A has better faction read"
|
|
136
|
+
The full dataset spine: snapshot, split, export, eval.
|
|
114
137
|
|
|
115
|
-
# 9. Export training data via repo-dataset
|
|
116
|
-
repo-dataset visual generate ./games/star-freight --format trl
|
|
117
|
-
repo-dataset visual inspect ./games/star-freight
|
|
118
138
|
```
|
|
139
|
+
snapshot --> split --> export --> eval-pack
|
|
140
|
+
| | | |
|
|
141
|
+
frozen subject package canon-aware
|
|
142
|
+
selection isolation (manifest, test instruments
|
|
143
|
+
+ lane metadata, (4 task types)
|
|
144
|
+
balance images,
|
|
145
|
+
checksums,
|
|
146
|
+
card)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
**Snapshots** freeze a deterministic selection of eligible records. Every inclusion has a reason trace. Config fingerprints ensure reproducibility.
|
|
150
|
+
|
|
151
|
+
**Splits** assign records to train/val/test partitions with subject isolation (no subject family appears in multiple splits) and lane-balanced distribution. Seeded PRNG ensures identical results from the same seed.
|
|
152
|
+
|
|
153
|
+
**Export packages** are self-contained: manifest, metadata.jsonl, images (symlinked or copied), splits, dataset card (markdown + JSON), and BSD-format checksums. Everything needed to rebuild the dataset from scratch.
|
|
119
154
|
|
|
120
|
-
|
|
155
|
+
**Eval packs** are canon-aware test instruments with four task types: lane coverage, forbidden drift, anchor/gold, and subject continuity. They prove the dataset spine is feeding future model evaluation, not just dumping files.
|
|
156
|
+
|
|
157
|
+
Export to downstream formats via [`repo-dataset`](https://github.com/mcp-tool-shop-org/repo-dataset) (TRL, LLaVA, Qwen2-VL, JSONL, Parquet, and more). `repo-dataset` handles format conversion; `sdlab` owns dataset truth.
|
|
158
|
+
|
|
159
|
+
## Star Freight example
|
|
160
|
+
|
|
161
|
+
Clone the repo for a complete working example: 1,182 records, 28 prompt waves, 5 factions, 7 lanes, 24 constitution rules, and 892 approved assets from a gritty sci-fi RPG.
|
|
121
162
|
|
|
122
163
|
```bash
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
164
|
+
git clone https://github.com/mcp-tool-shop-org/style-dataset-lab
|
|
165
|
+
cd style-dataset-lab
|
|
166
|
+
|
|
167
|
+
# Validate the project
|
|
168
|
+
sdlab project doctor --project star-freight
|
|
127
169
|
|
|
128
|
-
#
|
|
129
|
-
|
|
170
|
+
# Run the full dataset spine
|
|
171
|
+
sdlab snapshot create --project star-freight # 839 eligible records
|
|
172
|
+
sdlab split build --project star-freight # ~80/10/10, zero leakage
|
|
173
|
+
sdlab export build --project star-freight # package with checksums
|
|
174
|
+
sdlab eval-pack build --project star-freight # 78 eval records
|
|
130
175
|
```
|
|
131
176
|
|
|
132
|
-
##
|
|
177
|
+
## Migrating from v1.x
|
|
133
178
|
|
|
134
|
-
|
|
179
|
+
v2.0 renames `games/` to `projects/` and `--game` to `--project`:
|
|
135
180
|
|
|
136
|
-
```
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
prompts/ Prompt packs per wave (JSON: subjects, variations, defaults)
|
|
140
|
-
references/ IP-Adapter reference images
|
|
141
|
-
control-guides/ ControlNet guide overlays
|
|
142
|
-
identity-packets/ Named character identity spines
|
|
143
|
-
outputs/
|
|
144
|
-
candidates/ Raw generations (gitignored)
|
|
145
|
-
approved/ Curated approved (gitignored)
|
|
146
|
-
rejected/ Curated rejected (gitignored)
|
|
147
|
-
borderline/ Curated borderline (gitignored)
|
|
148
|
-
painterly/ Painterly-style variants (gitignored)
|
|
149
|
-
records/ Per-asset JSON (provenance + judgment + canon binding)
|
|
150
|
-
comparisons/ A-vs-B preference judgments
|
|
151
|
-
exports/ repo-dataset output (gitignored)
|
|
152
|
-
```
|
|
181
|
+
```bash
|
|
182
|
+
# Rename your data directory
|
|
183
|
+
mv games projects
|
|
153
184
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
checkpoint: dreamshaperXL_v21TurboDPMSDE.safetensors
|
|
158
|
-
lora: classipeintxl_v21.safetensors (weight: 1.0)
|
|
159
|
-
resolution: 1024x1024
|
|
160
|
-
steps: 8
|
|
161
|
-
cfg: 2.0
|
|
162
|
-
sampler: dpmpp_sde
|
|
163
|
-
scheduler: karras
|
|
164
|
-
speed: ~9s per image (RTX 5080)
|
|
185
|
+
# --game still works with a deprecation warning (removed in v3.0)
|
|
186
|
+
sdlab bind --game star-freight # works, prints warning
|
|
187
|
+
sdlab bind --project star-freight # canonical form
|
|
165
188
|
```
|
|
166
189
|
|
|
167
|
-
|
|
190
|
+
## Security model
|
|
191
|
+
|
|
192
|
+
**Local-only.** Talks to ComfyUI on `localhost:8188`. No telemetry, no analytics, no external requests. Images stay on your GPU and filesystem.
|
|
168
193
|
|
|
169
194
|
## Requirements
|
|
170
195
|
|
|
171
196
|
- [ComfyUI](https://github.com/comfyanonymous/ComfyUI) running on localhost:8188
|
|
172
|
-
-
|
|
197
|
+
- SDXL-compatible checkpoint + style LoRA
|
|
173
198
|
- Node.js 20+
|
|
174
199
|
- [`@mcptoolshop/repo-dataset`](https://github.com/mcp-tool-shop-org/repo-dataset) for training export
|
|
175
200
|
|
package/bin/sdlab.js
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* sdlab — CLI entry point for Style Dataset Lab.
|
|
5
|
+
*
|
|
6
|
+
* Thin dispatcher that routes subcommands to existing scripts.
|
|
7
|
+
* Each script exports run(argv) and is lazy-loaded on demand.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const COMMANDS = {
|
|
11
|
+
'init': '../scripts/init.js',
|
|
12
|
+
'generate': '../scripts/generate.js',
|
|
13
|
+
'generate:identity': '../scripts/generate-identity.js',
|
|
14
|
+
'generate:controlnet': '../scripts/generate-controlnet.js',
|
|
15
|
+
'generate:ipadapter': '../scripts/generate-ipadapter.js',
|
|
16
|
+
'curate': '../scripts/curate.js',
|
|
17
|
+
'compare': '../scripts/compare.js',
|
|
18
|
+
'bind': '../scripts/canon-bind.js',
|
|
19
|
+
'canon-bind': '../scripts/canon-bind.js',
|
|
20
|
+
'painterly': '../scripts/painterly.js',
|
|
21
|
+
'painterly:test': '../scripts/painterly-test.js',
|
|
22
|
+
'migrate': '../scripts/migrate-records.js',
|
|
23
|
+
'snapshot': '../scripts/snapshot.js',
|
|
24
|
+
'eligibility': '../scripts/eligibility.js',
|
|
25
|
+
'split': '../scripts/split.js',
|
|
26
|
+
'card': '../scripts/card.js',
|
|
27
|
+
'export': '../scripts/export.js',
|
|
28
|
+
'eval-pack': '../scripts/eval-pack.js',
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
// Two-word commands under "project" namespace
|
|
32
|
+
const PROJECT_COMMANDS = {
|
|
33
|
+
'doctor': '../scripts/doctor.js',
|
|
34
|
+
'migrate': '../scripts/migrate-records.js',
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
function printHelp() {
|
|
38
|
+
console.log(`\x1b[1msdlab\x1b[0m — Style Dataset Lab CLI\n`);
|
|
39
|
+
console.log('Usage: sdlab <command> [options]\n');
|
|
40
|
+
console.log('Project:');
|
|
41
|
+
console.log(' init <name> Scaffold a new project from a domain template');
|
|
42
|
+
console.log(' project doctor Validate project config and structure');
|
|
43
|
+
console.log(' project migrate Migrate record schemas');
|
|
44
|
+
console.log('');
|
|
45
|
+
console.log('Pipeline:');
|
|
46
|
+
console.log(' generate Generate candidates from a prompt pack');
|
|
47
|
+
console.log(' generate:identity Generate named-subject identity images');
|
|
48
|
+
console.log(' generate:controlnet ControlNet-guided structural generation');
|
|
49
|
+
console.log(' generate:ipadapter IP-Adapter reference-guided generation');
|
|
50
|
+
console.log(' curate Move candidate to approved/rejected/borderline');
|
|
51
|
+
console.log(' compare Record pairwise A-vs-B comparison');
|
|
52
|
+
console.log(' bind Bind approved records to constitution rules');
|
|
53
|
+
console.log(' painterly Post-processing painterly style pass');
|
|
54
|
+
console.log(' painterly:test Test denoise levels on reference images');
|
|
55
|
+
console.log('');
|
|
56
|
+
console.log('Dataset:');
|
|
57
|
+
console.log(' snapshot create Create a frozen dataset snapshot');
|
|
58
|
+
console.log(' snapshot list List all snapshots');
|
|
59
|
+
console.log(' snapshot show <id> Show snapshot details');
|
|
60
|
+
console.log(' snapshot diff <a> <b> Compare two snapshots');
|
|
61
|
+
console.log(' eligibility audit Audit record training eligibility');
|
|
62
|
+
console.log(' split build Build train/val/test split from snapshot');
|
|
63
|
+
console.log(' split list List all splits');
|
|
64
|
+
console.log(' split show <id> Show split details');
|
|
65
|
+
console.log(' split audit <id> Audit split for leakage and balance');
|
|
66
|
+
console.log(' card generate Generate dataset card (markdown + JSON)');
|
|
67
|
+
console.log(' export build Build self-contained export package');
|
|
68
|
+
console.log(' export list List all export packages');
|
|
69
|
+
console.log(' eval-pack build Build canon-aware eval pack');
|
|
70
|
+
console.log(' eval-pack list List eval packs');
|
|
71
|
+
console.log(' eval-pack show <id> Show eval pack details');
|
|
72
|
+
console.log('');
|
|
73
|
+
console.log('Options:');
|
|
74
|
+
console.log(' --project <name> Project to operate on (default: star-freight)');
|
|
75
|
+
console.log(' --game <name> Deprecated alias for --project');
|
|
76
|
+
console.log(' --help Show this help');
|
|
77
|
+
console.log('');
|
|
78
|
+
console.log('Examples:');
|
|
79
|
+
console.log(' sdlab init my-project --domain character-design');
|
|
80
|
+
console.log(' sdlab generate inputs/prompts/wave1.json --project star-freight');
|
|
81
|
+
console.log(' sdlab bind --stats --project star-freight');
|
|
82
|
+
console.log(' sdlab project doctor --project star-freight');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function main() {
|
|
86
|
+
const args = process.argv.slice(2);
|
|
87
|
+
|
|
88
|
+
if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
|
|
89
|
+
printHelp();
|
|
90
|
+
process.exit(0);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let command = args[0];
|
|
94
|
+
let commandArgs = args.slice(1);
|
|
95
|
+
|
|
96
|
+
// Handle "project <subcommand>" two-word form
|
|
97
|
+
if (command === 'project' && args[1] && PROJECT_COMMANDS[args[1]]) {
|
|
98
|
+
commandArgs = args.slice(2);
|
|
99
|
+
const modulePath = PROJECT_COMMANDS[args[1]];
|
|
100
|
+
try {
|
|
101
|
+
const mod = await import(modulePath);
|
|
102
|
+
await mod.run(commandArgs);
|
|
103
|
+
} catch (err) {
|
|
104
|
+
console.error(`\x1b[31mError:\x1b[0m ${err.message}`);
|
|
105
|
+
process.exit(1);
|
|
106
|
+
}
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const modulePath = COMMANDS[command];
|
|
111
|
+
if (!modulePath) {
|
|
112
|
+
console.error(`Unknown command: ${command}`);
|
|
113
|
+
console.error(`Run "sdlab --help" for available commands.`);
|
|
114
|
+
process.exit(1);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
try {
|
|
118
|
+
const mod = await import(modulePath);
|
|
119
|
+
await mod.run(commandArgs);
|
|
120
|
+
} catch (err) {
|
|
121
|
+
console.error(`\x1b[31mError:\x1b[0m ${err.message}`);
|
|
122
|
+
process.exit(1);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
main();
|
package/lib/args.js
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified CLI argument parser.
|
|
3
|
+
*
|
|
4
|
+
* Supports:
|
|
5
|
+
* --flag value (space-separated)
|
|
6
|
+
* --flag=value (equals form)
|
|
7
|
+
* --boolean-flag (boolean)
|
|
8
|
+
* positional args (collected in order)
|
|
9
|
+
* deprecated aliases (warns once, maps to canonical name)
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { warnDeprecated } from './deprecation.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* @typedef {Object} ArgSpec
|
|
16
|
+
* @property {Object<string, {type: 'string'|'boolean', default?: any, alias?: string}>} flags
|
|
17
|
+
* @property {string[]} [positionals] - names for positional args (in order)
|
|
18
|
+
* @property {Object<string, string>} [deprecated] - old flag name → new flag name
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Parse CLI arguments.
|
|
23
|
+
* @param {string[]} argv — typically process.argv.slice(2)
|
|
24
|
+
* @param {ArgSpec} spec
|
|
25
|
+
* @returns {{ flags: Object, positionals: string[], rest: string[] }}
|
|
26
|
+
*/
|
|
27
|
+
export function parseArgs(argv, spec = {}) {
|
|
28
|
+
const flagDefs = spec.flags || {};
|
|
29
|
+
const deprecated = spec.deprecated || {};
|
|
30
|
+
const positionalNames = spec.positionals || [];
|
|
31
|
+
|
|
32
|
+
const flags = {};
|
|
33
|
+
const positionals = [];
|
|
34
|
+
|
|
35
|
+
// Set defaults
|
|
36
|
+
for (const [name, def] of Object.entries(flagDefs)) {
|
|
37
|
+
if (def.default !== undefined) flags[name] = def.default;
|
|
38
|
+
else if (def.type === 'boolean') flags[name] = false;
|
|
39
|
+
else flags[name] = undefined;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
let i = 0;
|
|
43
|
+
while (i < argv.length) {
|
|
44
|
+
const arg = argv[i];
|
|
45
|
+
|
|
46
|
+
if (arg.startsWith('--')) {
|
|
47
|
+
let key, value;
|
|
48
|
+
|
|
49
|
+
if (arg.includes('=')) {
|
|
50
|
+
const eqIdx = arg.indexOf('=');
|
|
51
|
+
key = arg.slice(2, eqIdx);
|
|
52
|
+
value = arg.slice(eqIdx + 1);
|
|
53
|
+
} else {
|
|
54
|
+
key = arg.slice(2);
|
|
55
|
+
value = undefined;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Handle deprecated aliases
|
|
59
|
+
if (deprecated[key]) {
|
|
60
|
+
warnDeprecated(key, deprecated[key]);
|
|
61
|
+
key = deprecated[key];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Handle short aliases defined in flagDefs
|
|
65
|
+
const canonical = Object.entries(flagDefs).find(([, def]) => def.alias === key);
|
|
66
|
+
if (canonical) key = canonical[0];
|
|
67
|
+
|
|
68
|
+
// Normalize kebab-case to camelCase for lookup
|
|
69
|
+
const camelKey = key.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
|
|
70
|
+
const def = flagDefs[key] || flagDefs[camelKey];
|
|
71
|
+
|
|
72
|
+
if (def?.type === 'boolean') {
|
|
73
|
+
flags[key in flagDefs ? key : camelKey] = true;
|
|
74
|
+
} else if (value !== undefined) {
|
|
75
|
+
flags[key in flagDefs ? key : camelKey] = value;
|
|
76
|
+
} else if (i + 1 < argv.length && !argv[i + 1].startsWith('--')) {
|
|
77
|
+
flags[key in flagDefs ? key : camelKey] = argv[++i];
|
|
78
|
+
} else {
|
|
79
|
+
flags[key in flagDefs ? key : camelKey] = true;
|
|
80
|
+
}
|
|
81
|
+
} else {
|
|
82
|
+
positionals.push(arg);
|
|
83
|
+
}
|
|
84
|
+
i++;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Map positionals to named keys
|
|
88
|
+
const named = {};
|
|
89
|
+
for (let j = 0; j < positionalNames.length && j < positionals.length; j++) {
|
|
90
|
+
named[positionalNames[j]] = positionals[j];
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return { flags, positionals, named, rest: positionals.slice(positionalNames.length) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Extract the --project (or deprecated --game) value from argv.
|
|
98
|
+
* Convenience for scripts that just need the project name.
|
|
99
|
+
*/
|
|
100
|
+
export function getProjectName(argv) {
|
|
101
|
+
const { flags } = parseArgs(argv, {
|
|
102
|
+
flags: { project: { type: 'string', default: 'star-freight' } },
|
|
103
|
+
deprecated: { game: 'project' },
|
|
104
|
+
});
|
|
105
|
+
return flags.project;
|
|
106
|
+
}
|