@mcptoolshop/style-dataset-lab 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +70 -24
- package/bin/sdlab.js +23 -0
- package/lib/card.js +145 -0
- package/lib/config.js +144 -0
- package/lib/eligibility.js +110 -0
- package/lib/eval-pack.js +283 -0
- package/lib/export.js +259 -0
- package/lib/snapshot.js +211 -0
- package/lib/split.js +409 -0
- package/package.json +12 -10
- package/scripts/canon-bind.js +1 -87
- package/scripts/card.js +73 -0
- package/scripts/doctor.js +3 -0
- package/scripts/eligibility.js +91 -0
- package/scripts/eval-pack.js +92 -0
- package/scripts/export.js +85 -0
- package/scripts/init.js +3 -0
- package/scripts/snapshot.js +132 -0
- package/scripts/split.js +154 -0
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [2.1.0] - 2026-04-16
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **Dataset spine** -- complete snapshot-to-export pipeline with proof at every stage
|
|
10
|
+
- `sdlab snapshot create/list/show/diff` -- frozen, deterministic record selections with config fingerprinting
|
|
11
|
+
- `sdlab eligibility audit` -- training eligibility evaluation with explicit reason traces
|
|
12
|
+
- `sdlab split build/list/show/audit` -- subject-isolated, lane-balanced train/val/test splitting
|
|
13
|
+
- `sdlab card generate` -- dataset card generation (markdown + JSON twin)
|
|
14
|
+
- `sdlab export build/list` -- self-contained export packages with manifest, metadata, images, splits, checksums
|
|
15
|
+
- `sdlab eval-pack build/list/show` -- canon-aware eval packs (4 task types)
|
|
16
|
+
- **New library modules:**
|
|
17
|
+
- `lib/snapshot.js` -- snapshot creation, loading, diffing
|
|
18
|
+
- `lib/eligibility.js` -- eligibility evaluation with reason traces and exclusion categorization
|
|
19
|
+
- `lib/split.js` -- subject isolation (identity/lineage/suffix), mulberry32 PRNG, lane-balanced splitting, leakage audit
|
|
20
|
+
- `lib/card.js` -- dataset card rendering from snapshot + split + config
|
|
21
|
+
- `lib/export.js` -- export package builder with checksums and reproducibility manifest
|
|
22
|
+
- `lib/eval-pack.js` -- four eval task types (lane coverage, forbidden drift, anchor/gold, subject continuity)
|
|
23
|
+
- **Selection, split, and export profiles** in `lib/config.js` with sensible defaults
|
|
24
|
+
- **Detection functions** (`detectLane`, `detectGroup`) extracted from canon-bind into `lib/config.js` for reuse
|
|
25
|
+
|
|
26
|
+
### Key properties
|
|
27
|
+
|
|
28
|
+
- Snapshots are frozen -- once created, never silently changes
|
|
29
|
+
- Inclusion is explainable -- every record has a reason trace
|
|
30
|
+
- Splits preserve canon truth -- no subject family appears in multiple partitions
|
|
31
|
+
- Exports are reproducible -- rebuildable from snapshot ref + split ref + config fingerprint
|
|
32
|
+
|
|
5
33
|
## [2.0.0] - 2026-04-15
|
|
6
34
|
|
|
7
35
|
### Breaking changes
|
package/README.md
CHANGED
|
@@ -9,19 +9,28 @@
|
|
|
9
9
|
|
|
10
10
|
# style-dataset-lab
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
Define what your project looks like. Generate candidates. Curate against your canon. Bind approved work to constitution rules. Export trustworthy datasets for training, evaluation, and production reuse.
|
|
12
|
+
Turn approved visual work into versioned, review-backed datasets, splits, export packages, and eval packs.
|
|
15
13
|
|
|
16
14
|
## What this is
|
|
17
15
|
|
|
18
|
-
A **visual canon
|
|
16
|
+
A **visual canon and dataset production pipeline**. Define what your project looks like. Curate against constitution rules. Produce reproducible dataset packages with leakage-safe splits. Generate eval packs for future model verification.
|
|
17
|
+
|
|
18
|
+
The pipeline produces four artifacts:
|
|
19
|
+
|
|
20
|
+
| Artifact | What it is |
|
|
21
|
+
|----------|-----------|
|
|
22
|
+
| **Snapshot** | Frozen, fingerprinted selection of eligible records. Every inclusion has an explicit reason trace. |
|
|
23
|
+
| **Split** | Leakage-safe train/val/test partition. Records sharing a subject family always land in the same split. |
|
|
24
|
+
| **Export package** | Self-contained dataset: manifest, metadata, images, splits, dataset card, and checksums. |
|
|
25
|
+
| **Eval pack** | Canon-aware verification tasks: lane coverage, forbidden drift, anchor/gold, subject continuity. |
|
|
26
|
+
|
|
27
|
+
Every asset in the pipeline carries three things:
|
|
19
28
|
|
|
20
29
|
1. **Provenance** -- full generation history (checkpoint, LoRA, seed, sampler, cfg, timing)
|
|
21
30
|
2. **Canon binding** -- which constitution rules this asset passes, fails, or partially meets
|
|
22
31
|
3. **Quality judgment** -- approved/rejected/borderline with per-dimension scores
|
|
23
32
|
|
|
24
|
-
|
|
33
|
+
Works for game art, character design, creature design, architecture, vehicle/mech concepts, and any domain where visual production needs to stay on-model.
|
|
25
34
|
|
|
26
35
|
## Quick start
|
|
27
36
|
|
|
@@ -53,6 +62,16 @@ sdlab curate <id> <status> <explanation> # Record review judgment
|
|
|
53
62
|
sdlab compare <a> <b> <winner> <reason> # Pairwise A-vs-B comparison
|
|
54
63
|
sdlab bind [--project <name>] # Bind records to constitution rules
|
|
55
64
|
sdlab painterly [--project <name>] # Post-processing style pass
|
|
65
|
+
|
|
66
|
+
sdlab snapshot create [--profile <name>] # Create frozen dataset snapshot
|
|
67
|
+
sdlab snapshot list # List all snapshots
|
|
68
|
+
sdlab snapshot diff <a> <b> # Compare two snapshots
|
|
69
|
+
sdlab eligibility audit # Audit record training eligibility
|
|
70
|
+
sdlab split build [--snapshot <id>] # Build train/val/test split
|
|
71
|
+
sdlab split audit <id> # Audit split for leakage + balance
|
|
72
|
+
sdlab card generate # Generate dataset card (md + JSON)
|
|
73
|
+
sdlab export build [--snapshot <id>] # Build versioned export package
|
|
74
|
+
sdlab eval-pack build # Build canon-aware eval pack
|
|
56
75
|
```
|
|
57
76
|
|
|
58
77
|
All commands accept `--project <name>` (default: `star-freight`).
|
|
@@ -74,24 +93,31 @@ projects/
|
|
|
74
93
|
inputs/prompts/ Prompt packs (JSON)
|
|
75
94
|
outputs/ Generated images (gitignored)
|
|
76
95
|
comparisons/ A-vs-B preference judgments
|
|
77
|
-
|
|
96
|
+
snapshots/ Frozen dataset snapshots
|
|
97
|
+
splits/ Train/val/test partitions
|
|
98
|
+
exports/ Versioned export packages
|
|
99
|
+
eval-packs/ Canon-aware eval instruments
|
|
78
100
|
```
|
|
79
101
|
|
|
80
102
|
## Pipeline
|
|
81
103
|
|
|
82
104
|
```
|
|
83
|
-
canon
|
|
84
|
-
|
|
85
|
-
|
|
105
|
+
canon → generate → curate → bind → snapshot → split → export → eval
|
|
106
|
+
| | | | | | | |
|
|
107
|
+
rules ComfyUI judgment rules frozen subject package verify
|
|
108
|
+
selection isolation
|
|
86
109
|
```
|
|
87
110
|
|
|
88
111
|
1. **Define canon** -- write your style constitution and review rubric
|
|
89
|
-
2. **
|
|
90
|
-
3. **
|
|
91
|
-
4. **
|
|
92
|
-
5. **
|
|
93
|
-
6. **
|
|
94
|
-
7. **Export** --
|
|
112
|
+
2. **Generate** -- ComfyUI produces candidates with full provenance
|
|
113
|
+
3. **Curate** -- approve/reject with per-dimension scores and failure modes
|
|
114
|
+
4. **Bind** -- link each asset to constitution rules with pass/fail/partial verdicts
|
|
115
|
+
5. **Snapshot** -- freeze eligible records into a deterministic, fingerprinted selection
|
|
116
|
+
6. **Split** -- partition into train/val/test with subject isolation and lane balance
|
|
117
|
+
7. **Export** -- build a self-contained package with manifest, metadata, images, and checksums
|
|
118
|
+
8. **Eval** -- generate canon-aware test instruments for model verification
|
|
119
|
+
|
|
120
|
+
Downstream format conversion (TRL, LLaVA, Parquet, etc.) is handled by [`repo-dataset`](https://github.com/mcp-tool-shop-org/repo-dataset). `sdlab` owns dataset truth; `repo-dataset` renders it into specialized formats.
|
|
95
121
|
|
|
96
122
|
## Domain templates
|
|
97
123
|
|
|
@@ -105,17 +131,30 @@ Each domain template ships with lane definitions, constitution rules, scoring ru
|
|
|
105
131
|
| **architecture** | exterior, interior, streetscape, structural_detail, ruin, landscape | Structural plausibility, material consistency, perspective, era coherence |
|
|
106
132
|
| **vehicle-mech** | exterior, cockpit, component, schematic, silhouette_sheet, damage_variant | Mechanical logic, functional design language, access points, damage narrative |
|
|
107
133
|
|
|
108
|
-
##
|
|
134
|
+
## Dataset production
|
|
135
|
+
|
|
136
|
+
The full dataset spine: snapshot, split, export, eval.
|
|
109
137
|
|
|
110
|
-
|
|
138
|
+
```
|
|
139
|
+
snapshot --> split --> export --> eval-pack
|
|
140
|
+
| | | |
|
|
141
|
+
frozen subject package canon-aware
|
|
142
|
+
selection isolation (manifest, test instruments
|
|
143
|
+
+ lane metadata, (4 task types)
|
|
144
|
+
balance images,
|
|
145
|
+
checksums,
|
|
146
|
+
card)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
**Snapshots** freeze a deterministic selection of eligible records. Every inclusion has a reason trace. Config fingerprints ensure reproducibility.
|
|
150
|
+
|
|
151
|
+
**Splits** assign records to train/val/test partitions with subject isolation (no subject family appears in multiple splits) and lane-balanced distribution. Seeded PRNG ensures identical results from the same seed.
|
|
111
152
|
|
|
112
|
-
|
|
113
|
-
- **Canon-bound judgments** (not just labels -- reasoned verdicts against declared rules)
|
|
114
|
-
- **Pairwise preferences** (for DPO and ranking-based training)
|
|
115
|
-
- **Per-dimension scoring** (fine-grained quality signals, not just pass/fail)
|
|
116
|
-
- **Failure mode tagging** (what specifically went wrong and which rules were violated)
|
|
153
|
+
**Export packages** are self-contained: manifest, metadata.jsonl, images (symlinked or copied), splits, dataset card (markdown + JSON), and BSD-format checksums. Everything needed to rebuild the dataset from scratch.
|
|
117
154
|
|
|
118
|
-
|
|
155
|
+
**Eval packs** are canon-aware test instruments with four task types: lane coverage, forbidden drift, anchor/gold, and subject continuity. They prove the dataset spine is feeding future model evaluation, not just dumping files.
|
|
156
|
+
|
|
157
|
+
Export to downstream formats via [`repo-dataset`](https://github.com/mcp-tool-shop-org/repo-dataset) (TRL, LLaVA, Qwen2-VL, JSONL, Parquet, and more). `repo-dataset` handles format conversion; `sdlab` owns dataset truth.
|
|
119
158
|
|
|
120
159
|
## Star Freight example
|
|
121
160
|
|
|
@@ -124,8 +163,15 @@ Clone the repo for a complete working example: 1,182 records, 28 prompt waves, 5
|
|
|
124
163
|
```bash
|
|
125
164
|
git clone https://github.com/mcp-tool-shop-org/style-dataset-lab
|
|
126
165
|
cd style-dataset-lab
|
|
127
|
-
|
|
166
|
+
|
|
167
|
+
# Validate the project
|
|
128
168
|
sdlab project doctor --project star-freight
|
|
169
|
+
|
|
170
|
+
# Run the full dataset spine
|
|
171
|
+
sdlab snapshot create --project star-freight # 839 eligible records
|
|
172
|
+
sdlab split build --project star-freight # ~80/10/10, zero leakage
|
|
173
|
+
sdlab export build --project star-freight # package with checksums
|
|
174
|
+
sdlab eval-pack build --project star-freight # 78 eval records
|
|
129
175
|
```
|
|
130
176
|
|
|
131
177
|
## Migrating from v1.x
|
package/bin/sdlab.js
CHANGED
|
@@ -20,6 +20,12 @@ const COMMANDS = {
|
|
|
20
20
|
'painterly': '../scripts/painterly.js',
|
|
21
21
|
'painterly:test': '../scripts/painterly-test.js',
|
|
22
22
|
'migrate': '../scripts/migrate-records.js',
|
|
23
|
+
'snapshot': '../scripts/snapshot.js',
|
|
24
|
+
'eligibility': '../scripts/eligibility.js',
|
|
25
|
+
'split': '../scripts/split.js',
|
|
26
|
+
'card': '../scripts/card.js',
|
|
27
|
+
'export': '../scripts/export.js',
|
|
28
|
+
'eval-pack': '../scripts/eval-pack.js',
|
|
23
29
|
};
|
|
24
30
|
|
|
25
31
|
// Two-word commands under "project" namespace
|
|
@@ -47,6 +53,23 @@ function printHelp() {
|
|
|
47
53
|
console.log(' painterly Post-processing painterly style pass');
|
|
48
54
|
console.log(' painterly:test Test denoise levels on reference images');
|
|
49
55
|
console.log('');
|
|
56
|
+
console.log('Dataset:');
|
|
57
|
+
console.log(' snapshot create Create a frozen dataset snapshot');
|
|
58
|
+
console.log(' snapshot list List all snapshots');
|
|
59
|
+
console.log(' snapshot show <id> Show snapshot details');
|
|
60
|
+
console.log(' snapshot diff <a> <b> Compare two snapshots');
|
|
61
|
+
console.log(' eligibility audit Audit record training eligibility');
|
|
62
|
+
console.log(' split build Build train/val/test split from snapshot');
|
|
63
|
+
console.log(' split list List all splits');
|
|
64
|
+
console.log(' split show <id> Show split details');
|
|
65
|
+
console.log(' split audit <id> Audit split for leakage and balance');
|
|
66
|
+
console.log(' card generate Generate dataset card (markdown + JSON)');
|
|
67
|
+
console.log(' export build Build self-contained export package');
|
|
68
|
+
console.log(' export list List all export packages');
|
|
69
|
+
console.log(' eval-pack build Build canon-aware eval pack');
|
|
70
|
+
console.log(' eval-pack list List eval packs');
|
|
71
|
+
console.log(' eval-pack show <id> Show eval pack details');
|
|
72
|
+
console.log('');
|
|
50
73
|
console.log('Options:');
|
|
51
74
|
console.log(' --project <name> Project to operate on (default: star-freight)');
|
|
52
75
|
console.log(' --game <name> Deprecated alias for --project');
|
package/lib/card.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dataset card generation — markdown + JSON twin.
|
|
3
|
+
*
|
|
4
|
+
* Produces a human-readable dataset card summarizing provenance,
|
|
5
|
+
* selection criteria, split strategy, and distribution statistics.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { loadSnapshot, loadSnapshotIncluded } from './snapshot.js';
|
|
9
|
+
import { loadSplit, loadSplitAudit } from './split.js';
|
|
10
|
+
import { loadProjectConfig } from './config.js';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Generate a dataset card from snapshot + split + project config.
|
|
14
|
+
*
|
|
15
|
+
* @param {string} projectRoot
|
|
16
|
+
* @param {string} snapshotId
|
|
17
|
+
* @param {string} splitId
|
|
18
|
+
* @returns {Promise<{markdown: string, json: Object}>}
|
|
19
|
+
*/
|
|
20
|
+
export async function generateCard(projectRoot, snapshotId, splitId) {
|
|
21
|
+
const config = loadProjectConfig(projectRoot);
|
|
22
|
+
const snapshot = await loadSnapshot(projectRoot, snapshotId);
|
|
23
|
+
const split = await loadSplit(projectRoot, splitId);
|
|
24
|
+
const audit = await loadSplitAudit(projectRoot, splitId);
|
|
25
|
+
const included = await loadSnapshotIncluded(projectRoot, snapshotId);
|
|
26
|
+
|
|
27
|
+
const json = {
|
|
28
|
+
dataset_name: `${config.meta.name}-visual`,
|
|
29
|
+
version: config.meta.version || '1.0.0',
|
|
30
|
+
created_at: new Date().toISOString(),
|
|
31
|
+
created_by: 'sdlab-card-v1',
|
|
32
|
+
project: {
|
|
33
|
+
name: config.meta.name,
|
|
34
|
+
display_name: config.meta.display_name,
|
|
35
|
+
domain: config.meta.domain,
|
|
36
|
+
},
|
|
37
|
+
snapshot: {
|
|
38
|
+
id: snapshot.snapshot_id,
|
|
39
|
+
created_at: snapshot.created_at,
|
|
40
|
+
config_fingerprint: snapshot.config_fingerprint,
|
|
41
|
+
total_records: snapshot.counts.total_records,
|
|
42
|
+
included: snapshot.counts.included,
|
|
43
|
+
excluded: snapshot.counts.excluded,
|
|
44
|
+
},
|
|
45
|
+
selection: snapshot.selection_profile,
|
|
46
|
+
split: {
|
|
47
|
+
id: split.split_id,
|
|
48
|
+
strategy: split.profile.strategy,
|
|
49
|
+
seed: split.profile.seed,
|
|
50
|
+
ratios: {
|
|
51
|
+
train: split.profile.train_ratio,
|
|
52
|
+
val: split.profile.val_ratio,
|
|
53
|
+
test: split.profile.test_ratio,
|
|
54
|
+
},
|
|
55
|
+
counts: split.counts,
|
|
56
|
+
leakage_free: audit.leakage_check.passed,
|
|
57
|
+
},
|
|
58
|
+
lane_balance: audit.lane_balance,
|
|
59
|
+
constitution: {
|
|
60
|
+
rule_count: config.constitution.rules?.length || 0,
|
|
61
|
+
},
|
|
62
|
+
rubric: {
|
|
63
|
+
dimension_count: config.rubric.dimensions?.length || 0,
|
|
64
|
+
thresholds: config.rubric.thresholds,
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const markdown = renderMarkdown(json, config);
|
|
69
|
+
|
|
70
|
+
return { markdown, json };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function renderMarkdown(card, config) {
|
|
74
|
+
const lines = [];
|
|
75
|
+
const add = (s = '') => lines.push(s);
|
|
76
|
+
|
|
77
|
+
add(`# ${card.dataset_name}`);
|
|
78
|
+
add();
|
|
79
|
+
add(`**Project:** ${card.project.display_name} `);
|
|
80
|
+
add(`**Domain:** ${card.project.domain} `);
|
|
81
|
+
add(`**Version:** ${card.version} `);
|
|
82
|
+
add(`**Generated:** ${card.created_at} `);
|
|
83
|
+
add();
|
|
84
|
+
|
|
85
|
+
add('## Selection');
|
|
86
|
+
add();
|
|
87
|
+
add(`- **Snapshot:** \`${card.snapshot.id}\``);
|
|
88
|
+
add(`- **Config fingerprint:** \`${card.snapshot.config_fingerprint.slice(0, 16)}...\``);
|
|
89
|
+
add(`- **Total records evaluated:** ${card.snapshot.total_records}`);
|
|
90
|
+
add(`- **Included:** ${card.snapshot.included} (${(card.snapshot.included / card.snapshot.total_records * 100).toFixed(1)}%)`);
|
|
91
|
+
add(`- **Excluded:** ${card.snapshot.excluded}`);
|
|
92
|
+
add();
|
|
93
|
+
|
|
94
|
+
add('### Selection criteria');
|
|
95
|
+
add();
|
|
96
|
+
const sel = card.selection;
|
|
97
|
+
if (sel.require_judgment) add('- Requires human judgment');
|
|
98
|
+
if (sel.require_status?.length) add(`- Status: ${sel.require_status.join(', ')}`);
|
|
99
|
+
if (sel.require_canon_bound) add('- Must be canon-bound (constitution assertions)');
|
|
100
|
+
if (sel.minimum_pass_ratio != null) add(`- Minimum pass ratio: ${(sel.minimum_pass_ratio * 100).toFixed(0)}%`);
|
|
101
|
+
add();
|
|
102
|
+
|
|
103
|
+
add('## Split');
|
|
104
|
+
add();
|
|
105
|
+
add(`- **Split ID:** \`${card.split.id}\``);
|
|
106
|
+
add(`- **Strategy:** ${card.split.strategy}`);
|
|
107
|
+
add(`- **Seed:** ${card.split.seed}`);
|
|
108
|
+
add(`- **Subject leakage:** ${card.split.leakage_free ? 'None (verified)' : 'DETECTED'}`);
|
|
109
|
+
add();
|
|
110
|
+
|
|
111
|
+
add('| Partition | Records | % |');
|
|
112
|
+
add('|-----------|---------|---|');
|
|
113
|
+
const total = card.split.counts.total_records;
|
|
114
|
+
add(`| Train | ${card.split.counts.train} | ${(card.split.counts.train / total * 100).toFixed(1)}% |`);
|
|
115
|
+
add(`| Val | ${card.split.counts.val} | ${(card.split.counts.val / total * 100).toFixed(1)}% |`);
|
|
116
|
+
add(`| Test | ${card.split.counts.test} | ${(card.split.counts.test / total * 100).toFixed(1)}% |`);
|
|
117
|
+
add();
|
|
118
|
+
|
|
119
|
+
add('## Lane Balance');
|
|
120
|
+
add();
|
|
121
|
+
add('| Lane | Total | Train% | Val% | Test% |');
|
|
122
|
+
add('|------|-------|--------|------|-------|');
|
|
123
|
+
for (const [lane, b] of Object.entries(card.lane_balance).sort((a, b) => b[1].total - a[1].total)) {
|
|
124
|
+
add(`| ${lane} | ${b.total} | ${b.train_pct}% | ${b.val_pct}% | ${b.test_pct}% |`);
|
|
125
|
+
}
|
|
126
|
+
add();
|
|
127
|
+
|
|
128
|
+
add('## Quality Gates');
|
|
129
|
+
add();
|
|
130
|
+
add(`- **Constitution rules:** ${card.constitution.rule_count}`);
|
|
131
|
+
add(`- **Rubric dimensions:** ${card.rubric.dimension_count}`);
|
|
132
|
+
if (card.rubric.thresholds) {
|
|
133
|
+
add(`- **Pass threshold:** ${(card.rubric.thresholds.pass * 100).toFixed(0)}%`);
|
|
134
|
+
}
|
|
135
|
+
add();
|
|
136
|
+
|
|
137
|
+
add('## Provenance');
|
|
138
|
+
add();
|
|
139
|
+
add('This dataset was produced by [Style Dataset Lab](https://github.com/mcp-tool-shop-org/style-dataset-lab).');
|
|
140
|
+
add('Every included record has been human-reviewed, canon-bound, and verified against project constitution rules.');
|
|
141
|
+
add('Split strategy ensures no subject appears in multiple partitions.');
|
|
142
|
+
add();
|
|
143
|
+
|
|
144
|
+
return lines.join('\n');
|
|
145
|
+
}
|
package/lib/config.js
CHANGED
|
@@ -102,3 +102,147 @@ export function loadProjectConfig(projectRoot) {
|
|
|
102
102
|
terminology: loadTerminology(projectRoot),
|
|
103
103
|
};
|
|
104
104
|
}
|
|
105
|
+
|
|
106
|
+
// ─── Profile Loaders ─────────────────────────────────────────────────
|
|
107
|
+
|
|
108
|
+
const DEFAULT_SELECTION_PROFILE = {
|
|
109
|
+
require_judgment: true,
|
|
110
|
+
require_status: ['approved'],
|
|
111
|
+
require_canon_bound: true,
|
|
112
|
+
minimum_pass_ratio: 0.5,
|
|
113
|
+
rights_filter: null,
|
|
114
|
+
exclude_lanes: [],
|
|
115
|
+
max_per_subject: null,
|
|
116
|
+
max_per_lane: null,
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
const DEFAULT_SPLIT_PROFILE = {
|
|
120
|
+
profile_id: 'balanced-default',
|
|
121
|
+
strategy: 'subject-isolated',
|
|
122
|
+
train_ratio: 0.8,
|
|
123
|
+
val_ratio: 0.1,
|
|
124
|
+
test_ratio: 0.1,
|
|
125
|
+
lane_balance: true,
|
|
126
|
+
subject_isolation: true,
|
|
127
|
+
seed: 42,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const DEFAULT_EXPORT_PROFILE = {
|
|
131
|
+
profile_id: 'canonical-package',
|
|
132
|
+
include_images: true,
|
|
133
|
+
image_strategy: 'symlink',
|
|
134
|
+
include_splits: true,
|
|
135
|
+
include_card: true,
|
|
136
|
+
include_checksums: true,
|
|
137
|
+
metadata_fields: ['id', 'asset_path', 'provenance', 'judgment', 'canon', 'identity', 'lineage'],
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Load a selection profile from project or return built-in default.
|
|
142
|
+
*/
|
|
143
|
+
export function loadSelectionProfile(projectRoot, profileId) {
|
|
144
|
+
if (!profileId) return { ...DEFAULT_SELECTION_PROFILE };
|
|
145
|
+
const data = loadJsonFile(join(projectRoot, 'selection-profiles', `${profileId}.json`));
|
|
146
|
+
if (!data) return { ...DEFAULT_SELECTION_PROFILE };
|
|
147
|
+
return { ...DEFAULT_SELECTION_PROFILE, ...data };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Load a split profile from project or return built-in default.
|
|
152
|
+
*/
|
|
153
|
+
export function loadSplitProfile(projectRoot, profileId) {
|
|
154
|
+
if (!profileId) return { ...DEFAULT_SPLIT_PROFILE };
|
|
155
|
+
const data = loadJsonFile(join(projectRoot, 'split-profiles', `${profileId}.json`));
|
|
156
|
+
if (!data) return { ...DEFAULT_SPLIT_PROFILE };
|
|
157
|
+
return { ...DEFAULT_SPLIT_PROFILE, ...data };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Load an export profile from project or return built-in default.
|
|
162
|
+
*/
|
|
163
|
+
export function loadExportProfile(projectRoot, profileId) {
|
|
164
|
+
if (!profileId) return { ...DEFAULT_EXPORT_PROFILE };
|
|
165
|
+
const data = loadJsonFile(join(projectRoot, 'export-profiles', `${profileId}.json`));
|
|
166
|
+
if (!data) return { ...DEFAULT_EXPORT_PROFILE };
|
|
167
|
+
return { ...DEFAULT_EXPORT_PROFILE, ...data };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// ─── Detection Functions ─────────────────────────────────────────────
|
|
171
|
+
// Extracted from scripts/canon-bind.js for reuse in eligibility,
|
|
172
|
+
// snapshot, and split modules.
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Detect lane from asset ID using project lane config.
|
|
176
|
+
* Tests each lane's id_patterns as regex against the lowercase ID.
|
|
177
|
+
* Returns the first match, or the default_lane.
|
|
178
|
+
*/
|
|
179
|
+
export function detectLane(id, prompt, lanesConfig) {
|
|
180
|
+
const lower = id.toLowerCase();
|
|
181
|
+
for (const lane of lanesConfig.lanes) {
|
|
182
|
+
if (!lane.id_patterns || lane.id_patterns.length === 0) continue;
|
|
183
|
+
for (const pattern of lane.id_patterns) {
|
|
184
|
+
if (new RegExp(pattern).test(lower)) return lane.id;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return lanesConfig.default_lane;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Detect group (faction) from asset ID and prompt using terminology config.
|
|
192
|
+
* Uses explicit id_detection_order and prompt_detection_order when provided
|
|
193
|
+
* to preserve the exact detection priority of the original implementation.
|
|
194
|
+
*/
|
|
195
|
+
export function detectGroup(id, prompt, termConfig) {
|
|
196
|
+
const idOrder = termConfig.id_detection_order || Object.keys(termConfig.groups);
|
|
197
|
+
const promptOrder = termConfig.prompt_detection_order || Object.keys(termConfig.groups);
|
|
198
|
+
|
|
199
|
+
// 1. Primary ID patterns per group (id_detection_order)
|
|
200
|
+
for (const groupName of idOrder) {
|
|
201
|
+
const group = termConfig.groups[groupName];
|
|
202
|
+
if (!group?.id_patterns) continue;
|
|
203
|
+
for (const pattern of group.id_patterns) {
|
|
204
|
+
if (new RegExp(pattern).test(id)) return groupName;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// 2. Cross-group patterns
|
|
209
|
+
for (const [pattern, behavior] of Object.entries(termConfig.cross_group_patterns || {})) {
|
|
210
|
+
if (new RegExp(pattern).test(id)) {
|
|
211
|
+
if (behavior === 'detect_from_id_suffix') {
|
|
212
|
+
for (const groupName of idOrder) {
|
|
213
|
+
if (id.includes(groupName)) return groupName;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// 3. Edge defaults
|
|
221
|
+
for (const [pattern, groupName] of Object.entries(termConfig.edge_defaults || {})) {
|
|
222
|
+
if (new RegExp(pattern).test(id)) return groupName;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// 4. Prompt-based fallback (prompt_detection_order)
|
|
226
|
+
if (prompt) {
|
|
227
|
+
const p = prompt.toLowerCase();
|
|
228
|
+
for (const groupName of promptOrder) {
|
|
229
|
+
const group = termConfig.groups[groupName];
|
|
230
|
+
if (!group?.prompt_patterns || group.prompt_patterns.length === 0) continue;
|
|
231
|
+
for (const keyword of group.prompt_patterns) {
|
|
232
|
+
if (new RegExp(keyword).test(p)) return groupName;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// 5. Null-faction patterns (generic rejects — no faction)
|
|
238
|
+
for (const pattern of termConfig.null_faction_patterns || []) {
|
|
239
|
+
if (new RegExp(pattern).test(id)) return null;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// 6. ID fallback patterns
|
|
243
|
+
for (const [pattern, groupName] of Object.entries(termConfig.id_fallbacks || {})) {
|
|
244
|
+
if (new RegExp(pattern).test(id)) return groupName;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return null;
|
|
248
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Eligibility evaluation engine.
|
|
3
|
+
*
|
|
4
|
+
* Determines whether each record qualifies for inclusion in a dataset
|
|
5
|
+
* snapshot, with explicit reason traces for every decision.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { detectLane, detectGroup } from './config.js';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Evaluate a single record against a selection profile.
|
|
12
|
+
* @param {Object} record — full record object
|
|
13
|
+
* @param {Object} profile — selection profile
|
|
14
|
+
* @param {Object} lanesConfig — from lanes.json
|
|
15
|
+
* @param {Object} termConfig — from terminology.json
|
|
16
|
+
* @returns {{ eligible: boolean, reasons: string[] }}
|
|
17
|
+
*/
|
|
18
|
+
export function evaluateEligibility(record, profile, lanesConfig, termConfig) {
|
|
19
|
+
const reasons = [];
|
|
20
|
+
|
|
21
|
+
// 1. Judgment required
|
|
22
|
+
if (profile.require_judgment) {
|
|
23
|
+
if (!record.judgment) {
|
|
24
|
+
reasons.push('no judgment (record has not been reviewed)');
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// 2. Status filter
|
|
29
|
+
if (profile.require_status?.length > 0 && record.judgment) {
|
|
30
|
+
if (!profile.require_status.includes(record.judgment.status)) {
|
|
31
|
+
reasons.push(`status "${record.judgment.status}" not in [${profile.require_status.join(', ')}]`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// 3. Canon bound
|
|
36
|
+
if (profile.require_canon_bound) {
|
|
37
|
+
if (!record.canon?.assertions || record.canon.assertions.length === 0) {
|
|
38
|
+
reasons.push('not canon-bound (no assertions)');
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// 4. Minimum pass ratio
|
|
43
|
+
if (profile.minimum_pass_ratio != null && record.canon?.assertion_count > 0) {
|
|
44
|
+
const ratio = record.canon.pass_count / record.canon.assertion_count;
|
|
45
|
+
if (ratio < profile.minimum_pass_ratio) {
|
|
46
|
+
reasons.push(`pass ratio ${(ratio * 100).toFixed(0)}% below minimum ${(profile.minimum_pass_ratio * 100).toFixed(0)}%`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// 5. Lane exclusion
|
|
51
|
+
if (profile.exclude_lanes?.length > 0) {
|
|
52
|
+
const prompt = record.provenance?.prompt || '';
|
|
53
|
+
const lane = detectLane(record.id, prompt, lanesConfig);
|
|
54
|
+
if (profile.exclude_lanes.includes(lane)) {
|
|
55
|
+
reasons.push(`lane "${lane}" is excluded`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// 6. Rights filter
|
|
60
|
+
if (profile.rights_filter != null) {
|
|
61
|
+
const rights = record.rights_status || record.training_rights_status || null;
|
|
62
|
+
if (rights !== profile.rights_filter) {
|
|
63
|
+
reasons.push(`rights "${rights || 'unknown'}" does not match required "${profile.rights_filter}"`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// 7. Tag exclusion
|
|
68
|
+
if (profile.exclude_tags?.length > 0 && record.dataset_tags) {
|
|
69
|
+
const excluded = record.dataset_tags.filter(t => profile.exclude_tags.includes(t));
|
|
70
|
+
if (excluded.length > 0) {
|
|
71
|
+
reasons.push(`has excluded tags: ${excluded.join(', ')}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
eligible: reasons.length === 0,
|
|
77
|
+
reasons,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Categorize exclusion reasons for a set of records.
|
|
83
|
+
* @param {Array<{record_id: string, reasons: string[]}>} excluded
|
|
84
|
+
* @returns {Object} — category → count mapping
|
|
85
|
+
*/
|
|
86
|
+
export function categorizeExclusions(excluded) {
|
|
87
|
+
const categories = {
|
|
88
|
+
no_judgment: 0,
|
|
89
|
+
wrong_status: 0,
|
|
90
|
+
not_canon_bound: 0,
|
|
91
|
+
low_pass_ratio: 0,
|
|
92
|
+
excluded_lane: 0,
|
|
93
|
+
rights_filter: 0,
|
|
94
|
+
excluded_tags: 0,
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
for (const { reasons } of excluded) {
|
|
98
|
+
for (const r of reasons) {
|
|
99
|
+
if (r.includes('no judgment')) categories.no_judgment++;
|
|
100
|
+
else if (r.includes('status')) categories.wrong_status++;
|
|
101
|
+
else if (r.includes('canon-bound')) categories.not_canon_bound++;
|
|
102
|
+
else if (r.includes('pass ratio')) categories.low_pass_ratio++;
|
|
103
|
+
else if (r.includes('lane')) categories.excluded_lane++;
|
|
104
|
+
else if (r.includes('rights')) categories.rights_filter++;
|
|
105
|
+
else if (r.includes('tags')) categories.excluded_tags++;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return categories;
|
|
110
|
+
}
|