@mcptoolshop/style-dataset-lab 1.2.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +73 -0
- package/README.md +138 -113
- package/bin/sdlab.js +126 -0
- package/lib/args.js +106 -0
- package/lib/card.js +145 -0
- package/lib/comfyui.js +109 -0
- package/lib/config.js +248 -0
- package/lib/deprecation.js +14 -0
- package/lib/eligibility.js +110 -0
- package/lib/eval-pack.js +283 -0
- package/lib/export.js +259 -0
- package/lib/paths.js +38 -0
- package/lib/records.js +69 -0
- package/lib/snapshot.js +211 -0
- package/lib/split.js +409 -0
- package/package.json +20 -9
- package/scripts/bulk-curate-wave2-5.js +1 -1
- package/scripts/bulk-curate-waves11-18.js +1 -1
- package/scripts/canon-bind.js +188 -338
- package/scripts/card.js +73 -0
- package/scripts/compare.js +40 -30
- package/scripts/curate-wave25.js +1 -1
- package/scripts/curate.js +34 -28
- package/scripts/doctor.js +324 -0
- package/scripts/eligibility.js +91 -0
- package/scripts/eval-pack.js +92 -0
- package/scripts/export.js +85 -0
- package/scripts/generate-controlnet.js +36 -121
- package/scripts/generate-identity.js +38 -106
- package/scripts/generate-ipadapter.js +42 -119
- package/scripts/generate.js +22 -81
- package/scripts/init.js +229 -0
- package/scripts/migrate-records.js +15 -21
- package/scripts/painterly-test.js +20 -47
- package/scripts/painterly.js +27 -106
- package/scripts/snapshot.js +132 -0
- package/scripts/split.js +154 -0
- package/templates/domains/architecture/constitution.json +97 -0
- package/templates/domains/architecture/lanes.json +41 -0
- package/templates/domains/architecture/project.json +20 -0
- package/templates/domains/architecture/rubric.json +40 -0
- package/templates/domains/architecture/terminology.json +19 -0
- package/templates/domains/character-design/constitution.json +88 -0
- package/templates/domains/character-design/lanes.json +35 -0
- package/templates/domains/character-design/project.json +17 -0
- package/templates/domains/character-design/rubric.json +38 -0
- package/templates/domains/character-design/terminology.json +21 -0
- package/templates/domains/creature-design/constitution.json +87 -0
- package/templates/domains/creature-design/lanes.json +41 -0
- package/templates/domains/creature-design/project.json +18 -0
- package/templates/domains/creature-design/rubric.json +39 -0
- package/templates/domains/creature-design/terminology.json +20 -0
- package/templates/domains/game-art/constitution.json +113 -0
- package/templates/domains/game-art/lanes.json +47 -0
- package/templates/domains/game-art/project.json +17 -0
- package/templates/domains/game-art/rubric.json +36 -0
- package/templates/domains/game-art/terminology.json +30 -0
- package/templates/domains/vehicle-mech/constitution.json +87 -0
- package/templates/domains/vehicle-mech/lanes.json +41 -0
- package/templates/domains/vehicle-mech/project.json +17 -0
- package/templates/domains/vehicle-mech/rubric.json +40 -0
- package/templates/domains/vehicle-mech/terminology.json +21 -0
package/lib/snapshot.js
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Snapshot creation and management.
|
|
3
|
+
*
|
|
4
|
+
* A snapshot is a frozen, versioned selection of records from a project
|
|
5
|
+
* at a point in time. Once created, it never silently changes.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
|
|
9
|
+
import { readFileSync, existsSync, readdirSync } from 'node:fs';
|
|
10
|
+
import { join } from 'node:path';
|
|
11
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
12
|
+
import { loadProjectConfig, loadSelectionProfile, detectLane, detectGroup } from './config.js';
|
|
13
|
+
import { evaluateEligibility } from './eligibility.js';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Generate a snapshot ID: snap-YYYYMMDD-HHMMSS-XXXX
|
|
17
|
+
*/
|
|
18
|
+
function generateSnapshotId() {
|
|
19
|
+
const now = new Date();
|
|
20
|
+
const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14); // YYYYMMDDHHMMSS
|
|
21
|
+
const rand = randomBytes(2).toString('hex');
|
|
22
|
+
return `snap-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Compute config fingerprint — SHA-256 of all 5 config files concatenated.
|
|
27
|
+
* Deterministic: files read in alphabetical order.
|
|
28
|
+
*/
|
|
29
|
+
export function computeConfigFingerprint(projectRoot) {
|
|
30
|
+
const configFiles = [
|
|
31
|
+
'constitution.json',
|
|
32
|
+
'lanes.json',
|
|
33
|
+
'project.json',
|
|
34
|
+
'rubric.json',
|
|
35
|
+
'terminology.json',
|
|
36
|
+
];
|
|
37
|
+
|
|
38
|
+
const hash = createHash('sha256');
|
|
39
|
+
for (const file of configFiles) {
|
|
40
|
+
const filePath = join(projectRoot, file);
|
|
41
|
+
if (existsSync(filePath)) {
|
|
42
|
+
hash.update(`--- ${file} ---\n`);
|
|
43
|
+
hash.update(readFileSync(filePath, 'utf-8'));
|
|
44
|
+
hash.update('\n');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return hash.digest('hex');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Create a snapshot — frozen selection of eligible records.
|
|
52
|
+
* @param {string} projectRoot — absolute path to project
|
|
53
|
+
* @param {Object} profile — selection profile
|
|
54
|
+
* @returns {Promise<{snapshotId: string, included: number, excluded: number}>}
|
|
55
|
+
*/
|
|
56
|
+
export async function createSnapshot(projectRoot, profile) {
|
|
57
|
+
const config = loadProjectConfig(projectRoot);
|
|
58
|
+
const recordsDir = join(projectRoot, 'records');
|
|
59
|
+
const snapshotId = generateSnapshotId();
|
|
60
|
+
const snapshotDir = join(projectRoot, 'snapshots', snapshotId);
|
|
61
|
+
|
|
62
|
+
// Load all records (sorted for deterministic output)
|
|
63
|
+
const files = (await readdir(recordsDir))
|
|
64
|
+
.filter(f => f.endsWith('.json'))
|
|
65
|
+
.sort();
|
|
66
|
+
|
|
67
|
+
const included = [];
|
|
68
|
+
const excluded = [];
|
|
69
|
+
const laneDistribution = {};
|
|
70
|
+
const factionDistribution = {};
|
|
71
|
+
const verdictDistribution = { pass: 0, fail: 0, partial: 0 };
|
|
72
|
+
|
|
73
|
+
for (const file of files) {
|
|
74
|
+
const raw = await readFile(join(recordsDir, file), 'utf-8');
|
|
75
|
+
const record = JSON.parse(raw);
|
|
76
|
+
|
|
77
|
+
const { eligible, reasons } = evaluateEligibility(
|
|
78
|
+
record, profile, config.lanes, config.terminology
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
if (eligible) {
|
|
82
|
+
const prompt = record.provenance?.prompt || '';
|
|
83
|
+
const lane = detectLane(record.id, prompt, config.lanes);
|
|
84
|
+
const faction = detectGroup(record.id, prompt, config.terminology);
|
|
85
|
+
|
|
86
|
+
// Build reason trace
|
|
87
|
+
const passRatio = record.canon?.assertion_count > 0
|
|
88
|
+
? `${record.canon.pass_count}/${record.canon.assertion_count}`
|
|
89
|
+
: 'n/a';
|
|
90
|
+
const reason = `${record.judgment.status}, canon-bound, pass_ratio=${passRatio}, lane=${lane}`;
|
|
91
|
+
|
|
92
|
+
included.push({ record_id: record.id, reason });
|
|
93
|
+
|
|
94
|
+
// Track distributions
|
|
95
|
+
laneDistribution[lane] = (laneDistribution[lane] || 0) + 1;
|
|
96
|
+
factionDistribution[faction || 'unknown'] = (factionDistribution[faction || 'unknown'] || 0) + 1;
|
|
97
|
+
if (record.canon?.assertions) {
|
|
98
|
+
for (const a of record.canon.assertions) {
|
|
99
|
+
verdictDistribution[a.verdict] = (verdictDistribution[a.verdict] || 0) + 1;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
} else {
|
|
103
|
+
excluded.push({ record_id: record.id, reasons });
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Write snapshot files
|
|
108
|
+
await mkdir(snapshotDir, { recursive: true });
|
|
109
|
+
|
|
110
|
+
const manifest = {
|
|
111
|
+
snapshot_id: snapshotId,
|
|
112
|
+
created_at: new Date().toISOString(),
|
|
113
|
+
created_by: 'sdlab-snapshot-v1',
|
|
114
|
+
project_name: config.meta.name,
|
|
115
|
+
frozen: true,
|
|
116
|
+
selection_profile: profile,
|
|
117
|
+
config_fingerprint: computeConfigFingerprint(projectRoot),
|
|
118
|
+
config_versions: {
|
|
119
|
+
constitution: config.constitution.version || '1.0.0',
|
|
120
|
+
project: config.meta.version || '0.0.0',
|
|
121
|
+
},
|
|
122
|
+
counts: {
|
|
123
|
+
total_records: files.length,
|
|
124
|
+
evaluated: files.length,
|
|
125
|
+
included: included.length,
|
|
126
|
+
excluded: excluded.length,
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const summary = {
|
|
131
|
+
snapshot_id: snapshotId,
|
|
132
|
+
lane_distribution: laneDistribution,
|
|
133
|
+
faction_distribution: factionDistribution,
|
|
134
|
+
verdict_distribution: verdictDistribution,
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
await writeFile(join(snapshotDir, 'snapshot.json'), JSON.stringify(manifest, null, 2) + '\n');
|
|
138
|
+
await writeFile(join(snapshotDir, 'included.jsonl'), included.map(r => JSON.stringify(r)).join('\n') + '\n');
|
|
139
|
+
await writeFile(join(snapshotDir, 'excluded.jsonl'), excluded.map(r => JSON.stringify(r)).join('\n') + '\n');
|
|
140
|
+
await writeFile(join(snapshotDir, 'summary.json'), JSON.stringify(summary, null, 2) + '\n');
|
|
141
|
+
|
|
142
|
+
return { snapshotId, included: included.length, excluded: excluded.length };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Load a snapshot manifest.
|
|
147
|
+
*/
|
|
148
|
+
export async function loadSnapshot(projectRoot, snapshotId) {
|
|
149
|
+
const path = join(projectRoot, 'snapshots', snapshotId, 'snapshot.json');
|
|
150
|
+
if (!existsSync(path)) {
|
|
151
|
+
throw new Error(`Snapshot "${snapshotId}" not found at ${path}`);
|
|
152
|
+
}
|
|
153
|
+
return JSON.parse(await readFile(path, 'utf-8'));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Load included record IDs from a snapshot.
|
|
158
|
+
*/
|
|
159
|
+
export async function loadSnapshotIncluded(projectRoot, snapshotId) {
|
|
160
|
+
const path = join(projectRoot, 'snapshots', snapshotId, 'included.jsonl');
|
|
161
|
+
const raw = await readFile(path, 'utf-8');
|
|
162
|
+
return raw.trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* List all snapshots in a project.
|
|
167
|
+
*/
|
|
168
|
+
export async function listSnapshots(projectRoot) {
|
|
169
|
+
const snapshotsDir = join(projectRoot, 'snapshots');
|
|
170
|
+
if (!existsSync(snapshotsDir)) return [];
|
|
171
|
+
|
|
172
|
+
const entries = await readdir(snapshotsDir, { withFileTypes: true });
|
|
173
|
+
const snapshots = [];
|
|
174
|
+
|
|
175
|
+
for (const entry of entries) {
|
|
176
|
+
if (!entry.isDirectory() || !entry.name.startsWith('snap-')) continue;
|
|
177
|
+
try {
|
|
178
|
+
const manifest = JSON.parse(
|
|
179
|
+
await readFile(join(snapshotsDir, entry.name, 'snapshot.json'), 'utf-8')
|
|
180
|
+
);
|
|
181
|
+
snapshots.push({
|
|
182
|
+
id: manifest.snapshot_id,
|
|
183
|
+
created_at: manifest.created_at,
|
|
184
|
+
included: manifest.counts.included,
|
|
185
|
+
excluded: manifest.counts.excluded,
|
|
186
|
+
fingerprint: manifest.config_fingerprint.slice(0, 12),
|
|
187
|
+
});
|
|
188
|
+
} catch {
|
|
189
|
+
// Skip malformed snapshots
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return snapshots.sort((a, b) => a.created_at.localeCompare(b.created_at));
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Diff two snapshots — records added, removed, and shared.
|
|
198
|
+
*/
|
|
199
|
+
export async function diffSnapshots(projectRoot, idA, idB) {
|
|
200
|
+
const includedA = await loadSnapshotIncluded(projectRoot, idA);
|
|
201
|
+
const includedB = await loadSnapshotIncluded(projectRoot, idB);
|
|
202
|
+
|
|
203
|
+
const setA = new Set(includedA.map(r => r.record_id));
|
|
204
|
+
const setB = new Set(includedB.map(r => r.record_id));
|
|
205
|
+
|
|
206
|
+
const added = [...setB].filter(id => !setA.has(id));
|
|
207
|
+
const removed = [...setA].filter(id => !setB.has(id));
|
|
208
|
+
const shared = [...setA].filter(id => setB.has(id));
|
|
209
|
+
|
|
210
|
+
return { added, removed, shared, countA: setA.size, countB: setB.size };
|
|
211
|
+
}
|
package/lib/split.js
ADDED
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Split engine — subject-isolated, lane-balanced dataset splitting.
|
|
3
|
+
*
|
|
4
|
+
* Laws:
|
|
5
|
+
* 1. Records sharing a subject family always land in the same split
|
|
6
|
+
* 2. Lane balance is maintained within defined tolerances
|
|
7
|
+
* 3. Splits are deterministic (seeded PRNG, sorted inputs)
|
|
8
|
+
* 4. Every split produces an audit trail proving no leakage
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { existsSync } from 'node:fs';
|
|
14
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
15
|
+
import { loadProjectConfig, loadSplitProfile, detectLane } from './config.js';
|
|
16
|
+
import { loadSnapshot, loadSnapshotIncluded } from './snapshot.js';
|
|
17
|
+
import { loadRecord } from './records.js';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Mulberry32 — deterministic 32-bit PRNG.
|
|
21
|
+
* Same seed always produces the same sequence.
|
|
22
|
+
*/
|
|
23
|
+
function mulberry32(seed) {
|
|
24
|
+
let t = seed >>> 0;
|
|
25
|
+
return function () {
|
|
26
|
+
t = (t + 0x6d2b79f5) | 0;
|
|
27
|
+
let v = t;
|
|
28
|
+
v = Math.imul(v ^ (v >>> 15), v | 1);
|
|
29
|
+
v ^= v + Math.imul(v ^ (v >>> 7), v | 61);
|
|
30
|
+
return ((v ^ (v >>> 14)) >>> 0) / 4294967296;
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Fisher-Yates shuffle with seeded PRNG.
|
|
36
|
+
*/
|
|
37
|
+
function seededShuffle(arr, rng) {
|
|
38
|
+
const out = [...arr];
|
|
39
|
+
for (let i = out.length - 1; i > 0; i--) {
|
|
40
|
+
const j = Math.floor(rng() * (i + 1));
|
|
41
|
+
[out[i], out[j]] = [out[j], out[i]];
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Generate a split ID: split-YYYYMMDD-HHMMSS-XXXX
|
|
48
|
+
*/
|
|
49
|
+
function generateSplitId() {
|
|
50
|
+
const now = new Date();
|
|
51
|
+
const date = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
|
|
52
|
+
const rand = randomBytes(2).toString('hex');
|
|
53
|
+
return `split-${date.slice(0, 8)}-${date.slice(8, 14)}-${rand}`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Determine the subject family for a record.
|
|
58
|
+
*
|
|
59
|
+
* Priority:
|
|
60
|
+
* 1. identity.subject_name (named characters like renna_vasik)
|
|
61
|
+
* 2. lineage.derived_from_record_id (walk chain to root ancestor)
|
|
62
|
+
* 3. Strip _v\d+$, _s\d+$ suffixes from record ID
|
|
63
|
+
*/
|
|
64
|
+
function resolveSubjectFamily(record) {
|
|
65
|
+
// 1. Named subject
|
|
66
|
+
if (record.identity?.subject_name) {
|
|
67
|
+
return record.identity.subject_name;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// 2. Lineage chain — use root ancestor ID as family
|
|
71
|
+
if (record.lineage?.derived_from_record_id) {
|
|
72
|
+
// We return the ancestor ID; the caller's ancestor map handles chain walking
|
|
73
|
+
return record.lineage.derived_from_record_id;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// 3. Strip version/seed suffixes from record ID
|
|
77
|
+
return record.id.replace(/_[vs]\d+$/, '');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Walk lineage chains to find root ancestors.
|
|
82
|
+
* Returns a map: recordId → rootAncestorFamily
|
|
83
|
+
*/
|
|
84
|
+
function buildAncestorMap(records) {
|
|
85
|
+
const parentOf = new Map();
|
|
86
|
+
const idToRecord = new Map();
|
|
87
|
+
|
|
88
|
+
for (const record of records) {
|
|
89
|
+
idToRecord.set(record.id, record);
|
|
90
|
+
if (record.lineage?.derived_from_record_id) {
|
|
91
|
+
parentOf.set(record.id, record.lineage.derived_from_record_id);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Walk each chain to root (with cycle protection)
|
|
96
|
+
function findRoot(id, visited = new Set()) {
|
|
97
|
+
if (visited.has(id)) return id; // cycle — stop
|
|
98
|
+
visited.add(id);
|
|
99
|
+
const parent = parentOf.get(id);
|
|
100
|
+
if (!parent) return id;
|
|
101
|
+
return findRoot(parent, visited);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const ancestorMap = new Map();
|
|
105
|
+
for (const record of records) {
|
|
106
|
+
if (record.lineage?.derived_from_record_id) {
|
|
107
|
+
const rootId = findRoot(record.id);
|
|
108
|
+
// Use the root's subject family
|
|
109
|
+
const rootRecord = idToRecord.get(rootId);
|
|
110
|
+
if (rootRecord) {
|
|
111
|
+
ancestorMap.set(record.id, resolveSubjectFamily(rootRecord));
|
|
112
|
+
} else {
|
|
113
|
+
// Root not in dataset — strip suffixes from root ID
|
|
114
|
+
ancestorMap.set(record.id, rootId.replace(/_[vs]\d+$/, ''));
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return ancestorMap;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Create a split from a snapshot.
|
|
124
|
+
*
|
|
125
|
+
* @param {string} projectRoot — absolute path to project
|
|
126
|
+
* @param {string} snapshotId — snapshot to split
|
|
127
|
+
* @param {Object} profile — split profile (from loadSplitProfile)
|
|
128
|
+
* @returns {Promise<{splitId: string, train: number, val: number, test: number}>}
|
|
129
|
+
*/
|
|
130
|
+
export async function createSplit(projectRoot, snapshotId, profile) {
|
|
131
|
+
const config = loadProjectConfig(projectRoot);
|
|
132
|
+
const snapshot = await loadSnapshot(projectRoot, snapshotId);
|
|
133
|
+
const includedEntries = await loadSnapshotIncluded(projectRoot, snapshotId);
|
|
134
|
+
const recordsDir = join(projectRoot, 'records');
|
|
135
|
+
|
|
136
|
+
// Load full records for included entries
|
|
137
|
+
const records = [];
|
|
138
|
+
for (const entry of includedEntries) {
|
|
139
|
+
const record = await loadRecord(recordsDir, entry.record_id);
|
|
140
|
+
if (record) records.push(record);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Build ancestor map for lineage chain resolution
|
|
144
|
+
const ancestorMap = buildAncestorMap(records);
|
|
145
|
+
|
|
146
|
+
// Assign each record to a subject family
|
|
147
|
+
const familyMap = new Map(); // family → [record]
|
|
148
|
+
for (const record of records) {
|
|
149
|
+
let family;
|
|
150
|
+
if (ancestorMap.has(record.id)) {
|
|
151
|
+
family = ancestorMap.get(record.id);
|
|
152
|
+
} else {
|
|
153
|
+
family = resolveSubjectFamily(record);
|
|
154
|
+
}
|
|
155
|
+
if (!familyMap.has(family)) familyMap.set(family, []);
|
|
156
|
+
familyMap.get(family).push(record);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Detect primary lane per family (first record's lane)
|
|
160
|
+
const familyLane = new Map();
|
|
161
|
+
for (const [family, recs] of familyMap) {
|
|
162
|
+
const first = recs[0];
|
|
163
|
+
const prompt = first.provenance?.prompt || '';
|
|
164
|
+
const lane = detectLane(first.id, prompt, config.lanes);
|
|
165
|
+
familyLane.set(family, lane);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Group families by lane
|
|
169
|
+
const laneGroups = new Map(); // lane → [family]
|
|
170
|
+
for (const [family, lane] of familyLane) {
|
|
171
|
+
if (!laneGroups.has(lane)) laneGroups.set(lane, []);
|
|
172
|
+
laneGroups.get(lane).push(family);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Split families per lane using seeded PRNG
|
|
176
|
+
const rng = mulberry32(profile.seed || 42);
|
|
177
|
+
const trainFamilies = new Set();
|
|
178
|
+
const valFamilies = new Set();
|
|
179
|
+
const testFamilies = new Set();
|
|
180
|
+
const warnings = [];
|
|
181
|
+
|
|
182
|
+
const sortedLanes = [...laneGroups.keys()].sort();
|
|
183
|
+
for (const lane of sortedLanes) {
|
|
184
|
+
const families = laneGroups.get(lane).sort(); // sort for determinism before shuffle
|
|
185
|
+
const shuffled = seededShuffle(families, rng);
|
|
186
|
+
|
|
187
|
+
if (shuffled.length < 3) {
|
|
188
|
+
// Too few families — all to train
|
|
189
|
+
for (const f of shuffled) trainFamilies.add(f);
|
|
190
|
+
warnings.push(`lane "${lane}": only ${shuffled.length} families — all assigned to train`);
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Assign by ratio
|
|
195
|
+
const total = shuffled.length;
|
|
196
|
+
const trainCount = Math.round(total * profile.train_ratio);
|
|
197
|
+
const valCount = Math.round(total * profile.val_ratio);
|
|
198
|
+
// test gets the rest
|
|
199
|
+
|
|
200
|
+
let idx = 0;
|
|
201
|
+
for (let i = 0; i < trainCount && idx < total; i++, idx++) {
|
|
202
|
+
trainFamilies.add(shuffled[idx]);
|
|
203
|
+
}
|
|
204
|
+
for (let i = 0; i < valCount && idx < total; i++, idx++) {
|
|
205
|
+
valFamilies.add(shuffled[idx]);
|
|
206
|
+
}
|
|
207
|
+
while (idx < total) {
|
|
208
|
+
testFamilies.add(shuffled[idx]);
|
|
209
|
+
idx++;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Build record → split assignment
|
|
214
|
+
const trainRecords = [];
|
|
215
|
+
const valRecords = [];
|
|
216
|
+
const testRecords = [];
|
|
217
|
+
|
|
218
|
+
for (const record of records) {
|
|
219
|
+
let family;
|
|
220
|
+
if (ancestorMap.has(record.id)) {
|
|
221
|
+
family = ancestorMap.get(record.id);
|
|
222
|
+
} else {
|
|
223
|
+
family = resolveSubjectFamily(record);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (trainFamilies.has(family)) trainRecords.push(record.id);
|
|
227
|
+
else if (valFamilies.has(family)) valRecords.push(record.id);
|
|
228
|
+
else if (testFamilies.has(family)) testRecords.push(record.id);
|
|
229
|
+
else trainRecords.push(record.id); // fallback: unassigned → train
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Sort for deterministic output
|
|
233
|
+
trainRecords.sort();
|
|
234
|
+
valRecords.sort();
|
|
235
|
+
testRecords.sort();
|
|
236
|
+
|
|
237
|
+
// Leakage audit
|
|
238
|
+
const leakageIssues = auditLeakage(familyMap, trainRecords, valRecords, testRecords);
|
|
239
|
+
|
|
240
|
+
// Lane balance audit
|
|
241
|
+
const laneBalance = auditLaneBalance(
|
|
242
|
+
records, trainRecords, valRecords, testRecords, config.lanes
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
// Write output
|
|
246
|
+
const splitId = generateSplitId();
|
|
247
|
+
const splitDir = join(projectRoot, 'splits', splitId);
|
|
248
|
+
await mkdir(splitDir, { recursive: true });
|
|
249
|
+
|
|
250
|
+
const manifest = {
|
|
251
|
+
split_id: splitId,
|
|
252
|
+
created_at: new Date().toISOString(),
|
|
253
|
+
created_by: 'sdlab-split-v1',
|
|
254
|
+
snapshot_id: snapshotId,
|
|
255
|
+
snapshot_fingerprint: snapshot.config_fingerprint,
|
|
256
|
+
profile,
|
|
257
|
+
counts: {
|
|
258
|
+
total_records: records.length,
|
|
259
|
+
total_families: familyMap.size,
|
|
260
|
+
train: trainRecords.length,
|
|
261
|
+
val: valRecords.length,
|
|
262
|
+
test: testRecords.length,
|
|
263
|
+
},
|
|
264
|
+
warnings,
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
const audit = {
|
|
268
|
+
split_id: splitId,
|
|
269
|
+
leakage_check: {
|
|
270
|
+
passed: leakageIssues.length === 0,
|
|
271
|
+
issues: leakageIssues,
|
|
272
|
+
},
|
|
273
|
+
lane_balance: laneBalance,
|
|
274
|
+
family_count: familyMap.size,
|
|
275
|
+
families_per_split: {
|
|
276
|
+
train: trainFamilies.size,
|
|
277
|
+
val: valFamilies.size,
|
|
278
|
+
test: testFamilies.size,
|
|
279
|
+
},
|
|
280
|
+
};
|
|
281
|
+
|
|
282
|
+
await writeFile(join(splitDir, 'split.json'), JSON.stringify(manifest, null, 2) + '\n');
|
|
283
|
+
await writeFile(join(splitDir, 'train.jsonl'), trainRecords.map(id => JSON.stringify({ record_id: id })).join('\n') + '\n');
|
|
284
|
+
await writeFile(join(splitDir, 'val.jsonl'), valRecords.map(id => JSON.stringify({ record_id: id })).join('\n') + '\n');
|
|
285
|
+
await writeFile(join(splitDir, 'test.jsonl'), testRecords.map(id => JSON.stringify({ record_id: id })).join('\n') + '\n');
|
|
286
|
+
await writeFile(join(splitDir, 'audit.json'), JSON.stringify(audit, null, 2) + '\n');
|
|
287
|
+
|
|
288
|
+
return { splitId, train: trainRecords.length, val: valRecords.length, test: testRecords.length };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Audit for subject leakage across splits.
|
|
293
|
+
* Returns an array of issues (empty = clean).
|
|
294
|
+
*/
|
|
295
|
+
function auditLeakage(familyMap, trainIds, valIds, testIds) {
|
|
296
|
+
const trainSet = new Set(trainIds);
|
|
297
|
+
const valSet = new Set(valIds);
|
|
298
|
+
const testSet = new Set(testIds);
|
|
299
|
+
const issues = [];
|
|
300
|
+
|
|
301
|
+
for (const [family, records] of familyMap) {
|
|
302
|
+
const splits = new Set();
|
|
303
|
+
for (const r of records) {
|
|
304
|
+
if (trainSet.has(r.id)) splits.add('train');
|
|
305
|
+
if (valSet.has(r.id)) splits.add('val');
|
|
306
|
+
if (testSet.has(r.id)) splits.add('test');
|
|
307
|
+
}
|
|
308
|
+
if (splits.size > 1) {
|
|
309
|
+
issues.push({
|
|
310
|
+
family,
|
|
311
|
+
record_count: records.length,
|
|
312
|
+
leaked_to: [...splits],
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
return issues;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Audit lane balance across splits.
|
|
322
|
+
*/
|
|
323
|
+
function auditLaneBalance(records, trainIds, valIds, testIds, lanesConfig) {
|
|
324
|
+
const trainSet = new Set(trainIds);
|
|
325
|
+
const valSet = new Set(valIds);
|
|
326
|
+
const testSet = new Set(testIds);
|
|
327
|
+
const balance = {};
|
|
328
|
+
|
|
329
|
+
for (const record of records) {
|
|
330
|
+
const prompt = record.provenance?.prompt || '';
|
|
331
|
+
const lane = detectLane(record.id, prompt, lanesConfig);
|
|
332
|
+
if (!balance[lane]) balance[lane] = { total: 0, train: 0, val: 0, test: 0 };
|
|
333
|
+
balance[lane].total++;
|
|
334
|
+
if (trainSet.has(record.id)) balance[lane].train++;
|
|
335
|
+
else if (valSet.has(record.id)) balance[lane].val++;
|
|
336
|
+
else if (testSet.has(record.id)) balance[lane].test++;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Compute percentages
|
|
340
|
+
for (const lane of Object.keys(balance)) {
|
|
341
|
+
const b = balance[lane];
|
|
342
|
+
b.train_pct = b.total > 0 ? +(b.train / b.total * 100).toFixed(1) : 0;
|
|
343
|
+
b.val_pct = b.total > 0 ? +(b.val / b.total * 100).toFixed(1) : 0;
|
|
344
|
+
b.test_pct = b.total > 0 ? +(b.test / b.total * 100).toFixed(1) : 0;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
return balance;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Load a split manifest.
|
|
352
|
+
*/
|
|
353
|
+
export async function loadSplit(projectRoot, splitId) {
|
|
354
|
+
const path = join(projectRoot, 'splits', splitId, 'split.json');
|
|
355
|
+
if (!existsSync(path)) {
|
|
356
|
+
throw new Error(`Split "${splitId}" not found at ${path}`);
|
|
357
|
+
}
|
|
358
|
+
return JSON.parse(await readFile(path, 'utf-8'));
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Load split record IDs for a partition.
|
|
363
|
+
*/
|
|
364
|
+
export async function loadSplitPartition(projectRoot, splitId, partition) {
|
|
365
|
+
const path = join(projectRoot, 'splits', splitId, `${partition}.jsonl`);
|
|
366
|
+
const raw = await readFile(path, 'utf-8');
|
|
367
|
+
return raw.trim().split('\n').filter(Boolean).map(line => JSON.parse(line));
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Load split audit.
|
|
372
|
+
*/
|
|
373
|
+
export async function loadSplitAudit(projectRoot, splitId) {
|
|
374
|
+
const path = join(projectRoot, 'splits', splitId, 'audit.json');
|
|
375
|
+
return JSON.parse(await readFile(path, 'utf-8'));
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* List all splits in a project.
|
|
380
|
+
*/
|
|
381
|
+
export async function listSplits(projectRoot) {
|
|
382
|
+
const splitsDir = join(projectRoot, 'splits');
|
|
383
|
+
if (!existsSync(splitsDir)) return [];
|
|
384
|
+
|
|
385
|
+
const entries = await readdir(splitsDir, { withFileTypes: true });
|
|
386
|
+
const splits = [];
|
|
387
|
+
|
|
388
|
+
for (const entry of entries) {
|
|
389
|
+
if (!entry.isDirectory() || !entry.name.startsWith('split-')) continue;
|
|
390
|
+
try {
|
|
391
|
+
const manifest = JSON.parse(
|
|
392
|
+
await readFile(join(splitsDir, entry.name, 'split.json'), 'utf-8')
|
|
393
|
+
);
|
|
394
|
+
splits.push({
|
|
395
|
+
id: manifest.split_id,
|
|
396
|
+
created_at: manifest.created_at,
|
|
397
|
+
snapshot_id: manifest.snapshot_id,
|
|
398
|
+
train: manifest.counts.train,
|
|
399
|
+
val: manifest.counts.val,
|
|
400
|
+
test: manifest.counts.test,
|
|
401
|
+
families: manifest.counts.total_families,
|
|
402
|
+
});
|
|
403
|
+
} catch {
|
|
404
|
+
// Skip malformed
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
return splits.sort((a, b) => a.created_at.localeCompare(b.created_at));
|
|
409
|
+
}
|
package/package.json
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mcptoolshop/style-dataset-lab",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "Visual dataset pipeline —
|
|
3
|
+
"version": "2.1.0",
|
|
4
|
+
"description": "Visual canon and dataset production pipeline — snapshots, splits, versioned exports, eval packs, and canon-bound curation for any image production domain",
|
|
5
5
|
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"sdlab": "./bin/sdlab.js"
|
|
8
|
+
},
|
|
6
9
|
"scripts": {
|
|
7
10
|
"generate": "node scripts/generate.js",
|
|
8
11
|
"curate": "node scripts/curate.js",
|
|
@@ -10,10 +13,12 @@
|
|
|
10
13
|
"canon-bind": "node scripts/canon-bind.js",
|
|
11
14
|
"painterly": "node scripts/painterly.js",
|
|
12
15
|
"generate:identity": "node scripts/generate-identity.js",
|
|
13
|
-
"export": "echo 'Usage: repo-dataset visual generate ./
|
|
14
|
-
"inspect": "echo 'Usage: repo-dataset visual inspect ./
|
|
16
|
+
"export": "echo 'Usage: repo-dataset visual generate ./projects/<project> --format trl'",
|
|
17
|
+
"inspect": "echo 'Usage: repo-dataset visual inspect ./projects/<project>'"
|
|
15
18
|
},
|
|
16
19
|
"files": [
|
|
20
|
+
"bin/",
|
|
21
|
+
"lib/",
|
|
17
22
|
"scripts/",
|
|
18
23
|
"templates/",
|
|
19
24
|
"README.md",
|
|
@@ -23,12 +28,18 @@
|
|
|
23
28
|
],
|
|
24
29
|
"keywords": [
|
|
25
30
|
"dataset",
|
|
26
|
-
"
|
|
31
|
+
"computer-vision",
|
|
27
32
|
"training-data",
|
|
28
|
-
"
|
|
29
|
-
"
|
|
30
|
-
"
|
|
31
|
-
"
|
|
33
|
+
"image-dataset",
|
|
34
|
+
"evaluation",
|
|
35
|
+
"curation",
|
|
36
|
+
"canon",
|
|
37
|
+
"visual-development",
|
|
38
|
+
"concept-art",
|
|
39
|
+
"style-guide",
|
|
40
|
+
"lora",
|
|
41
|
+
"multimodal",
|
|
42
|
+
"comfyui"
|
|
32
43
|
],
|
|
33
44
|
"homepage": "https://mcp-tool-shop-org.github.io/style-dataset-lab/",
|
|
34
45
|
"repository": {
|
|
@@ -11,7 +11,7 @@ import { join } from "node:path";
|
|
|
11
11
|
|
|
12
12
|
const REPO_ROOT = new URL("..", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
|
|
13
13
|
const GAME = process.argv.find((a, i) => process.argv[i - 1] === '--game') || 'star-freight';
|
|
14
|
-
const GAME_ROOT = join(REPO_ROOT, '
|
|
14
|
+
const GAME_ROOT = join(REPO_ROOT, 'projects', GAME);
|
|
15
15
|
|
|
16
16
|
const DEFAULT_APPROVED_SCORES = {
|
|
17
17
|
silhouette_clarity: 0.8, palette_adherence: 0.8, material_fidelity: 0.8,
|
|
@@ -17,7 +17,7 @@ import { join, basename } from 'path';
|
|
|
17
17
|
|
|
18
18
|
const REPO_ROOT = new URL("..", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
|
|
19
19
|
const GAME = process.argv.find((a, i) => process.argv[i - 1] === '--game') || 'star-freight';
|
|
20
|
-
const GAME_ROOT = join(REPO_ROOT, '
|
|
20
|
+
const GAME_ROOT = join(REPO_ROOT, 'projects', GAME);
|
|
21
21
|
|
|
22
22
|
const RECORDS_DIR = join(GAME_ROOT, 'records');
|
|
23
23
|
const CANDIDATES_DIR = join(GAME_ROOT, 'outputs', 'candidates');
|