@mcptoolshop/style-dataset-lab 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.es.md +76 -144
- package/README.fr.md +75 -143
- package/README.hi.md +75 -143
- package/README.it.md +76 -144
- package/README.ja.md +75 -143
- package/README.md +80 -94
- package/README.pt-BR.md +75 -143
- package/README.zh.md +76 -144
- package/SECURITY.md +3 -1
- package/bin/sdlab.js +21 -21
- package/lib/errors.js +82 -0
- package/lib/log.js +36 -0
- package/lib/snapshot.js +5 -2
- package/lib/split.js +5 -2
- package/package.json +3 -4
- package/scripts/snapshot.js +27 -23
- package/scripts/split.js +3 -1
package/bin/sdlab.js
CHANGED
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
* Each script exports run(argv) and is lazy-loaded on demand.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import { enableDebug, handleCliError, inputError } from '../lib/errors.js';
|
|
11
|
+
import { setLogLevel } from '../lib/log.js';
|
|
12
|
+
|
|
10
13
|
const COMMANDS = {
|
|
11
14
|
'init': '../scripts/init.js',
|
|
12
15
|
'generate': '../scripts/generate.js',
|
|
@@ -99,18 +102,27 @@ function printHelp() {
|
|
|
99
102
|
console.log('Options:');
|
|
100
103
|
console.log(' --project <name> Project to operate on (default: star-freight)');
|
|
101
104
|
console.log(' --game <name> Deprecated alias for --project');
|
|
105
|
+
console.log(' --debug Show stack traces on error');
|
|
106
|
+
console.log(' --verbose Verbose output');
|
|
107
|
+
console.log(' --quiet Suppress non-essential output');
|
|
108
|
+
console.log(' --dry-run Preview changes without writing (where supported)');
|
|
102
109
|
console.log(' --help Show this help');
|
|
103
110
|
console.log('');
|
|
104
111
|
console.log('Examples:');
|
|
105
112
|
console.log(' sdlab init my-project --domain character-design');
|
|
106
|
-
console.log(' sdlab
|
|
107
|
-
console.log(' sdlab
|
|
108
|
-
console.log(' sdlab
|
|
113
|
+
console.log(' sdlab snapshot create --project star-freight');
|
|
114
|
+
console.log(' sdlab split build --dry-run --project star-freight');
|
|
115
|
+
console.log(' sdlab training-package build --debug --project star-freight');
|
|
109
116
|
}
|
|
110
117
|
|
|
111
118
|
async function main() {
|
|
112
119
|
const args = process.argv.slice(2);
|
|
113
120
|
|
|
121
|
+
// Global flags
|
|
122
|
+
if (args.includes('--debug')) enableDebug();
|
|
123
|
+
if (args.includes('--verbose')) setLogLevel('verbose');
|
|
124
|
+
if (args.includes('--quiet')) setLogLevel('quiet');
|
|
125
|
+
|
|
114
126
|
if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
|
|
115
127
|
printHelp();
|
|
116
128
|
process.exit(0);
|
|
@@ -123,30 +135,18 @@ async function main() {
|
|
|
123
135
|
if (command === 'project' && args[1] && PROJECT_COMMANDS[args[1]]) {
|
|
124
136
|
commandArgs = args.slice(2);
|
|
125
137
|
const modulePath = PROJECT_COMMANDS[args[1]];
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
await mod.run(commandArgs);
|
|
129
|
-
} catch (err) {
|
|
130
|
-
console.error(`\x1b[31mError:\x1b[0m ${err.message}`);
|
|
131
|
-
process.exit(1);
|
|
132
|
-
}
|
|
138
|
+
const mod = await import(modulePath);
|
|
139
|
+
await mod.run(commandArgs);
|
|
133
140
|
return;
|
|
134
141
|
}
|
|
135
142
|
|
|
136
143
|
const modulePath = COMMANDS[command];
|
|
137
144
|
if (!modulePath) {
|
|
138
|
-
|
|
139
|
-
console.error(`Run "sdlab --help" for available commands.`);
|
|
140
|
-
process.exit(1);
|
|
145
|
+
throw inputError('INPUT_UNKNOWN_COMMAND', `Unknown command: ${command}`, 'Run "sdlab --help" for available commands.');
|
|
141
146
|
}
|
|
142
147
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
await mod.run(commandArgs);
|
|
146
|
-
} catch (err) {
|
|
147
|
-
console.error(`\x1b[31mError:\x1b[0m ${err.message}`);
|
|
148
|
-
process.exit(1);
|
|
149
|
-
}
|
|
148
|
+
const mod = await import(modulePath);
|
|
149
|
+
await mod.run(commandArgs);
|
|
150
150
|
}
|
|
151
151
|
|
|
152
|
-
main();
|
|
152
|
+
main().catch(handleCliError);
|
package/lib/errors.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Structured error shape for Style Dataset Lab.
|
|
3
|
+
*
|
|
4
|
+
* Every user-facing error has: code, message, hint.
|
|
5
|
+
* Exit codes: 0 ok, 1 user error, 2 runtime error.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export class SdlabError extends Error {
|
|
9
|
+
/**
|
|
10
|
+
* @param {string} code — error code (e.g. INPUT_MISSING_FLAG)
|
|
11
|
+
* @param {string} message — human-readable description
|
|
12
|
+
* @param {Object} options
|
|
13
|
+
* @param {string} [options.hint] — actionable fix suggestion
|
|
14
|
+
* @param {Error} [options.cause] — original error
|
|
15
|
+
* @param {number} [options.exitCode] — 1 (user) or 2 (runtime)
|
|
16
|
+
*/
|
|
17
|
+
constructor(code, message, { hint, cause, exitCode } = {}) {
|
|
18
|
+
super(message);
|
|
19
|
+
this.name = 'SdlabError';
|
|
20
|
+
this.code = code;
|
|
21
|
+
this.hint = hint || null;
|
|
22
|
+
this.cause = cause || null;
|
|
23
|
+
this.exitCode = exitCode || 1;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
toJSON() {
|
|
27
|
+
return {
|
|
28
|
+
code: this.code,
|
|
29
|
+
message: this.message,
|
|
30
|
+
hint: this.hint,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// ─── Convenience constructors ──────────────────────────────────────
|
|
36
|
+
|
|
37
|
+
export function inputError(code, message, hint) {
|
|
38
|
+
return new SdlabError(code, message, { hint, exitCode: 1 });
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function runtimeError(code, message, hint, cause) {
|
|
42
|
+
return new SdlabError(code, message, { hint, cause, exitCode: 2 });
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// ─── CLI error handler ─────────────────────────────────────────────
|
|
46
|
+
|
|
47
|
+
let debugMode = false;
|
|
48
|
+
|
|
49
|
+
export function enableDebug() {
|
|
50
|
+
debugMode = true;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function isDebug() {
|
|
54
|
+
return debugMode;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Handle an error at the CLI boundary.
|
|
59
|
+
* Shows structured output by default, stack trace with --debug.
|
|
60
|
+
*/
|
|
61
|
+
export function handleCliError(err) {
|
|
62
|
+
if (err instanceof SdlabError) {
|
|
63
|
+
console.error(`\x1b[31mError [${err.code}]:\x1b[0m ${err.message}`);
|
|
64
|
+
if (err.hint) {
|
|
65
|
+
console.error(`\x1b[33mHint:\x1b[0m ${err.hint}`);
|
|
66
|
+
}
|
|
67
|
+
if (debugMode && err.stack) {
|
|
68
|
+
console.error('\n' + err.stack);
|
|
69
|
+
}
|
|
70
|
+
if (debugMode && err.cause) {
|
|
71
|
+
console.error('\nCaused by:', err.cause);
|
|
72
|
+
}
|
|
73
|
+
process.exit(err.exitCode);
|
|
74
|
+
} else {
|
|
75
|
+
// Unstructured error — runtime failure
|
|
76
|
+
console.error(`\x1b[31mError:\x1b[0m ${err.message || err}`);
|
|
77
|
+
if (debugMode && err.stack) {
|
|
78
|
+
console.error('\n' + err.stack);
|
|
79
|
+
}
|
|
80
|
+
process.exit(2);
|
|
81
|
+
}
|
|
82
|
+
}
|
package/lib/log.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logging levels for CLI output.
|
|
3
|
+
*
|
|
4
|
+
* Levels: quiet < normal < verbose
|
|
5
|
+
* Set via --quiet or --verbose flags (parsed by caller).
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
let level = 'normal'; // quiet | normal | verbose
|
|
9
|
+
|
|
10
|
+
export function setLogLevel(l) {
|
|
11
|
+
level = l;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function getLogLevel() {
|
|
15
|
+
return level;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Always shown (errors, final results). */
|
|
19
|
+
export function log(...args) {
|
|
20
|
+
console.log(...args);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Shown at normal and verbose levels (progress, summaries). */
|
|
24
|
+
export function info(...args) {
|
|
25
|
+
if (level !== 'quiet') console.log(...args);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Shown only at verbose level (per-record details, debug context). */
|
|
29
|
+
export function verbose(...args) {
|
|
30
|
+
if (level === 'verbose') console.log(...args);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Always shown on stderr. */
|
|
34
|
+
export function warn(...args) {
|
|
35
|
+
console.error('\x1b[33m⚠\x1b[0m', ...args);
|
|
36
|
+
}
|
package/lib/snapshot.js
CHANGED
|
@@ -53,7 +53,7 @@ export function computeConfigFingerprint(projectRoot) {
|
|
|
53
53
|
* @param {Object} profile — selection profile
|
|
54
54
|
* @returns {Promise<{snapshotId: string, included: number, excluded: number}>}
|
|
55
55
|
*/
|
|
56
|
-
export async function createSnapshot(projectRoot, profile) {
|
|
56
|
+
export async function createSnapshot(projectRoot, profile, options = {}) {
|
|
57
57
|
const config = loadProjectConfig(projectRoot);
|
|
58
58
|
const recordsDir = join(projectRoot, 'records');
|
|
59
59
|
const snapshotId = generateSnapshotId();
|
|
@@ -104,7 +104,10 @@ export async function createSnapshot(projectRoot, profile) {
|
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
// Write snapshot files
|
|
107
|
+
// Write snapshot files (skip if dry run)
|
|
108
|
+
if (options.dryRun) {
|
|
109
|
+
return { snapshotId, included: included.length, excluded: excluded.length };
|
|
110
|
+
}
|
|
108
111
|
await mkdir(snapshotDir, { recursive: true });
|
|
109
112
|
|
|
110
113
|
const manifest = {
|
package/lib/split.js
CHANGED
|
@@ -127,7 +127,7 @@ function buildAncestorMap(records) {
|
|
|
127
127
|
* @param {Object} profile — split profile (from loadSplitProfile)
|
|
128
128
|
* @returns {Promise<{splitId: string, train: number, val: number, test: number}>}
|
|
129
129
|
*/
|
|
130
|
-
export async function createSplit(projectRoot, snapshotId, profile) {
|
|
130
|
+
export async function createSplit(projectRoot, snapshotId, profile, options = {}) {
|
|
131
131
|
const config = loadProjectConfig(projectRoot);
|
|
132
132
|
const snapshot = await loadSnapshot(projectRoot, snapshotId);
|
|
133
133
|
const includedEntries = await loadSnapshotIncluded(projectRoot, snapshotId);
|
|
@@ -242,8 +242,11 @@ export async function createSplit(projectRoot, snapshotId, profile) {
|
|
|
242
242
|
records, trainRecords, valRecords, testRecords, config.lanes
|
|
243
243
|
);
|
|
244
244
|
|
|
245
|
-
// Write output
|
|
245
|
+
// Write output (skip if dry run)
|
|
246
246
|
const splitId = generateSplitId();
|
|
247
|
+
if (options.dryRun) {
|
|
248
|
+
return { splitId, train: trainRecords.length, val: valRecords.length, test: testRecords.length };
|
|
249
|
+
}
|
|
247
250
|
const splitDir = join(projectRoot, 'splits', splitId);
|
|
248
251
|
await mkdir(splitDir, { recursive: true });
|
|
249
252
|
|
package/package.json
CHANGED
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mcptoolshop/style-dataset-lab",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.0",
|
|
4
4
|
"description": "Visual canon and dataset production pipeline — snapshots, splits, versioned exports, eval packs, and canon-bound curation for any image production domain",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"sdlab": "./bin/sdlab.js"
|
|
8
8
|
},
|
|
9
9
|
"scripts": {
|
|
10
|
+
"verify": "node bin/sdlab.js --help && node bin/sdlab.js project doctor --project star-freight",
|
|
10
11
|
"generate": "node scripts/generate.js",
|
|
11
12
|
"curate": "node scripts/curate.js",
|
|
12
13
|
"compare": "node scripts/compare.js",
|
|
13
14
|
"canon-bind": "node scripts/canon-bind.js",
|
|
14
15
|
"painterly": "node scripts/painterly.js",
|
|
15
|
-
"generate:identity": "node scripts/generate-identity.js"
|
|
16
|
-
"export": "echo 'Usage: repo-dataset visual generate ./projects/<project> --format trl'",
|
|
17
|
-
"inspect": "echo 'Usage: repo-dataset visual inspect ./projects/<project>'"
|
|
16
|
+
"generate:identity": "node scripts/generate-identity.js"
|
|
18
17
|
},
|
|
19
18
|
"files": [
|
|
20
19
|
"bin/",
|
package/scripts/snapshot.js
CHANGED
|
@@ -32,39 +32,43 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
32
32
|
const profileIdx = args.indexOf('--profile');
|
|
33
33
|
const profileId = profileIdx >= 0 ? args[profileIdx + 1] : null;
|
|
34
34
|
const profile = loadSelectionProfile(projectRoot, profileId);
|
|
35
|
+
const dryRun = argv.includes('--dry-run');
|
|
35
36
|
|
|
36
37
|
console.log(`\x1b[1msdlab snapshot create\x1b[0m — ${projectName}`);
|
|
37
38
|
console.log(` Profile: ${profileId || 'training-default (built-in)'}`);
|
|
39
|
+
if (dryRun) console.log(' Mode: DRY RUN (no files written)');
|
|
38
40
|
console.log('');
|
|
39
41
|
|
|
40
|
-
const result = await createSnapshot(projectRoot, profile);
|
|
42
|
+
const result = await createSnapshot(projectRoot, profile, { dryRun });
|
|
41
43
|
|
|
42
44
|
console.log(` \x1b[32m✓\x1b[0m Snapshot: ${result.snapshotId}`);
|
|
43
45
|
console.log(` Included: ${result.included}`);
|
|
44
46
|
console.log(` Excluded: ${result.excluded}`);
|
|
45
47
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
48
|
+
if (!dryRun) {
|
|
49
|
+
// Read and display summary
|
|
50
|
+
const summary = JSON.parse(
|
|
51
|
+
await readFile(join(projectRoot, 'snapshots', result.snapshotId, 'summary.json'), 'utf-8')
|
|
52
|
+
);
|
|
53
|
+
console.log('\n Lane distribution:');
|
|
54
|
+
for (const [lane, count] of Object.entries(summary.lane_distribution).sort((a, b) => b[1] - a[1])) {
|
|
55
|
+
console.log(` ${lane}: ${count}`);
|
|
56
|
+
}
|
|
57
|
+
console.log('\n Faction distribution:');
|
|
58
|
+
for (const [f, c] of Object.entries(summary.faction_distribution).sort((a, b) => b[1] - a[1])) {
|
|
59
|
+
console.log(` ${f}: ${c}`);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Read and categorize exclusions
|
|
63
|
+
const excludedRaw = await readFile(
|
|
64
|
+
join(projectRoot, 'snapshots', result.snapshotId, 'excluded.jsonl'), 'utf-8'
|
|
65
|
+
);
|
|
66
|
+
const excluded = excludedRaw.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
|
|
67
|
+
const categories = categorizeExclusions(excluded);
|
|
68
|
+
console.log('\n Exclusion reasons:');
|
|
69
|
+
for (const [cat, count] of Object.entries(categories)) {
|
|
70
|
+
if (count > 0) console.log(` ${cat}: ${count}`);
|
|
71
|
+
}
|
|
68
72
|
}
|
|
69
73
|
|
|
70
74
|
} else if (subcommand === 'list') {
|
package/scripts/split.js
CHANGED
|
@@ -42,6 +42,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
42
42
|
const profileIdx = args.indexOf('--profile');
|
|
43
43
|
const profileId = profileIdx >= 0 ? args[profileIdx + 1] : null;
|
|
44
44
|
const profile = loadSplitProfile(projectRoot, profileId);
|
|
45
|
+
const dryRun = argv.includes('--dry-run');
|
|
45
46
|
|
|
46
47
|
console.log(`\x1b[1msdlab split build\x1b[0m — ${projectName}`);
|
|
47
48
|
console.log(` Snapshot: ${snapshotId}`);
|
|
@@ -49,9 +50,10 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
49
50
|
console.log(` Strategy: ${profile.strategy}`);
|
|
50
51
|
console.log(` Ratios: ${profile.train_ratio}/${profile.val_ratio}/${profile.test_ratio}`);
|
|
51
52
|
console.log(` Seed: ${profile.seed}`);
|
|
53
|
+
if (dryRun) console.log(' Mode: DRY RUN (no files written)');
|
|
52
54
|
console.log('');
|
|
53
55
|
|
|
54
|
-
const result = await createSplit(projectRoot, snapshotId, profile);
|
|
56
|
+
const result = await createSplit(projectRoot, snapshotId, profile, { dryRun });
|
|
55
57
|
|
|
56
58
|
console.log(` \x1b[32m✓\x1b[0m Split: ${result.splitId}`);
|
|
57
59
|
console.log(` Train: ${result.train}`);
|