@mcptoolshop/style-dataset-lab 2.3.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +178 -0
- package/CODE_OF_CONDUCT.md +9 -0
- package/CONTRIBUTING.md +71 -0
- package/README.md +82 -9
- package/SECURITY.md +6 -5
- package/bin/sdlab.js +359 -11
- package/lib/adapters/comfyui-runner.js +170 -0
- package/lib/adapters/comfyui-workflows.js +163 -0
- package/lib/adapters/diffusers-lora.js +7 -2
- package/lib/adapters/generic-image-caption.js +6 -1
- package/lib/args.js +200 -13
- package/lib/batch-compiler.js +122 -0
- package/lib/batch-modes.js +111 -0
- package/lib/batch-runs.js +237 -0
- package/lib/batch-sheet-render.js +209 -0
- package/lib/brief-compiler.js +625 -0
- package/lib/brief-render.js +205 -0
- package/lib/card.js +11 -4
- package/lib/comfyui.js +100 -6
- package/lib/config.js +68 -3
- package/lib/critique-engine.js +413 -0
- package/lib/critique-render.js +171 -0
- package/lib/deprecation.js +1 -1
- package/lib/eval-pack.js +94 -4
- package/lib/eval-runs.js +42 -3
- package/lib/export.js +144 -34
- package/lib/generated-provenance.js +87 -0
- package/lib/implementation-packs.js +15 -1
- package/lib/log.js +41 -4
- package/lib/paths.js +72 -3
- package/lib/refine-briefs.js +336 -0
- package/lib/reingest-selected.js +184 -0
- package/lib/reingest.js +15 -1
- package/lib/run-summary.js +132 -0
- package/lib/runtime-runs.js +218 -0
- package/lib/selections.js +375 -0
- package/lib/snapshot.js +100 -7
- package/lib/split.js +175 -17
- package/lib/training-manifests.js +17 -3
- package/lib/training-packages.js +28 -14
- package/lib/workflow-profiles.js +161 -0
- package/package.json +13 -4
- package/scripts/batch-generate.js +176 -0
- package/scripts/batch-sheet.js +46 -0
- package/scripts/batch-show.js +86 -0
- package/scripts/brief-compile.js +134 -0
- package/scripts/brief-show.js +53 -0
- package/scripts/canon-bind.js +7 -9
- package/scripts/card.js +22 -25
- package/scripts/compare.js +38 -33
- package/scripts/critique-show.js +50 -0
- package/scripts/critique.js +70 -0
- package/scripts/curate.js +91 -46
- package/scripts/doctor.js +172 -6
- package/scripts/eligibility.js +17 -14
- package/scripts/eval-pack.js +13 -11
- package/scripts/eval-run.js +30 -29
- package/scripts/export.js +27 -31
- package/scripts/generate-controlnet.js +31 -17
- package/scripts/generate-identity.js +37 -20
- package/scripts/generate-ipadapter.js +22 -16
- package/scripts/generate.js +68 -13
- package/scripts/implementation-pack.js +19 -14
- package/scripts/init.js +137 -17
- package/scripts/painterly.js +8 -10
- package/scripts/refine.js +119 -0
- package/scripts/reingest-selected.js +78 -0
- package/scripts/reingest.js +37 -18
- package/scripts/run-generate.js +76 -0
- package/scripts/run-list.js +58 -0
- package/scripts/run-show.js +63 -0
- package/scripts/select.js +119 -0
- package/scripts/selection-show.js +88 -0
- package/scripts/snapshot.js +22 -24
- package/scripts/split.js +26 -31
- package/scripts/training-manifest.js +29 -24
- package/scripts/training-package.js +24 -20
- package/scripts/training-profile.js +13 -8
- package/scripts/workflow-list.js +61 -0
- package/scripts/workflow-show.js +97 -0
- package/templates/domains/architecture/inputs/prompts/example-wave.json +32 -0
- package/templates/domains/architecture/terminology.json +10 -8
- package/templates/domains/architecture/workflows/profiles/environment-moodboard.json +62 -0
- package/templates/domains/architecture/workflows/profiles/interior-variant-pack.json +60 -0
- package/templates/domains/character-design/inputs/prompts/example-wave.json +32 -0
- package/templates/domains/character-design/project.json +4 -2
- package/templates/domains/character-design/workflows/profiles/character-portrait-set.json +61 -0
- package/templates/domains/character-design/workflows/profiles/expression-sheet.json +59 -0
- package/templates/domains/creature-design/inputs/prompts/example-wave.json +32 -0
- package/templates/domains/creature-design/project.json +4 -2
- package/templates/domains/creature-design/terminology.json +1 -0
- package/templates/domains/creature-design/workflows/profiles/creature-portrait-set.json +61 -0
- package/templates/domains/creature-design/workflows/profiles/silhouette-exploration.json +58 -0
- package/templates/domains/game-art/inputs/prompts/example-wave.json +37 -0
- package/templates/domains/game-art/project.json +4 -2
- package/templates/domains/game-art/workflows/profiles/character-portrait-set.json +62 -0
- package/templates/domains/game-art/workflows/profiles/environment-moodboard.json +61 -0
- package/templates/domains/vehicle-mech/inputs/prompts/example-wave.json +37 -0
- package/templates/domains/vehicle-mech/project.json +4 -2
- package/templates/domains/vehicle-mech/workflows/profiles/variant-pack.json +60 -0
- package/templates/domains/vehicle-mech/workflows/profiles/vehicle-silhouette-pack.json +59 -0
- package/scripts/bulk-curate-wave2-5.js +0 -138
- package/scripts/bulk-curate-waves11-18.js +0 -413
- package/scripts/curate-wave25.js +0 -209
- package/scripts/migrate-records.js +0 -141
- package/templates/inputs/prompts/example-wave.json +0 -27
package/lib/eval-pack.js
CHANGED
|
@@ -16,6 +16,8 @@ import { existsSync } from 'node:fs';
|
|
|
16
16
|
import { randomBytes } from 'node:crypto';
|
|
17
17
|
import { loadProjectConfig, detectLane, detectGroup } from './config.js';
|
|
18
18
|
import { loadRecord } from './records.js';
|
|
19
|
+
import { computeConfigFingerprint, SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
|
|
20
|
+
import { inputError } from './errors.js';
|
|
19
21
|
|
|
20
22
|
/**
|
|
21
23
|
* Generate an eval-pack ID: eval-YYYYMMDD-HHMMSS-XXXX
|
|
@@ -44,11 +46,18 @@ export async function buildEvalPack(projectRoot, options = {}) {
|
|
|
44
46
|
const maxDrift = options.maxDrift || 20;
|
|
45
47
|
const maxContinuity = options.maxContinuity || 10;
|
|
46
48
|
|
|
47
|
-
// Load all records
|
|
49
|
+
// Load all records (DB-003-adjacent: skip unparseable files rather than
|
|
50
|
+
// aborting the entire eval-pack build — same graceful-degradation shape
|
|
51
|
+
// snapshot.js uses).
|
|
48
52
|
const records = [];
|
|
49
53
|
for (const file of files) {
|
|
50
|
-
|
|
51
|
-
|
|
54
|
+
try {
|
|
55
|
+
const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
|
|
56
|
+
records.push(record);
|
|
57
|
+
} catch {
|
|
58
|
+
// Skip unreadable records; eval pack is a best-effort selection and
|
|
59
|
+
// one bad file must not kill the whole build.
|
|
60
|
+
}
|
|
52
61
|
}
|
|
53
62
|
|
|
54
63
|
// Task 1: Lane coverage
|
|
@@ -66,19 +75,60 @@ export async function buildEvalPack(projectRoot, options = {}) {
|
|
|
66
75
|
// Write eval pack
|
|
67
76
|
const evalId = generateEvalId();
|
|
68
77
|
const evalDir = join(projectRoot, 'eval-packs', evalId);
|
|
78
|
+
// D-005: refuse to silently overwrite on ID collision.
|
|
79
|
+
if (existsSync(evalDir)) {
|
|
80
|
+
throw inputError(
|
|
81
|
+
'EVAL_PACK_ID_COLLISION',
|
|
82
|
+
`Eval pack directory already exists: ${evalDir}`,
|
|
83
|
+
'Retry to generate a fresh eval pack ID.',
|
|
84
|
+
);
|
|
85
|
+
}
|
|
69
86
|
await mkdir(evalDir, { recursive: true });
|
|
70
87
|
|
|
88
|
+
// DB-014: compute `records_available` so the manifest reports sampled vs.
|
|
89
|
+
// available. Operators can see when they're leaving records on the table.
|
|
90
|
+
const laneAvailable = countLaneCoverageCandidates(records, config);
|
|
91
|
+
const driftAvailable = countForbiddenDriftCandidates(records);
|
|
92
|
+
const anchorAvailable = countAnchorGoldCandidates(records, config);
|
|
93
|
+
const continuityAvailable = countSubjectContinuityCandidates(records);
|
|
94
|
+
|
|
71
95
|
const manifest = {
|
|
96
|
+
// DB-001 / DB-006: stamp manifest schema version
|
|
97
|
+
schema_version: SCHEMA_VERSION,
|
|
72
98
|
eval_id: evalId,
|
|
73
99
|
created_at: new Date().toISOString(),
|
|
74
100
|
created_by: 'sdlab-eval-v1',
|
|
75
101
|
project: config.meta.name,
|
|
102
|
+
// D-017: capture config fingerprint so eval verdicts can detect canon drift.
|
|
103
|
+
config_fingerprint: computeConfigFingerprint(projectRoot),
|
|
76
104
|
task_counts: {
|
|
77
105
|
lane_coverage: laneCoverage.length,
|
|
78
106
|
forbidden_drift: forbiddenDrift.length,
|
|
79
107
|
anchor_gold: anchorGold.length,
|
|
80
108
|
subject_continuity: subjectContinuity.length,
|
|
81
109
|
},
|
|
110
|
+
// DB-014: coverage — sampled vs. available per task
|
|
111
|
+
records_sampled: {
|
|
112
|
+
lane_coverage: laneCoverage.length,
|
|
113
|
+
forbidden_drift: forbiddenDrift.length,
|
|
114
|
+
anchor_gold: anchorGold.length,
|
|
115
|
+
subject_continuity: subjectContinuity.length,
|
|
116
|
+
total: laneCoverage.length + forbiddenDrift.length +
|
|
117
|
+
anchorGold.length + subjectContinuity.length,
|
|
118
|
+
},
|
|
119
|
+
records_available: {
|
|
120
|
+
lane_coverage: laneAvailable,
|
|
121
|
+
forbidden_drift: driftAvailable,
|
|
122
|
+
anchor_gold: anchorAvailable,
|
|
123
|
+
subject_continuity: continuityAvailable,
|
|
124
|
+
total: laneAvailable + driftAvailable + anchorAvailable + continuityAvailable,
|
|
125
|
+
},
|
|
126
|
+
caps: {
|
|
127
|
+
max_per_lane: maxPerLane,
|
|
128
|
+
max_per_faction: maxPerFaction,
|
|
129
|
+
max_drift: maxDrift,
|
|
130
|
+
max_continuity: maxContinuity,
|
|
131
|
+
},
|
|
82
132
|
total_records: laneCoverage.length + forbiddenDrift.length +
|
|
83
133
|
anchorGold.length + subjectContinuity.length,
|
|
84
134
|
};
|
|
@@ -95,6 +145,44 @@ export async function buildEvalPack(projectRoot, options = {}) {
|
|
|
95
145
|
};
|
|
96
146
|
}
|
|
97
147
|
|
|
148
|
+
/** DB-014: count approved+canon-bound records (the lane-coverage candidate pool). */
|
|
149
|
+
function countLaneCoverageCandidates(records, _config) {
|
|
150
|
+
let n = 0;
|
|
151
|
+
for (const r of records) {
|
|
152
|
+
if (r.judgment?.status === 'approved' && r.canon?.assertions?.length) n++;
|
|
153
|
+
}
|
|
154
|
+
return n;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** DB-014: count records that qualify as forbidden-drift candidates. */
|
|
158
|
+
function countForbiddenDriftCandidates(records) {
|
|
159
|
+
let n = 0;
|
|
160
|
+
for (const r of records) {
|
|
161
|
+
const status = r.judgment?.status;
|
|
162
|
+
if (status !== 'rejected' && status !== 'borderline') continue;
|
|
163
|
+
const failureModes = r.judgment?.failure_modes || [];
|
|
164
|
+
const failedAssertions = (r.canon?.assertions || [])
|
|
165
|
+
.filter(a => a.verdict === 'fail').length;
|
|
166
|
+
if (failureModes.length > 0 || failedAssertions > 0 || r.judgment?.explanation) n++;
|
|
167
|
+
}
|
|
168
|
+
return n;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** DB-014: count approved+canon-bound records (also anchor-gold pool). */
|
|
172
|
+
function countAnchorGoldCandidates(records, _config) {
|
|
173
|
+
return countLaneCoverageCandidates(records, _config);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** DB-014: count named subjects (subject-continuity pool). */
|
|
177
|
+
function countSubjectContinuityCandidates(records) {
|
|
178
|
+
const subjects = new Set();
|
|
179
|
+
for (const r of records) {
|
|
180
|
+
const s = r.identity?.subject_name;
|
|
181
|
+
if (s) subjects.add(s);
|
|
182
|
+
}
|
|
183
|
+
return subjects.size;
|
|
184
|
+
}
|
|
185
|
+
|
|
98
186
|
/**
|
|
99
187
|
* Task 1: Lane coverage — best approved record per lane.
|
|
100
188
|
*/
|
|
@@ -279,5 +367,7 @@ export async function loadEvalPack(projectRoot, evalId) {
|
|
|
279
367
|
if (!existsSync(path)) {
|
|
280
368
|
throw new Error(`Eval pack "${evalId}" not found at ${path}`);
|
|
281
369
|
}
|
|
282
|
-
|
|
370
|
+
const manifest = JSON.parse(await readFile(path, 'utf-8'));
|
|
371
|
+
checkManifestVersion(manifest, 'eval-pack');
|
|
372
|
+
return manifest;
|
|
283
373
|
}
|
package/lib/eval-runs.js
CHANGED
|
@@ -13,6 +13,11 @@ import { randomBytes } from 'node:crypto';
|
|
|
13
13
|
import { loadTrainingManifest } from './training-manifests.js';
|
|
14
14
|
import { loadEvalPack } from './eval-pack.js';
|
|
15
15
|
import { loadProjectConfig, detectLane, detectGroup } from './config.js';
|
|
16
|
+
import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
|
|
17
|
+
|
|
18
|
+
// DB-008: cap sample record_ids per failure bucket so scorecards stay small
|
|
19
|
+
// but the operator has enough detail to start investigation.
|
|
20
|
+
const SAMPLE_LIMIT = 5;
|
|
16
21
|
|
|
17
22
|
/**
|
|
18
23
|
* Generate an eval run ID: er-YYYYMMDD-HHMMSS-XXXX
|
|
@@ -42,6 +47,8 @@ export async function createEvalRun(projectRoot, manifestId, evalPackId) {
|
|
|
42
47
|
await mkdir(evalRunDir, { recursive: true });
|
|
43
48
|
|
|
44
49
|
const runManifest = {
|
|
50
|
+
// DB-001 / DB-006: stamp manifest schema version
|
|
51
|
+
schema_version: SCHEMA_VERSION,
|
|
45
52
|
eval_run_id: evalRunId,
|
|
46
53
|
created_at: new Date().toISOString(),
|
|
47
54
|
created_by: 'sdlab-eval-run-v1',
|
|
@@ -148,16 +155,25 @@ export async function scoreEvalRun(projectRoot, evalRunId, outputsPath) {
|
|
|
148
155
|
function scoreLaneCoverage(tasks, outputMap) {
|
|
149
156
|
let covered = 0;
|
|
150
157
|
const lanes = new Set();
|
|
158
|
+
// DB-008: collect record_ids that missed coverage so the operator can drill
|
|
159
|
+
// down from "covered: 7/10" to "which 3 weren't covered".
|
|
160
|
+
const missedSamples = [];
|
|
151
161
|
for (const task of tasks) {
|
|
152
162
|
lanes.add(task.lane);
|
|
153
163
|
const output = outputMap.get(task.record_id);
|
|
154
|
-
if (output?.accept)
|
|
164
|
+
if (output?.accept) {
|
|
165
|
+
covered++;
|
|
166
|
+
} else if (missedSamples.length < SAMPLE_LIMIT) {
|
|
167
|
+
missedSamples.push(task.record_id);
|
|
168
|
+
}
|
|
155
169
|
}
|
|
156
170
|
const score = tasks.length > 0 ? covered / tasks.length : 0;
|
|
157
171
|
return {
|
|
158
172
|
task: 'lane_coverage',
|
|
159
173
|
total: tasks.length,
|
|
160
174
|
covered,
|
|
175
|
+
missed: tasks.length - covered,
|
|
176
|
+
missed_sample_record_ids: missedSamples,
|
|
161
177
|
lanes_represented: lanes.size,
|
|
162
178
|
score: +score.toFixed(3),
|
|
163
179
|
passed: score >= 0.7,
|
|
@@ -168,12 +184,19 @@ function scoreForbiddenDrift(tasks, outputMap) {
|
|
|
168
184
|
// For forbidden drift, we check that rejected/borderline records are NOT accepted
|
|
169
185
|
let correctRejections = 0;
|
|
170
186
|
let falseAcceptances = 0;
|
|
187
|
+
// DB-008: name the records that were wrongly accepted. Without this, a
|
|
188
|
+
// scorecard that says "false_acceptances: 3" forces the operator to diff
|
|
189
|
+
// outputs.jsonl against the eval pack manually.
|
|
190
|
+
const falseAcceptanceSamples = [];
|
|
171
191
|
for (const task of tasks) {
|
|
172
192
|
const output = outputMap.get(task.record_id);
|
|
173
193
|
if (!output || !output.accept) {
|
|
174
194
|
correctRejections++;
|
|
175
195
|
} else {
|
|
176
196
|
falseAcceptances++;
|
|
197
|
+
if (falseAcceptanceSamples.length < SAMPLE_LIMIT) {
|
|
198
|
+
falseAcceptanceSamples.push(task.record_id);
|
|
199
|
+
}
|
|
177
200
|
}
|
|
178
201
|
}
|
|
179
202
|
const score = tasks.length > 0 ? correctRejections / tasks.length : 1;
|
|
@@ -182,6 +205,7 @@ function scoreForbiddenDrift(tasks, outputMap) {
|
|
|
182
205
|
total: tasks.length,
|
|
183
206
|
correct_rejections: correctRejections,
|
|
184
207
|
false_acceptances: falseAcceptances,
|
|
208
|
+
false_acceptance_sample_record_ids: falseAcceptanceSamples,
|
|
185
209
|
score: +score.toFixed(3),
|
|
186
210
|
passed: falseAcceptances === 0,
|
|
187
211
|
};
|
|
@@ -189,15 +213,23 @@ function scoreForbiddenDrift(tasks, outputMap) {
|
|
|
189
213
|
|
|
190
214
|
function scoreAnchorGold(tasks, outputMap) {
|
|
191
215
|
let matched = 0;
|
|
216
|
+
// DB-008: which anchor records failed to match?
|
|
217
|
+
const missedSamples = [];
|
|
192
218
|
for (const task of tasks) {
|
|
193
219
|
const output = outputMap.get(task.record_id);
|
|
194
|
-
if (output?.accept)
|
|
220
|
+
if (output?.accept) {
|
|
221
|
+
matched++;
|
|
222
|
+
} else if (missedSamples.length < SAMPLE_LIMIT) {
|
|
223
|
+
missedSamples.push(task.record_id);
|
|
224
|
+
}
|
|
195
225
|
}
|
|
196
226
|
const score = tasks.length > 0 ? matched / tasks.length : 0;
|
|
197
227
|
return {
|
|
198
228
|
task: 'anchor_gold',
|
|
199
229
|
total: tasks.length,
|
|
200
230
|
matched,
|
|
231
|
+
missed: tasks.length - matched,
|
|
232
|
+
missed_sample_record_ids: missedSamples,
|
|
201
233
|
score: +score.toFixed(3),
|
|
202
234
|
passed: score >= 0.8,
|
|
203
235
|
};
|
|
@@ -205,16 +237,23 @@ function scoreAnchorGold(tasks, outputMap) {
|
|
|
205
237
|
|
|
206
238
|
function scoreSubjectContinuity(tasks, outputMap) {
|
|
207
239
|
let subjectsWithCoverage = 0;
|
|
240
|
+
// DB-008: which named subjects lost continuity coverage?
|
|
241
|
+
const missedSamples = [];
|
|
208
242
|
for (const task of tasks) {
|
|
209
243
|
const recordIds = task.record_ids || [];
|
|
210
244
|
const accepted = recordIds.filter(id => outputMap.get(id)?.accept).length;
|
|
211
|
-
if (accepted >= 2)
|
|
245
|
+
if (accepted >= 2) {
|
|
246
|
+
subjectsWithCoverage++;
|
|
247
|
+
} else if (missedSamples.length < SAMPLE_LIMIT) {
|
|
248
|
+
missedSamples.push(task.subject_name || recordIds[0] || 'unknown');
|
|
249
|
+
}
|
|
212
250
|
}
|
|
213
251
|
const score = tasks.length > 0 ? subjectsWithCoverage / tasks.length : 0;
|
|
214
252
|
return {
|
|
215
253
|
task: 'subject_continuity',
|
|
216
254
|
total_subjects: tasks.length,
|
|
217
255
|
subjects_with_coverage: subjectsWithCoverage,
|
|
256
|
+
missed_subject_samples: missedSamples,
|
|
218
257
|
score: +score.toFixed(3),
|
|
219
258
|
passed: score >= 0.6,
|
|
220
259
|
};
|
package/lib/export.js
CHANGED
|
@@ -6,14 +6,17 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { readFile, writeFile, mkdir, readdir, symlink, copyFile, stat } from 'node:fs/promises';
|
|
9
|
-
import { join, relative, resolve } from 'node:path';
|
|
9
|
+
import { join, relative, resolve, sep, basename, isAbsolute } from 'node:path';
|
|
10
10
|
import { existsSync } from 'node:fs';
|
|
11
|
+
import { platform } from 'node:os';
|
|
11
12
|
import { createHash, randomBytes } from 'node:crypto';
|
|
12
13
|
import { loadProjectConfig, loadExportProfile, detectLane, detectGroup } from './config.js';
|
|
13
|
-
import { loadSnapshot, loadSnapshotIncluded, computeConfigFingerprint } from './snapshot.js';
|
|
14
|
+
import { loadSnapshot, loadSnapshotIncluded, computeConfigFingerprint, SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
|
|
14
15
|
import { loadSplit, loadSplitPartition } from './split.js';
|
|
15
16
|
import { loadRecord } from './records.js';
|
|
16
17
|
import { generateCard } from './card.js';
|
|
18
|
+
import { inputError } from './errors.js';
|
|
19
|
+
import { warn } from './log.js';
|
|
17
20
|
|
|
18
21
|
/**
|
|
19
22
|
* Generate an export ID: export-YYYYMMDD-HHMMSS-XXXX
|
|
@@ -54,6 +57,29 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
|
|
|
54
57
|
|
|
55
58
|
const exportId = generateExportId();
|
|
56
59
|
const exportDir = join(projectRoot, 'exports', exportId);
|
|
60
|
+
|
|
61
|
+
// D-005: existsSync guard prevents silent overwrite on ID collision
|
|
62
|
+
if (existsSync(exportDir)) {
|
|
63
|
+
throw inputError(
|
|
64
|
+
'EXPORT_ID_COLLISION',
|
|
65
|
+
`Export directory already exists: ${exportDir}`,
|
|
66
|
+
'Retry to generate a fresh export ID, or delete the existing directory if this was intentional.',
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// D-001: Verify fingerprint has not drifted since snapshot creation.
|
|
71
|
+
// We inherit the snapshot's fingerprint in the export manifest; if a caller
|
|
72
|
+
// has edited project config mid-flight we surface that loudly rather than
|
|
73
|
+
// silently embedding a new fingerprint that masks reproducibility loss.
|
|
74
|
+
const currentFingerprint = computeConfigFingerprint(projectRoot);
|
|
75
|
+
if (snapshot.config_fingerprint && currentFingerprint !== snapshot.config_fingerprint) {
|
|
76
|
+
throw inputError(
|
|
77
|
+
'FINGERPRINT_DRIFT',
|
|
78
|
+
`Project config fingerprint has changed since snapshot "${snapshotId}" was created.`,
|
|
79
|
+
`Snapshot fingerprint: ${snapshot.config_fingerprint.slice(0, 16)}..., current: ${currentFingerprint.slice(0, 16)}.... Create a fresh snapshot from the updated config, or revert the config edits.`,
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
|
|
57
83
|
await mkdir(exportDir, { recursive: true });
|
|
58
84
|
|
|
59
85
|
// Build metadata + collect image paths
|
|
@@ -61,6 +87,8 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
|
|
|
61
87
|
const imagePaths = [];
|
|
62
88
|
const checksumEntries = [];
|
|
63
89
|
|
|
90
|
+
const projectRootResolved = resolve(projectRoot);
|
|
91
|
+
|
|
64
92
|
for (const entry of included) {
|
|
65
93
|
const record = await loadRecord(recordsDir, entry.record_id);
|
|
66
94
|
if (!record) continue;
|
|
@@ -87,10 +115,33 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
|
|
|
87
115
|
|
|
88
116
|
// Track image for linking
|
|
89
117
|
if (profile.include_images && record.asset_path) {
|
|
118
|
+
// D-007: validate asset_path does not escape projectRoot, use
|
|
119
|
+
// platform-safe basename, and name output files by record.id
|
|
120
|
+
// (D-009) to avoid collisions on shared basenames.
|
|
121
|
+
if (isAbsolute(record.asset_path)) {
|
|
122
|
+
throw inputError(
|
|
123
|
+
'ASSET_PATH_INVALID',
|
|
124
|
+
`Record "${record.id}" has absolute asset_path: ${record.asset_path}`,
|
|
125
|
+
'asset_path must be relative to the project root.',
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
const resolvedSource = resolve(projectRootResolved, record.asset_path);
|
|
129
|
+
if (resolvedSource !== projectRootResolved &&
|
|
130
|
+
!resolvedSource.startsWith(projectRootResolved + sep)) {
|
|
131
|
+
throw inputError(
|
|
132
|
+
'ASSET_PATH_TRAVERSAL',
|
|
133
|
+
`Record "${record.id}" asset_path escapes projectRoot: ${record.asset_path}`,
|
|
134
|
+
'Reject path traversal segments (e.g. "../") in asset_path.',
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
const sourceBasename = basename(record.asset_path);
|
|
138
|
+
const extMatch = sourceBasename.match(/\.[^.]+$/);
|
|
139
|
+
const ext = extMatch ? extMatch[0] : '';
|
|
140
|
+
const safeFilename = `${record.id}${ext}`;
|
|
90
141
|
imagePaths.push({
|
|
91
142
|
record_id: record.id,
|
|
92
|
-
source:
|
|
93
|
-
filename:
|
|
143
|
+
source: resolvedSource,
|
|
144
|
+
filename: safeFilename,
|
|
94
145
|
});
|
|
95
146
|
}
|
|
96
147
|
}
|
|
@@ -100,27 +151,70 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
|
|
|
100
151
|
await writeFile(metadataPath, metadata.map(r => JSON.stringify(r)).join('\n') + '\n');
|
|
101
152
|
|
|
102
153
|
// Images directory
|
|
154
|
+
const warnings = [];
|
|
155
|
+
// DB-002: separate accounting for expected vs. actually placed vs. failed.
|
|
156
|
+
// Previously `counts.images` reported `sources_that_existed`, which hid
|
|
157
|
+
// per-file symlink/copy failures and made partial exports look complete.
|
|
158
|
+
let imagesExpected = 0;
|
|
159
|
+
let imagesActual = 0;
|
|
160
|
+
const imagesFailed = []; // [{record_id, reason}]
|
|
103
161
|
if (profile.include_images && imagePaths.length > 0) {
|
|
162
|
+
// D-008: Windows symlink semantics differ (require admin or dev mode);
|
|
163
|
+
// refuse symlink mode on Windows and require --copy for portability.
|
|
164
|
+
if (!options.copy && platform() === 'win32') {
|
|
165
|
+
throw inputError(
|
|
166
|
+
'SYMLINK_UNSUPPORTED_WINDOWS',
|
|
167
|
+
'Symlink mode is not supported on Windows export (target semantics differ and break on archive).',
|
|
168
|
+
'Re-run with --copy to produce a portable, self-contained export.',
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
if (!options.copy) {
|
|
172
|
+
warnings.push(
|
|
173
|
+
'Export uses relative symlinks — package is NOT portable across machines. ' +
|
|
174
|
+
'Use --copy for cross-machine transfer or archival.',
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
104
178
|
const imagesDir = join(exportDir, 'images');
|
|
105
179
|
await mkdir(imagesDir, { recursive: true });
|
|
106
180
|
|
|
107
|
-
|
|
181
|
+
imagesExpected = imagePaths.length;
|
|
108
182
|
for (const img of imagePaths) {
|
|
109
183
|
const dest = join(imagesDir, img.filename);
|
|
110
|
-
if (existsSync(img.source)) {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
184
|
+
if (!existsSync(img.source)) {
|
|
185
|
+
// DB-002: source-missing is a distinct failure mode from link/copy error.
|
|
186
|
+
imagesFailed.push({ record_id: img.record_id, reason: 'source_missing' });
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
if (options.copy) {
|
|
191
|
+
await copyFile(img.source, dest);
|
|
192
|
+
} else {
|
|
193
|
+
const relPath = relative(imagesDir, img.source);
|
|
194
|
+
await symlink(relPath, dest);
|
|
121
195
|
}
|
|
196
|
+
imagesActual++;
|
|
197
|
+
} catch (err) {
|
|
198
|
+
// DB-002: capture the specific placement failure for operator triage.
|
|
199
|
+
imagesFailed.push({
|
|
200
|
+
record_id: img.record_id,
|
|
201
|
+
reason: options.copy ? 'copy_failed' : 'link_failed',
|
|
202
|
+
detail: err.message,
|
|
203
|
+
});
|
|
122
204
|
}
|
|
123
205
|
}
|
|
206
|
+
|
|
207
|
+
// DB-002: surface partial placement failures loudly — silent `catch {}`
|
|
208
|
+
// previously hid this and made exports appear complete when 20% of
|
|
209
|
+
// images had actually been dropped.
|
|
210
|
+
if (imagesFailed.length > 0) {
|
|
211
|
+
const sample = imagesFailed.slice(0, 10).map(f => `${f.record_id} (${f.reason})`);
|
|
212
|
+
const msg =
|
|
213
|
+
`export: ${imagesFailed.length} of ${imagesExpected} images failed to place` +
|
|
214
|
+
(imagesFailed.length > 10 ? ` — first 10: ${sample.join(', ')}, ...` : ` — ${sample.join(', ')}`);
|
|
215
|
+
warn(msg);
|
|
216
|
+
warnings.push(msg);
|
|
217
|
+
}
|
|
124
218
|
}
|
|
125
219
|
|
|
126
220
|
// Splits
|
|
@@ -144,34 +238,35 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
|
|
|
144
238
|
await writeFile(join(exportDir, 'dataset-card.json'), JSON.stringify(json, null, 2) + '\n');
|
|
145
239
|
}
|
|
146
240
|
|
|
147
|
-
//
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
const allFiles = await collectFiles(exportDir);
|
|
151
|
-
for (const filePath of allFiles.sort()) {
|
|
152
|
-
const relPath = relative(exportDir, filePath);
|
|
153
|
-
if (relPath === 'checksums.txt') continue;
|
|
154
|
-
const hash = await sha256File(filePath);
|
|
155
|
-
checksumEntries.push(`SHA256 (${relPath}) = ${hash}`);
|
|
156
|
-
}
|
|
157
|
-
await writeFile(join(exportDir, 'checksums.txt'), checksumEntries.join('\n') + '\n');
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
// Manifest (written last — references everything)
|
|
241
|
+
// D-015: Write manifest + summary FIRST so they are covered by checksums.
|
|
242
|
+
// D-001: Inherit snapshot's config_fingerprint (we verified above it matches
|
|
243
|
+
// current state) — never recompute independently at export time.
|
|
161
244
|
const manifest = {
|
|
245
|
+
// DB-001 / DB-006: stamp manifest schema version
|
|
246
|
+
schema_version: SCHEMA_VERSION,
|
|
162
247
|
export_id: exportId,
|
|
163
248
|
created_at: new Date().toISOString(),
|
|
164
249
|
created_by: 'sdlab-export-v1',
|
|
165
250
|
project: config.meta.name,
|
|
166
251
|
snapshot_id: snapshotId,
|
|
167
252
|
split_id: splitId,
|
|
168
|
-
config_fingerprint:
|
|
253
|
+
config_fingerprint: snapshot.config_fingerprint,
|
|
169
254
|
profile,
|
|
255
|
+
warnings,
|
|
170
256
|
counts: {
|
|
171
257
|
records: metadata.length,
|
|
172
|
-
|
|
173
|
-
|
|
258
|
+
// DB-002: images count is the actual placed count (not source-exists).
|
|
259
|
+
// Keep top-level `images` for backward compat with older manifest readers
|
|
260
|
+
// that indexed it as the summary number, but add structured detail.
|
|
261
|
+
images: imagesActual,
|
|
262
|
+
images_expected: imagesExpected,
|
|
263
|
+
images_actual: imagesActual,
|
|
264
|
+
images_failed: imagesFailed.length,
|
|
265
|
+
checksums: 0, // updated after checksum pass below
|
|
174
266
|
},
|
|
267
|
+
// DB-002: record-level failure detail so operators can investigate
|
|
268
|
+
// without re-opening the directory to discover what's missing.
|
|
269
|
+
images_failed: imagesFailed,
|
|
175
270
|
};
|
|
176
271
|
await writeFile(join(exportDir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
|
|
177
272
|
|
|
@@ -188,9 +283,22 @@ export async function buildExport(projectRoot, snapshotId, splitId, options = {}
|
|
|
188
283
|
test: split.counts.test,
|
|
189
284
|
},
|
|
190
285
|
config_fingerprint: manifest.config_fingerprint,
|
|
286
|
+
warnings,
|
|
191
287
|
};
|
|
192
288
|
await writeFile(join(exportDir, 'summary.json'), JSON.stringify(summary, null, 2) + '\n');
|
|
193
289
|
|
|
290
|
+
// Checksums (computed AFTER manifest+summary so they are included)
|
|
291
|
+
if (profile.include_checksums) {
|
|
292
|
+
const allFiles = await collectFiles(exportDir);
|
|
293
|
+
for (const filePath of allFiles.sort()) {
|
|
294
|
+
const relPath = relative(exportDir, filePath);
|
|
295
|
+
if (relPath === 'checksums.txt') continue;
|
|
296
|
+
const hash = await sha256File(filePath);
|
|
297
|
+
checksumEntries.push(`SHA256 (${relPath}) = ${hash}`);
|
|
298
|
+
}
|
|
299
|
+
await writeFile(join(exportDir, 'checksums.txt'), checksumEntries.join('\n') + '\n');
|
|
300
|
+
}
|
|
301
|
+
|
|
194
302
|
return {
|
|
195
303
|
exportId,
|
|
196
304
|
recordCount: metadata.length,
|
|
@@ -255,5 +363,7 @@ export async function loadExport(projectRoot, exportId) {
|
|
|
255
363
|
if (!existsSync(path)) {
|
|
256
364
|
throw new Error(`Export "${exportId}" not found at ${path}`);
|
|
257
365
|
}
|
|
258
|
-
|
|
366
|
+
const manifest = JSON.parse(await readFile(path, 'utf-8'));
|
|
367
|
+
checkManifestVersion(manifest, 'export');
|
|
368
|
+
return manifest;
|
|
259
369
|
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generated provenance builder — constructs the provenance block
|
|
3
|
+
* attached to records created from selected run/batch outputs.
|
|
4
|
+
*
|
|
5
|
+
* Normalizes source context (run vs batch) into a single shape
|
|
6
|
+
* that traces back through brief → workflow → seed → config.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
10
|
+
import { join } from 'node:path';
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Build a generation provenance payload for a selected output.
|
|
15
|
+
*
|
|
16
|
+
* @param {Object} opts
|
|
17
|
+
* @param {'run'|'batch'} opts.sourceType
|
|
18
|
+
* @param {string} opts.sourceId — run_id or batch_id
|
|
19
|
+
* @param {Object} opts.runManifest — the run manifest containing this output
|
|
20
|
+
* @param {Object} [opts.batchManifest] — batch manifest if source is batch
|
|
21
|
+
* @param {Object} opts.item — the selection item (slot_or_output, filename, seed)
|
|
22
|
+
* @param {string} opts.projectRoot
|
|
23
|
+
* @returns {Object} generation_provenance block
|
|
24
|
+
*/
|
|
25
|
+
export function buildGeneratedProvenance({
|
|
26
|
+
sourceType,
|
|
27
|
+
sourceId,
|
|
28
|
+
runManifest,
|
|
29
|
+
batchManifest,
|
|
30
|
+
item,
|
|
31
|
+
projectRoot,
|
|
32
|
+
}) {
|
|
33
|
+
const prov = {
|
|
34
|
+
source_type: sourceType,
|
|
35
|
+
source_id: sourceId,
|
|
36
|
+
run_id: runManifest.run_id,
|
|
37
|
+
brief_id: runManifest.brief_id,
|
|
38
|
+
workflow_id: runManifest.workflow_template_id,
|
|
39
|
+
selected_filename: item.filename,
|
|
40
|
+
generated_at: runManifest.created_at,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
if (sourceType === 'batch' && batchManifest) {
|
|
44
|
+
prov.batch_id = batchManifest.batch_id;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Seed from the item or from the run manifest outputs
|
|
48
|
+
if (item.seed !== undefined) {
|
|
49
|
+
prov.seed = item.seed;
|
|
50
|
+
} else if (runManifest.outputs) {
|
|
51
|
+
const match = runManifest.outputs.find(o => o.filename === item.filename);
|
|
52
|
+
if (match?.seed !== undefined) prov.seed = match.seed;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Adapter target from run manifest
|
|
56
|
+
if (runManifest.adapter_target) {
|
|
57
|
+
prov.adapter_target = runManifest.adapter_target;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Subject from run brief or batch manifest
|
|
61
|
+
if (batchManifest?.subject_id) {
|
|
62
|
+
prov.subject_id = batchManifest.subject_id;
|
|
63
|
+
} else {
|
|
64
|
+
// Try to read subject from brief
|
|
65
|
+
const briefPath = join(projectRoot, 'briefs', `${runManifest.brief_id}.json`);
|
|
66
|
+
if (existsSync(briefPath)) {
|
|
67
|
+
try {
|
|
68
|
+
const brief = JSON.parse(readFileSync(briefPath, 'utf-8'));
|
|
69
|
+
if (brief.subject_id) prov.subject_id = brief.subject_id;
|
|
70
|
+
if (brief.parent_brief_id) prov.parent_brief_id = brief.parent_brief_id;
|
|
71
|
+
} catch { /* skip */ }
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Config fingerprint — hash the run manifest for reproducibility tracing
|
|
76
|
+
prov.config_fingerprint = createHash('sha256')
|
|
77
|
+
.update(JSON.stringify({
|
|
78
|
+
brief_id: runManifest.brief_id,
|
|
79
|
+
workflow_template_id: runManifest.workflow_template_id,
|
|
80
|
+
adapter_target: runManifest.adapter_target,
|
|
81
|
+
seed_plan: runManifest.seed_plan,
|
|
82
|
+
}))
|
|
83
|
+
.digest('hex')
|
|
84
|
+
.slice(0, 16);
|
|
85
|
+
|
|
86
|
+
return prov;
|
|
87
|
+
}
|
|
@@ -17,6 +17,8 @@ import { loadTrainingProfile } from './training-profiles.js';
|
|
|
17
17
|
import { loadProjectConfig, detectLane, detectGroup } from './config.js';
|
|
18
18
|
import { loadSplitPartition } from './split.js';
|
|
19
19
|
import { loadRecord } from './records.js';
|
|
20
|
+
import { inputError } from './errors.js';
|
|
21
|
+
import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
|
|
20
22
|
|
|
21
23
|
function generateImplId() {
|
|
22
24
|
const now = new Date();
|
|
@@ -138,9 +140,19 @@ export async function buildImplementationPack(projectRoot, manifestId) {
|
|
|
138
140
|
// Write implementation pack
|
|
139
141
|
const implId = generateImplId();
|
|
140
142
|
const implDir = join(projectRoot, 'training', 'implementations', implId);
|
|
143
|
+
// D-005: refuse to silently overwrite on ID collision.
|
|
144
|
+
if (existsSync(implDir)) {
|
|
145
|
+
throw inputError(
|
|
146
|
+
'IMPL_PACK_ID_COLLISION',
|
|
147
|
+
`Implementation pack directory already exists: ${implDir}`,
|
|
148
|
+
'Retry to generate a fresh implementation pack ID.',
|
|
149
|
+
);
|
|
150
|
+
}
|
|
141
151
|
await mkdir(implDir, { recursive: true });
|
|
142
152
|
|
|
143
153
|
const implManifest = {
|
|
154
|
+
// DB-001 / DB-006: stamp manifest schema version
|
|
155
|
+
schema_version: SCHEMA_VERSION,
|
|
144
156
|
implementation_pack_id: implId,
|
|
145
157
|
created_at: new Date().toISOString(),
|
|
146
158
|
created_by: 'sdlab-implementation-v1',
|
|
@@ -312,5 +324,7 @@ export async function loadImplementationPack(projectRoot, implId) {
|
|
|
312
324
|
if (!existsSync(path)) {
|
|
313
325
|
throw new Error(`Implementation pack "${implId}" not found at ${path}`);
|
|
314
326
|
}
|
|
315
|
-
|
|
327
|
+
const manifest = JSON.parse(await readFile(path, 'utf-8'));
|
|
328
|
+
checkManifestVersion(manifest, 'implementation-pack');
|
|
329
|
+
return manifest;
|
|
316
330
|
}
|