@mcptoolshop/style-dataset-lab 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.es.md +35 -25
- package/README.fr.md +38 -28
- package/README.hi.md +36 -26
- package/README.it.md +37 -27
- package/README.ja.md +40 -30
- package/README.md +15 -5
- package/README.pt-BR.md +35 -25
- package/README.zh.md +33 -23
- package/bin/sdlab.js +51 -4
- package/lib/adapters/ai-toolkit.js +37 -5
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +10 -2
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +13 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +16 -1
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/measure.js +422 -0
- package/lib/paths.js +196 -7
- package/lib/reingest-selected.js +98 -16
- package/lib/run-manifest.js +102 -16
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/workflow-profiles.js +24 -3
- package/package.json +3 -2
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +27 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +181 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +48 -5
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
package/lib/ingest.js
ADDED
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ingest engine — bring pre-existing images into the pipeline as bare,
|
|
3
|
+
* uncurated candidate records.
|
|
4
|
+
*
|
|
5
|
+
* The gap this closes: there was no CLI path to get an image that was NOT
|
|
6
|
+
* produced by `sdlab generate`/`run generate`/`batch generate` into the
|
|
7
|
+
* project. `sdlab reingest generated` requires a trained LoRA's training
|
|
8
|
+
* manifest (lib/reingest.js loadTrainingManifest() unconditionally) — it
|
|
9
|
+
* exists to bring TRAINED-MODEL outputs back in, not pre-training
|
|
10
|
+
* candidates. `sdlab reingest selected` requires a selection id, which only
|
|
11
|
+
* the production loop can mint. Neither can take "a folder of images from
|
|
12
|
+
* somewhere else" (hand-picked references, a batch rendered in external
|
|
13
|
+
* Python against Comfy Cloud, salvage from a lost pipeline run) and turn it
|
|
14
|
+
* into records ready for `sdlab curate`. This module is that third path.
|
|
15
|
+
*
|
|
16
|
+
* Non-negotiables this module holds itself to:
|
|
17
|
+
* - NEVER invents a judgment, a score, or a caption. Every record ships
|
|
18
|
+
* with judgment: null, canon: null (via buildBaseRecord, the same
|
|
19
|
+
* shared helper lib/reingest.js already uses) — uncurated, exactly like
|
|
20
|
+
* everything else that hasn't been through `sdlab curate` yet.
|
|
21
|
+
* - provenance.source is always 'external' — honestly distinguishable
|
|
22
|
+
* from provenance.source === 'generated' (lib/reingest.js) and from the
|
|
23
|
+
* flat generation_provenance shape lib/generated-provenance.js builds
|
|
24
|
+
* for selected run/batch outputs. An ingested record's generation
|
|
25
|
+
* details (prompt, seed, ...) are attached ONLY when a --wave file or a
|
|
26
|
+
* receipts.jsonl actually supplies them for that specific id — never
|
|
27
|
+
* guessed, never interpolated from a sibling.
|
|
28
|
+
* - id derivation is validated, not silently rewritten: a filename stem
|
|
29
|
+
* that fails the SAME allowlist lib/export.js already uses for every
|
|
30
|
+
* other record-id-becomes-a-filename boundary (SAFE_ID_RE / assertSafeId
|
|
31
|
+
* — SDL-H5) is rejected per-file with a reason, not coerced into
|
|
32
|
+
* something "close enough". A directory scan is bulk, externally-named
|
|
33
|
+
* input by nature (unlike an already-pipeline-produced record id), so
|
|
34
|
+
* one unsafe filename is reported and skipped rather than aborting the
|
|
35
|
+
* whole ingest — the vector is closed either way, since no path is ever
|
|
36
|
+
* built from a name that fails the check.
|
|
37
|
+
* - --dry-run performs zero filesystem writes: no mkdir, no image copy, no
|
|
38
|
+
* record write. (SDL-M11 fixed five scripts that violated exactly this.)
|
|
39
|
+
* - idempotent: an id whose record already exists is skipped, never
|
|
40
|
+
* duplicated or clobbered, and every skip is reported back to the
|
|
41
|
+
* caller (not just counted).
|
|
42
|
+
* - crash-safe: both the copied image and the written record land via
|
|
43
|
+
* temp-file-then-rename, mirroring the atomic pattern lib/runtime-runs.js
|
|
44
|
+
* (saveRunManifest/checkpointRunManifest) and lib/batch-runs.js already
|
|
45
|
+
* use. A process killed mid-write must never leave a truncated file at
|
|
46
|
+
* the FINAL path that a later run's existsSync() mistakes for "done".
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
import { readFile, writeFile, readdir, stat, mkdir, copyFile, rename, unlink } from 'node:fs/promises';
|
|
50
|
+
import { join, basename, extname, relative, sep } from 'node:path';
|
|
51
|
+
import { existsSync } from 'node:fs';
|
|
52
|
+
import { buildBaseRecord } from './records.js';
|
|
53
|
+
import { assertSafeId } from './export.js';
|
|
54
|
+
import { inputError } from './errors.js';
|
|
55
|
+
import { warn, verbose } from './log.js';
|
|
56
|
+
|
|
57
|
+
// Mirrors lib/reingest.js's IMAGE_EXTENSIONS. Not imported from there —
|
|
58
|
+
// that module doesn't export the constant, and this file's ownership
|
|
59
|
+
// boundary for this change doesn't include editing lib/reingest.js.
|
|
60
|
+
const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
|
|
61
|
+
|
|
62
|
+
const RECEIPTS_FILENAME = 'receipts.jsonl';
|
|
63
|
+
|
|
64
|
+
/** Normalize a path to forward slashes for storage (matches asset_path style used throughout this codebase). */
|
|
65
|
+
function toPortablePath(p) {
|
|
66
|
+
return p.split(sep).join('/');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ─── Atomic write helpers ──────────────────────────────────────────
|
|
70
|
+
//
|
|
71
|
+
// Same temp-file-then-rename shape as lib/runtime-runs.js's
|
|
72
|
+
// atomicWriteJson / lib/batch-runs.js's atomicWriteJson / lib/reingest-
|
|
73
|
+
// selected.js's atomicWriteText. None of those three are exported (each
|
|
74
|
+
// module keeps its own copy — that is the established convention in this
|
|
75
|
+
// codebase for this exact ~10-line helper, not an oversight to "fix" by
|
|
76
|
+
// centralizing here), so this is a fourth copy of the same pattern rather
|
|
77
|
+
// than a new invention.
|
|
78
|
+
|
|
79
|
+
async function atomicWriteFile(targetPath, body) {
|
|
80
|
+
const tmpPath = `${targetPath}.tmp-${process.pid}-${Date.now()}`;
|
|
81
|
+
try {
|
|
82
|
+
await writeFile(tmpPath, body);
|
|
83
|
+
await rename(tmpPath, targetPath);
|
|
84
|
+
} catch (err) {
|
|
85
|
+
try { await unlink(tmpPath); } catch { /* best-effort cleanup */ }
|
|
86
|
+
throw err;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Copy a file atomically: copy to a temp path in the SAME directory as
|
|
92
|
+
* destPath, then rename. A process killed mid-copy leaves only the
|
|
93
|
+
* uniquely-named temp file behind — never a partial file sitting at
|
|
94
|
+
* destPath that a later run's `existsSync(destPath)` would mistake for a
|
|
95
|
+
* complete, already-ingested image and skip re-copying forever.
|
|
96
|
+
*/
|
|
97
|
+
async function atomicCopyFile(srcPath, destPath) {
|
|
98
|
+
const tmpPath = `${destPath}.tmp-${process.pid}-${Date.now()}`;
|
|
99
|
+
try {
|
|
100
|
+
await copyFile(srcPath, tmpPath);
|
|
101
|
+
await rename(tmpPath, destPath);
|
|
102
|
+
} catch (err) {
|
|
103
|
+
try { await unlink(tmpPath); } catch { /* best-effort cleanup */ }
|
|
104
|
+
throw err;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// ─── Wave / receipts loaders ────────────────────────────────────────
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Load a wave pack (the shape of projects/salt-road/inputs/prompts/
|
|
112
|
+
* wave-v2.json: `{ formula_ref?, wave?, items: [{ id, seed, subject, ... }] }`)
|
|
113
|
+
* into a Map<id, item>. Falls back to a `subjects` array (the
|
|
114
|
+
* `sdlab generate` prompt-pack shape) if `items` is absent, so a wave file
|
|
115
|
+
* authored either way is still usable.
|
|
116
|
+
*
|
|
117
|
+
* Returns an empty index when wavePath is falsy — --wave is opt-in
|
|
118
|
+
* enrichment, not a requirement. A wavePath that IS given but cannot be
|
|
119
|
+
* read/parsed is a user error worth stopping for (they asked for that
|
|
120
|
+
* file specifically), not a silent "provenance unknown" fallback.
|
|
121
|
+
*
|
|
122
|
+
* @param {string|null|undefined} wavePath — absolute, already-validated path
|
|
123
|
+
* @returns {Promise<{index: Map<string,Object>, formulaRef: string|null, waveName: string|null}>}
|
|
124
|
+
*/
|
|
125
|
+
export async function loadWaveIndex(wavePath) {
|
|
126
|
+
const index = new Map();
|
|
127
|
+
if (!wavePath) return { index, formulaRef: null, waveName: null };
|
|
128
|
+
|
|
129
|
+
if (!existsSync(wavePath)) {
|
|
130
|
+
throw inputError(
|
|
131
|
+
'INPUT_WAVE_NOT_FOUND',
|
|
132
|
+
`--wave file not found: ${wavePath}`,
|
|
133
|
+
'Check the path. Wave files live under inputs/prompts/*.json — see projects/salt-road/inputs/prompts/wave-v2.json for the shape.'
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
let raw;
|
|
138
|
+
try {
|
|
139
|
+
raw = JSON.parse(await readFile(wavePath, 'utf-8'));
|
|
140
|
+
} catch (err) {
|
|
141
|
+
throw inputError(
|
|
142
|
+
'INPUT_INVALID_JSON',
|
|
143
|
+
`Failed to parse --wave file ${wavePath}: ${err.message}`,
|
|
144
|
+
'Check the file for JSON syntax errors.'
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const items = Array.isArray(raw.items) ? raw.items
|
|
149
|
+
: Array.isArray(raw.subjects) ? raw.subjects
|
|
150
|
+
: null;
|
|
151
|
+
|
|
152
|
+
if (!items) {
|
|
153
|
+
warn(`ingest: --wave file ${wavePath} has no "items" or "subjects" array — no entries will be attached.`);
|
|
154
|
+
return { index, formulaRef: raw.formula_ref || null, waveName: raw.wave || raw.name || null };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
for (const item of items) {
|
|
158
|
+
if (item && typeof item.id === 'string') index.set(item.id, item);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return { index, formulaRef: raw.formula_ref || null, waveName: raw.wave || raw.name || null };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Load a receipts.jsonl (the shape of projects/salt-road/inbox/generated/
|
|
166
|
+
* set-v2-2026-07-30/receipts.jsonl: one `{id, status, prompt_id, seed, secs}`
|
|
167
|
+
* object per line) into a Map<id, receipt>.
|
|
168
|
+
*
|
|
169
|
+
* Best-effort by design: a missing file yields an empty Map (this is
|
|
170
|
+
* opportunistic enrichment co-located with the images, not a requirement),
|
|
171
|
+
* and a malformed INDIVIDUAL line is skipped with a warning rather than
|
|
172
|
+
* failing the whole load — one corrupt trailing line from an interrupted
|
|
173
|
+
* write must not blank out every other real receipt in the file.
|
|
174
|
+
*
|
|
175
|
+
* @param {string} receiptsPath — absolute path
|
|
176
|
+
* @returns {Promise<Map<string,Object>>}
|
|
177
|
+
*/
|
|
178
|
+
export async function loadReceiptsIndex(receiptsPath) {
|
|
179
|
+
const index = new Map();
|
|
180
|
+
if (!receiptsPath || !existsSync(receiptsPath)) return index;
|
|
181
|
+
|
|
182
|
+
const raw = await readFile(receiptsPath, 'utf-8');
|
|
183
|
+
const lines = raw.split('\n');
|
|
184
|
+
for (let i = 0; i < lines.length; i++) {
|
|
185
|
+
const trimmed = lines[i].trim();
|
|
186
|
+
if (!trimmed) continue;
|
|
187
|
+
try {
|
|
188
|
+
const entry = JSON.parse(trimmed);
|
|
189
|
+
if (entry && typeof entry.id === 'string') index.set(entry.id, entry);
|
|
190
|
+
} catch (err) {
|
|
191
|
+
warn(`ingest: ${receiptsPath}:${i + 1} is not valid JSON — line skipped (${err.message})`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return index;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// ─── Provenance builder ─────────────────────────────────────────────
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Build the provenance block for one ingested image. Additive-only: a
|
|
201
|
+
* field is set exactly when a real source (the --wave entry or the
|
|
202
|
+
* receipts.jsonl entry for this id) supplied it. Nothing here is ever
|
|
203
|
+
* synthesized, interpolated, or defaulted from another record.
|
|
204
|
+
*/
|
|
205
|
+
function buildExternalProvenance({ file, sourcePath, waveEntry, waveMeta, receiptEntry, receiptsRelPath }) {
|
|
206
|
+
const prov = {
|
|
207
|
+
source: 'external',
|
|
208
|
+
ingested_at: new Date().toISOString(),
|
|
209
|
+
original_filename: file,
|
|
210
|
+
original_path: sourcePath,
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
const notes = [];
|
|
214
|
+
|
|
215
|
+
if (waveEntry) {
|
|
216
|
+
prov.wave_file = waveMeta.waveRelPath;
|
|
217
|
+
if (waveMeta.waveName) prov.wave_name = waveMeta.waveName;
|
|
218
|
+
if (waveMeta.formulaRef) {
|
|
219
|
+
prov.formula_ref = waveMeta.formulaRef;
|
|
220
|
+
notes.push(
|
|
221
|
+
'preamble_ref/negative_ref are keys into the wave\'s formula_ref file, not literal prompt/negative text.'
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
if (waveEntry.seed !== undefined) prov.seed = waveEntry.seed;
|
|
225
|
+
const promptText = waveEntry.subject ?? waveEntry.prompt;
|
|
226
|
+
if (promptText !== undefined) prov.prompt = promptText;
|
|
227
|
+
if (waveEntry.preamble !== undefined) prov.preamble_ref = waveEntry.preamble;
|
|
228
|
+
if (waveEntry.neg !== undefined) prov.negative_ref = waveEntry.neg;
|
|
229
|
+
if (waveEntry.negExtra !== undefined) prov.negative_extra = waveEntry.negExtra;
|
|
230
|
+
if (waveEntry.cn !== undefined) prov.controlnet = waveEntry.cn;
|
|
231
|
+
if (waveEntry.enriched_by !== undefined) prov.enriched_by = waveEntry.enriched_by;
|
|
232
|
+
} else if (waveMeta.wavePath) {
|
|
233
|
+
notes.push(`No entry for this id was found in --wave file ${waveMeta.waveRelPath}.`);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (receiptEntry) {
|
|
237
|
+
prov.receipts_file = receiptsRelPath;
|
|
238
|
+
if (receiptEntry.prompt_id !== undefined) prov.prompt_id = receiptEntry.prompt_id;
|
|
239
|
+
// Wave seed wins if both are present and disagree; a receipt seed is
|
|
240
|
+
// still useful when the wave lacked one — never overwritten if already set.
|
|
241
|
+
if (receiptEntry.seed !== undefined && prov.seed === undefined) prov.seed = receiptEntry.seed;
|
|
242
|
+
if (receiptEntry.secs !== undefined) prov.generation_seconds = receiptEntry.secs;
|
|
243
|
+
if (receiptEntry.status !== undefined) prov.receipt_status = receiptEntry.status;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
prov.generation_provenance_known = Boolean(waveEntry || receiptEntry);
|
|
247
|
+
if (!waveEntry && !receiptEntry) {
|
|
248
|
+
notes.push(
|
|
249
|
+
waveMeta.wavePath
|
|
250
|
+
? 'No matching wave or receipts.jsonl entry for this id — generation details are unknown.'
|
|
251
|
+
: 'No --wave file supplied and no receipts.jsonl found alongside the source images — generation details are unknown.'
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
if (notes.length) prov.provenance_note = notes.join(' ');
|
|
255
|
+
|
|
256
|
+
return prov;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// ─── Main entry point ────────────────────────────────────────────────
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Ingest a directory of pre-existing images into a project as bare,
|
|
263
|
+
* uncurated candidate records ready for `sdlab curate`.
|
|
264
|
+
*
|
|
265
|
+
* @param {string} projectRoot — absolute project root (already resolved by the caller)
|
|
266
|
+
* @param {string} sourceDir — absolute path to a directory of images (already validated by the caller)
|
|
267
|
+
* @param {Object} [options]
|
|
268
|
+
* @param {string} [options.wavePath] — absolute, already-validated path to a wave JSON file
|
|
269
|
+
* @param {boolean} [options.dryRun]
|
|
270
|
+
* @returns {Promise<{created: string[], skipped: string[], rejected: {file:string, reason:string}[], receiptsFile: string|null}>}
|
|
271
|
+
*/
|
|
272
|
+
export async function ingestDirectory(projectRoot, sourceDir, options = {}) {
|
|
273
|
+
const dryRun = Boolean(options.dryRun);
|
|
274
|
+
|
|
275
|
+
if (!existsSync(sourceDir)) {
|
|
276
|
+
throw inputError(
|
|
277
|
+
'INPUT_SOURCE_NOT_FOUND',
|
|
278
|
+
`Source directory not found: ${sourceDir}`,
|
|
279
|
+
'Check the path passed to "sdlab ingest <dir>".'
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
const recordsDir = join(projectRoot, 'records');
|
|
284
|
+
const candidatesDir = join(projectRoot, 'outputs', 'candidates');
|
|
285
|
+
|
|
286
|
+
// Wave + receipts are loaded up front, once, before touching any output
|
|
287
|
+
// directory — reading is safe under --dry-run (no mutation).
|
|
288
|
+
const wavePath = options.wavePath || null;
|
|
289
|
+
const { index: waveIndex, formulaRef, waveName } = await loadWaveIndex(wavePath);
|
|
290
|
+
const waveMeta = {
|
|
291
|
+
wavePath,
|
|
292
|
+
waveRelPath: wavePath ? toPortablePath(relative(projectRoot, wavePath)) : null,
|
|
293
|
+
formulaRef,
|
|
294
|
+
waveName,
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
const receiptsPath = join(sourceDir, RECEIPTS_FILENAME);
|
|
298
|
+
const receiptsIndex = await loadReceiptsIndex(receiptsPath);
|
|
299
|
+
const receiptsFound = receiptsIndex.size > 0;
|
|
300
|
+
const receiptsRelPath = receiptsFound ? toPortablePath(relative(projectRoot, receiptsPath)) : null;
|
|
301
|
+
if (receiptsFound) {
|
|
302
|
+
verbose(`ingest: loaded ${receiptsIndex.size} receipt(s) from ${receiptsPath}`);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// SDL-M11 discipline: a --dry-run preview must be side-effect-free. Only
|
|
306
|
+
// create output directories on a real run.
|
|
307
|
+
if (!dryRun) {
|
|
308
|
+
await mkdir(recordsDir, { recursive: true });
|
|
309
|
+
await mkdir(candidatesDir, { recursive: true });
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const entries = (await readdir(sourceDir)).sort();
|
|
313
|
+
const imageFiles = entries.filter((f) => IMAGE_EXTENSIONS.has(extname(f).toLowerCase()));
|
|
314
|
+
|
|
315
|
+
const created = [];
|
|
316
|
+
const skipped = [];
|
|
317
|
+
const rejected = [];
|
|
318
|
+
|
|
319
|
+
for (const file of imageFiles) {
|
|
320
|
+
const ext = extname(file);
|
|
321
|
+
const stem = basename(file, ext);
|
|
322
|
+
|
|
323
|
+
// SDL-H5-style boundary: a filename becoming a record id (and later a
|
|
324
|
+
// path segment for the copied image / record file) is a path-traversal
|
|
325
|
+
// vector. Reuses the exact allowlist lib/export.js already applies at
|
|
326
|
+
// every other record-id-becomes-a-filename boundary in this codebase.
|
|
327
|
+
// Unlike that adapter-layer call site (which hard-throws because an
|
|
328
|
+
// unsafe id THERE means a pipeline-internal invariant already broke),
|
|
329
|
+
// this is a first-touch scan of an arbitrary, externally-named
|
|
330
|
+
// directory — one badly-named file (spaces, unicode, an accidental
|
|
331
|
+
// "..") is the normal case, not an attack. It is rejected and reported
|
|
332
|
+
// so the other 99 files in the same directory still get ingested; the
|
|
333
|
+
// vector is closed identically either way, since no path is ever built
|
|
334
|
+
// from a name that fails this check.
|
|
335
|
+
try {
|
|
336
|
+
assertSafeId(stem, `filename "${file}" in ${sourceDir}`);
|
|
337
|
+
} catch (err) {
|
|
338
|
+
rejected.push({ file, reason: err.message });
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const id = stem;
|
|
343
|
+
const recordPath = join(recordsDir, `${id}.json`);
|
|
344
|
+
|
|
345
|
+
if (existsSync(recordPath)) {
|
|
346
|
+
skipped.push(id);
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
if (dryRun) {
|
|
351
|
+
// Preview only — no stat(), no copy, no write. Report what WOULD be
|
|
352
|
+
// created without touching the filesystem beyond the read-only
|
|
353
|
+
// listing/existence checks already done above.
|
|
354
|
+
created.push(id);
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const sourcePath = join(sourceDir, file);
|
|
359
|
+
const destPath = join(candidatesDir, file);
|
|
360
|
+
const assetPath = toPortablePath(join('outputs', 'candidates', file));
|
|
361
|
+
|
|
362
|
+
const fileStat = await stat(sourcePath);
|
|
363
|
+
|
|
364
|
+
if (!existsSync(destPath)) {
|
|
365
|
+
await atomicCopyFile(sourcePath, destPath);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
const waveEntry = waveIndex.get(id) || null;
|
|
369
|
+
const receiptEntry = receiptsIndex.get(id) || null;
|
|
370
|
+
const provenance = buildExternalProvenance({
|
|
371
|
+
file,
|
|
372
|
+
sourcePath,
|
|
373
|
+
waveEntry,
|
|
374
|
+
waveMeta,
|
|
375
|
+
receiptEntry,
|
|
376
|
+
receiptsRelPath,
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
const record = buildBaseRecord(
|
|
380
|
+
id,
|
|
381
|
+
assetPath,
|
|
382
|
+
{ width: null, height: null, bytes: fileStat.size },
|
|
383
|
+
provenance
|
|
384
|
+
);
|
|
385
|
+
|
|
386
|
+
await atomicWriteFile(recordPath, JSON.stringify(record, null, 2) + '\n');
|
|
387
|
+
created.push(id);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
return {
|
|
391
|
+
created,
|
|
392
|
+
skipped,
|
|
393
|
+
rejected,
|
|
394
|
+
receiptsFile: receiptsFound ? receiptsRelPath : null,
|
|
395
|
+
};
|
|
396
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* judgment-provenance.js — tell a judgment somebody MADE from one a script MINTED.
|
|
3
|
+
*
|
|
4
|
+
* Law 2 of this product is "inclusion is explainable": every record carries a
|
|
5
|
+
* reason it was included or excluded. That law is only worth anything if the
|
|
6
|
+
* reason describes the actual image.
|
|
7
|
+
*
|
|
8
|
+
* It currently doesn't, at scale. Measured on `projects/star-freight`
|
|
9
|
+
* (1,182 records) on 2026-08-04:
|
|
10
|
+
*
|
|
11
|
+
* reviewer records distinct explanations max reuse
|
|
12
|
+
* bulk_curate_v2 566 ) 82 across both x144
|
|
13
|
+
* bulk_curate_v1 322 )
|
|
14
|
+
* wave25-script 60 8 x32
|
|
15
|
+
* human:mike 110 110 x1
|
|
16
|
+
* (no judgment) 124 — —
|
|
17
|
+
*
|
|
18
|
+
* 948 of 1,182 records — 80% — carry a rationale that was selected by matching
|
|
19
|
+
* the record id against a prefix regex, not by looking at the image. One string
|
|
20
|
+
* ("On-style gritty space costume. Faction-appropriate material vocabulary and
|
|
21
|
+
* wear level…") appears verbatim on 144 different images. The three scripts
|
|
22
|
+
* that wrote them (`bulk-curate-wave2-5.js`, `bulk-curate-waves11-18.js`,
|
|
23
|
+
* `curate-wave25.js`) were deleted in this repo's Stage A health pass for
|
|
24
|
+
* exactly this reason — but deleting the tool did not retract its output, and
|
|
25
|
+
* that output still flows into eligibility, snapshots, splits, exports and
|
|
26
|
+
* training packages.
|
|
27
|
+
*
|
|
28
|
+
* The `human:mike` row is the control that makes this a finding rather than a
|
|
29
|
+
* guess: 110 records, 110 distinct explanations, nothing reused. Genuine
|
|
30
|
+
* curation in this corpus looks like one rationale per image.
|
|
31
|
+
*
|
|
32
|
+
* ─────────────────────────────────────────────────────────────────────────
|
|
33
|
+
* THIS MODULE READS. IT NEVER WRITES.
|
|
34
|
+
*
|
|
35
|
+
* The Director's ruling (2026-08-04) was "mark, don't touch": make the
|
|
36
|
+
* distinction visible everywhere it matters, and leave the records alone. So
|
|
37
|
+
* there is no migration here, no backfilled flag, no rewritten judgment. The
|
|
38
|
+
* signal is derived at read time from data already on disk, which also means
|
|
39
|
+
* it cannot drift out of sync with the records and needs no re-run when they
|
|
40
|
+
* change.
|
|
41
|
+
* ─────────────────────────────────────────────────────────────────────────
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Reviewer ids known to have been written by a bulk script rather than a
|
|
46
|
+
* person. Matched against `judgment.reviewer`.
|
|
47
|
+
*
|
|
48
|
+
* Deliberately a conservative allowlist of OBSERVED offenders, not a clever
|
|
49
|
+
* heuristic like /script|bot|auto/. A false positive here defames a real
|
|
50
|
+
* human judgment, which is worse than missing one: the corpus-level reuse
|
|
51
|
+
* statistic below catches anything this list doesn't, without needing to
|
|
52
|
+
* guess from a name.
|
|
53
|
+
*/
|
|
54
|
+
export const SCRIPT_REVIEWER_PATTERNS = Object.freeze([
|
|
55
|
+
/^bulk_curate(_v\d+)?$/i,
|
|
56
|
+
/^wave\d+-script$/i,
|
|
57
|
+
]);
|
|
58
|
+
|
|
59
|
+
/** A judgment is machine-minted, human-made, or absent. */
|
|
60
|
+
export const JUDGMENT_ORIGIN = Object.freeze({
|
|
61
|
+
HUMAN: 'human',
|
|
62
|
+
SCRIPT: 'script',
|
|
63
|
+
UNKNOWN: 'unknown',
|
|
64
|
+
NONE: 'none',
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Classify a single record's judgment origin from its reviewer id.
|
|
69
|
+
*
|
|
70
|
+
* @param {object} record
|
|
71
|
+
* @returns {'human'|'script'|'unknown'|'none'}
|
|
72
|
+
*/
|
|
73
|
+
export function classifyJudgmentOrigin(record) {
|
|
74
|
+
const judgment = record?.judgment;
|
|
75
|
+
if (!judgment || typeof judgment !== 'object') return JUDGMENT_ORIGIN.NONE;
|
|
76
|
+
|
|
77
|
+
const reviewer = judgment.reviewer;
|
|
78
|
+
if (typeof reviewer !== 'string' || reviewer.trim() === '') {
|
|
79
|
+
return JUDGMENT_ORIGIN.UNKNOWN;
|
|
80
|
+
}
|
|
81
|
+
if (SCRIPT_REVIEWER_PATTERNS.some((re) => re.test(reviewer))) {
|
|
82
|
+
return JUDGMENT_ORIGIN.SCRIPT;
|
|
83
|
+
}
|
|
84
|
+
// `human:<name>` is this corpus's convention for a person.
|
|
85
|
+
if (/^human(:|$)/i.test(reviewer)) return JUDGMENT_ORIGIN.HUMAN;
|
|
86
|
+
|
|
87
|
+
// A reviewer id we don't recognise. NOT assumed human — that assumption is
|
|
88
|
+
// how 948 fabricated rationales passed as curation for months.
|
|
89
|
+
return JUDGMENT_ORIGIN.UNKNOWN;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Corpus-level rationale-reuse audit.
|
|
94
|
+
*
|
|
95
|
+
* This is the evidence the reviewer-name check cannot give you, and it is the
|
|
96
|
+
* part that generalises: a rationale reused verbatim across many images did
|
|
97
|
+
* not describe any one of them, whoever or whatever wrote it. It catches a
|
|
98
|
+
* future bulk script under a reviewer id nobody has added to the list above.
|
|
99
|
+
*
|
|
100
|
+
* `reuseThreshold` is the count at which sharing stops being coincidence.
|
|
101
|
+
* Two records can legitimately share a terse reason ("out of focus"); 144
|
|
102
|
+
* cannot.
|
|
103
|
+
*
|
|
104
|
+
* @param {object[]} records
|
|
105
|
+
* @param {{reuseThreshold?: number}} [opts]
|
|
106
|
+
*/
|
|
107
|
+
export function auditJudgmentProvenance(records, opts = {}) {
|
|
108
|
+
const reuseThreshold = opts.reuseThreshold ?? 3;
|
|
109
|
+
|
|
110
|
+
const byOrigin = { human: 0, script: 0, unknown: 0, none: 0 };
|
|
111
|
+
const byReviewer = new Map();
|
|
112
|
+
const byExplanation = new Map();
|
|
113
|
+
|
|
114
|
+
for (const record of records) {
|
|
115
|
+
const origin = classifyJudgmentOrigin(record);
|
|
116
|
+
byOrigin[origin]++;
|
|
117
|
+
|
|
118
|
+
const judgment = record?.judgment;
|
|
119
|
+
if (!judgment) continue;
|
|
120
|
+
|
|
121
|
+
const reviewer = typeof judgment.reviewer === 'string' ? judgment.reviewer : '(unset)';
|
|
122
|
+
byReviewer.set(reviewer, (byReviewer.get(reviewer) ?? 0) + 1);
|
|
123
|
+
|
|
124
|
+
const explanation = judgment.explanation;
|
|
125
|
+
if (typeof explanation === 'string' && explanation.trim() !== '') {
|
|
126
|
+
const key = explanation.trim();
|
|
127
|
+
if (!byExplanation.has(key)) byExplanation.set(key, []);
|
|
128
|
+
byExplanation.get(key).push(record.id);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const reused = [...byExplanation.entries()]
|
|
133
|
+
.filter(([, ids]) => ids.length >= reuseThreshold)
|
|
134
|
+
.map(([explanation, ids]) => ({
|
|
135
|
+
explanation,
|
|
136
|
+
count: ids.length,
|
|
137
|
+
sample_record_ids: ids.slice(0, 5),
|
|
138
|
+
}))
|
|
139
|
+
.sort((a, b) => b.count - a.count);
|
|
140
|
+
|
|
141
|
+
const judgedCount = byOrigin.human + byOrigin.script + byOrigin.unknown;
|
|
142
|
+
const reusedRecordCount = reused.reduce((sum, r) => sum + r.count, 0);
|
|
143
|
+
|
|
144
|
+
return {
|
|
145
|
+
total_records: records.length,
|
|
146
|
+
judged: judgedCount,
|
|
147
|
+
by_origin: byOrigin,
|
|
148
|
+
by_reviewer: Object.fromEntries([...byReviewer.entries()].sort((a, b) => b[1] - a[1])),
|
|
149
|
+
// How many judged records carry a rationale that is not unique to them.
|
|
150
|
+
// This is the honest headline number — it does not depend on recognising
|
|
151
|
+
// a reviewer id.
|
|
152
|
+
reused_rationale_records: reusedRecordCount,
|
|
153
|
+
reused_rationale_ratio: judgedCount > 0 ? reusedRecordCount / judgedCount : 0,
|
|
154
|
+
reuse_threshold: reuseThreshold,
|
|
155
|
+
// Worst offenders first, capped — this is for an operator to read, not a
|
|
156
|
+
// full dump of a 1,182-record corpus.
|
|
157
|
+
top_reused: reused.slice(0, 10),
|
|
158
|
+
// True when any judged record's rationale is shared at or above the
|
|
159
|
+
// threshold. Consumers gate their wording on this rather than re-deriving.
|
|
160
|
+
has_reused_rationales: reusedRecordCount > 0,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* One-line human summary for CLI output. Returns null when there is nothing
|
|
166
|
+
* to report, so a clean corpus prints nothing rather than a reassuring banner
|
|
167
|
+
* nobody reads.
|
|
168
|
+
*
|
|
169
|
+
* @param {ReturnType<typeof auditJudgmentProvenance>} audit
|
|
170
|
+
* @returns {string|null}
|
|
171
|
+
*/
|
|
172
|
+
export function formatProvenanceWarning(audit) {
|
|
173
|
+
// ONLY a script-minted judgment is asserted as a Law 2 problem.
|
|
174
|
+
//
|
|
175
|
+
// Reuse alone is NOT evidence of fabrication, and an earlier revision of
|
|
176
|
+
// this function got that wrong. Measured on `projects/tallow-fen`: 54 of
|
|
177
|
+
// 293 rationales are reused, and they are legitimate — two strings, ×28 and
|
|
178
|
+
// ×26, each accurately describing ONE systematic defect across exactly the
|
|
179
|
+
// images that have it ("style on-target but the style descriptor's
|
|
180
|
+
// lantern-light noun literalized"). Reviewers there are `human:mike` and
|
|
181
|
+
// `claude:fable-5`; zero scripts. That is good curation of a bad wave.
|
|
182
|
+
//
|
|
183
|
+
// The heuristic cannot separate "144 images share a canned category blurb
|
|
184
|
+
// chosen by id-prefix" from "28 images share a true description of one
|
|
185
|
+
// shared flaw" — so it must not claim to. A gate that fires when nothing is
|
|
186
|
+
// wrong teaches its reader to ignore it, which is the same defect class as
|
|
187
|
+
// a gate that cannot fire, pointed the other way.
|
|
188
|
+
if (audit.by_origin.script === 0) return null;
|
|
189
|
+
|
|
190
|
+
const pct = Math.round((audit.by_origin.script / Math.max(audit.judged, 1)) * 100);
|
|
191
|
+
let msg =
|
|
192
|
+
`Judgment provenance: ${audit.by_origin.script} of ${audit.judged} judgments (${pct}%) ` +
|
|
193
|
+
`were written by a bulk script, not a reviewer. Those rationales were selected by ` +
|
|
194
|
+
`matching the record id against a pattern — they do not describe the image. ` +
|
|
195
|
+
`Treat them as uncurated for Law 2 purposes.`;
|
|
196
|
+
|
|
197
|
+
if (audit.reused_rationale_records > 0) {
|
|
198
|
+
msg += ` (${audit.reused_rationale_records} rationales are also reused verbatim across records.)`;
|
|
199
|
+
}
|
|
200
|
+
return msg;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Neutral, non-accusatory reuse note — for corpora with no script reviewer but
|
|
205
|
+
* heavy rationale reuse. Reuse there is usually a real systematic defect
|
|
206
|
+
* described accurately across a wave, which is worth SEEING but is not a
|
|
207
|
+
* finding. Returns null when there is nothing to say.
|
|
208
|
+
*
|
|
209
|
+
* @param {ReturnType<typeof auditJudgmentProvenance>} audit
|
|
210
|
+
* @returns {string|null}
|
|
211
|
+
*/
|
|
212
|
+
export function formatReuseNote(audit) {
|
|
213
|
+
if (audit.by_origin.script > 0) return null; // the warning above covers it
|
|
214
|
+
if (audit.reused_rationale_records === 0) return null;
|
|
215
|
+
|
|
216
|
+
const pct = Math.round(audit.reused_rationale_ratio * 100);
|
|
217
|
+
return `${audit.reused_rationale_records} of ${audit.judged} rationales (${pct}%) are shared ` +
|
|
218
|
+
`across records — often one systematic defect described across the wave it affected. ` +
|
|
219
|
+
`Worth a glance, not necessarily a problem.`;
|
|
220
|
+
}
|