@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/README.es.md +117 -55
  3. package/README.fr.md +125 -65
  4. package/README.hi.md +117 -68
  5. package/README.it.md +119 -58
  6. package/README.ja.md +124 -74
  7. package/README.md +73 -7
  8. package/README.pt-BR.md +128 -63
  9. package/README.zh.md +116 -65
  10. package/bin/sdlab.js +100 -4
  11. package/lib/adapters/ai-toolkit.js +224 -35
  12. package/lib/adapters/comfyui-runner.js +79 -26
  13. package/lib/adapters/comfyui-workflows.js +94 -11
  14. package/lib/adapters/diffusers-lora.js +37 -5
  15. package/lib/adapters/generic-image-caption.js +43 -5
  16. package/lib/args.js +16 -1
  17. package/lib/asset-ingest.js +575 -0
  18. package/lib/asset-masks.js +323 -0
  19. package/lib/asset-source.js +1176 -0
  20. package/lib/batch-modes.js +29 -8
  21. package/lib/batch-runs.js +29 -1
  22. package/lib/batch-sheet-render.js +13 -3
  23. package/lib/brief-compiler.js +19 -0
  24. package/lib/candidate-sheet.js +327 -0
  25. package/lib/canon-author.js +350 -0
  26. package/lib/canon-build/build.js +146 -93
  27. package/lib/canon-review-gate.js +205 -0
  28. package/lib/captions.js +57 -0
  29. package/lib/card.js +74 -2
  30. package/lib/colab-launcher-guards.js +285 -0
  31. package/lib/comfyui.js +116 -18
  32. package/lib/config.js +116 -12
  33. package/lib/critique-engine.js +36 -2
  34. package/lib/eligibility.js +121 -9
  35. package/lib/eval-pack.js +14 -14
  36. package/lib/eval-runs.js +33 -0
  37. package/lib/export.js +122 -34
  38. package/lib/freeze-gate.js +16 -3
  39. package/lib/freeze-stamp.js +28 -3
  40. package/lib/generated-provenance.js +194 -6
  41. package/lib/implementation-packs.js +95 -17
  42. package/lib/ingest.js +396 -0
  43. package/lib/judgment-provenance.js +220 -0
  44. package/lib/lane1-render.js +668 -0
  45. package/lib/measure.js +422 -0
  46. package/lib/memory-store.js +663 -0
  47. package/lib/model-licences.js +305 -0
  48. package/lib/npy-meta.js +130 -0
  49. package/lib/paths.js +196 -7
  50. package/lib/png-meta.js +512 -0
  51. package/lib/records.js +54 -8
  52. package/lib/reingest-selected.js +113 -17
  53. package/lib/reingest.js +27 -3
  54. package/lib/run-manifest.js +159 -25
  55. package/lib/run-summary.js +12 -6
  56. package/lib/runtime-runs.js +45 -4
  57. package/lib/snapshot.js +114 -15
  58. package/lib/split.js +364 -48
  59. package/lib/train-runs.js +214 -0
  60. package/lib/training-manifests.js +114 -21
  61. package/lib/training-packages.js +147 -15
  62. package/lib/verifier.js +3 -0
  63. package/lib/verify-chain.js +367 -0
  64. package/lib/workflow-profiles.js +24 -3
  65. package/package.json +4 -3
  66. package/runtime/comfyui/txt2img-qwen.json +38 -0
  67. package/runtime/comfyui/txt2img-standard.json +21 -0
  68. package/scripts/_resume.js +62 -0
  69. package/scripts/_shared-args.js +44 -0
  70. package/scripts/asset-ingest.js +160 -0
  71. package/scripts/brief-compile.js +2 -1
  72. package/scripts/brief-show.js +2 -1
  73. package/scripts/canon-bind.js +2 -2
  74. package/scripts/canon-build.js +27 -20
  75. package/scripts/canon-draft.js +82 -0
  76. package/scripts/canon-drift.js +16 -7
  77. package/scripts/canon-freeze-status.js +14 -6
  78. package/scripts/canon-freeze.js +31 -11
  79. package/scripts/canon-unfreeze.js +17 -9
  80. package/scripts/card.js +2 -2
  81. package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
  82. package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
  83. package/scripts/colab/colab_window.sh +587 -0
  84. package/scripts/colab/lane1_render.py +970 -0
  85. package/scripts/colab/launch_guards.py +564 -0
  86. package/scripts/colab/launch_lane1_stub.sh +309 -0
  87. package/scripts/colab/launch_lane3_stub.sh +366 -0
  88. package/scripts/colab/remote_runner.py +59 -0
  89. package/scripts/compare.js +2 -2
  90. package/scripts/critique.js +16 -1
  91. package/scripts/curate.js +2 -2
  92. package/scripts/doctor.js +182 -55
  93. package/scripts/eligibility.js +42 -2
  94. package/scripts/eval-pack.js +2 -2
  95. package/scripts/eval-run.js +2 -2
  96. package/scripts/export.js +2 -2
  97. package/scripts/generate-controlnet.js +45 -25
  98. package/scripts/generate-identity.js +53 -23
  99. package/scripts/generate-ipadapter.js +44 -23
  100. package/scripts/generate.js +86 -17
  101. package/scripts/implementation-pack.js +2 -2
  102. package/scripts/ingest.js +145 -0
  103. package/scripts/init.js +73 -13
  104. package/scripts/measure.js +170 -0
  105. package/scripts/measure_image.py +458 -0
  106. package/scripts/migrate-records.js +142 -0
  107. package/scripts/painterly-test.js +4 -3
  108. package/scripts/painterly.js +7 -3
  109. package/scripts/qwen_generate.py +138 -20
  110. package/scripts/reingest.js +2 -2
  111. package/scripts/sheet.js +142 -0
  112. package/scripts/snapshot.js +2 -2
  113. package/scripts/split.js +2 -2
  114. package/scripts/training-manifest.js +2 -2
  115. package/scripts/training-package.js +2 -2
  116. package/scripts/training-profile.js +2 -2
  117. package/scripts/verify.js +67 -0
  118. package/scripts/workflow-list.js +2 -1
  119. package/scripts/workflow-show.js +2 -1
package/lib/ingest.js ADDED
@@ -0,0 +1,396 @@
1
+ /**
2
+ * Ingest engine — bring pre-existing images into the pipeline as bare,
3
+ * uncurated candidate records.
4
+ *
5
+ * The gap this closes: there was no CLI path to get an image that was NOT
6
+ * produced by `sdlab generate`/`run generate`/`batch generate` into the
7
+ * project. `sdlab reingest generated` requires a trained LoRA's training
8
+ * manifest (lib/reingest.js loadTrainingManifest() unconditionally) — it
9
+ * exists to bring TRAINED-MODEL outputs back in, not pre-training
10
+ * candidates. `sdlab reingest selected` requires a selection id, which only
11
+ * the production loop can mint. Neither can take "a folder of images from
12
+ * somewhere else" (hand-picked references, a batch rendered in external
13
+ * Python against Comfy Cloud, salvage from a lost pipeline run) and turn it
14
+ * into records ready for `sdlab curate`. This module is that third path.
15
+ *
16
+ * Non-negotiables this module holds itself to:
17
+ * - NEVER invents a judgment, a score, or a caption. Every record ships
18
+ * with judgment: null, canon: null (via buildBaseRecord, the same
19
+ * shared helper lib/reingest.js already uses) — uncurated, exactly like
20
+ * everything else that hasn't been through `sdlab curate` yet.
21
+ * - provenance.source is always 'external' — honestly distinguishable
22
+ * from provenance.source === 'generated' (lib/reingest.js) and from the
23
+ * flat generation_provenance shape lib/generated-provenance.js builds
24
+ * for selected run/batch outputs. An ingested record's generation
25
+ * details (prompt, seed, ...) are attached ONLY when a --wave file or a
26
+ * receipts.jsonl actually supplies them for that specific id — never
27
+ * guessed, never interpolated from a sibling.
28
+ * - id derivation is validated, not silently rewritten: a filename stem
29
+ * that fails the SAME allowlist lib/export.js already uses for every
30
+ * other record-id-becomes-a-filename boundary (SAFE_ID_RE / assertSafeId
31
+ * — SDL-H5) is rejected per-file with a reason, not coerced into
32
+ * something "close enough". A directory scan is bulk, externally-named
33
+ * input by nature (unlike an already-pipeline-produced record id), so
34
+ * one unsafe filename is reported and skipped rather than aborting the
35
+ * whole ingest — the vector is closed either way, since no path is ever
36
+ * built from a name that fails the check.
37
+ * - --dry-run performs zero filesystem writes: no mkdir, no image copy, no
38
+ * record write. (SDL-M11 fixed five scripts that violated exactly this.)
39
+ * - idempotent: an id whose record already exists is skipped, never
40
+ * duplicated or clobbered, and every skip is reported back to the
41
+ * caller (not just counted).
42
+ * - crash-safe: both the copied image and the written record land via
43
+ * temp-file-then-rename, mirroring the atomic pattern lib/runtime-runs.js
44
+ * (saveRunManifest/checkpointRunManifest) and lib/batch-runs.js already
45
+ * use. A process killed mid-write must never leave a truncated file at
46
+ * the FINAL path that a later run's existsSync() mistakes for "done".
47
+ */
48
+
49
+ import { readFile, writeFile, readdir, stat, mkdir, copyFile, rename, unlink } from 'node:fs/promises';
50
+ import { join, basename, extname, relative, sep } from 'node:path';
51
+ import { existsSync } from 'node:fs';
52
+ import { buildBaseRecord } from './records.js';
53
+ import { assertSafeId } from './export.js';
54
+ import { inputError } from './errors.js';
55
+ import { warn, verbose } from './log.js';
56
+
57
+ // Mirrors lib/reingest.js's IMAGE_EXTENSIONS. Not imported from there —
58
+ // that module doesn't export the constant, and this file's ownership
59
+ // boundary for this change doesn't include editing lib/reingest.js.
60
+ const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.webp']);
61
+
62
+ const RECEIPTS_FILENAME = 'receipts.jsonl';
63
+
64
+ /** Normalize a path to forward slashes for storage (matches asset_path style used throughout this codebase). */
65
+ function toPortablePath(p) {
66
+ return p.split(sep).join('/');
67
+ }
68
+
69
+ // ─── Atomic write helpers ──────────────────────────────────────────
70
+ //
71
+ // Same temp-file-then-rename shape as lib/runtime-runs.js's
72
+ // atomicWriteJson / lib/batch-runs.js's atomicWriteJson / lib/reingest-
73
+ // selected.js's atomicWriteText. None of those three are exported (each
74
+ // module keeps its own copy — that is the established convention in this
75
+ // codebase for this exact ~10-line helper, not an oversight to "fix" by
76
+ // centralizing here), so this is a fourth copy of the same pattern rather
77
+ // than a new invention.
78
+
79
+ async function atomicWriteFile(targetPath, body) {
80
+ const tmpPath = `${targetPath}.tmp-${process.pid}-${Date.now()}`;
81
+ try {
82
+ await writeFile(tmpPath, body);
83
+ await rename(tmpPath, targetPath);
84
+ } catch (err) {
85
+ try { await unlink(tmpPath); } catch { /* best-effort cleanup */ }
86
+ throw err;
87
+ }
88
+ }
89
+
90
+ /**
91
+ * Copy a file atomically: copy to a temp path in the SAME directory as
92
+ * destPath, then rename. A process killed mid-copy leaves only the
93
+ * uniquely-named temp file behind — never a partial file sitting at
94
+ * destPath that a later run's `existsSync(destPath)` would mistake for a
95
+ * complete, already-ingested image and skip re-copying forever.
96
+ */
97
+ async function atomicCopyFile(srcPath, destPath) {
98
+ const tmpPath = `${destPath}.tmp-${process.pid}-${Date.now()}`;
99
+ try {
100
+ await copyFile(srcPath, tmpPath);
101
+ await rename(tmpPath, destPath);
102
+ } catch (err) {
103
+ try { await unlink(tmpPath); } catch { /* best-effort cleanup */ }
104
+ throw err;
105
+ }
106
+ }
107
+
108
+ // ─── Wave / receipts loaders ────────────────────────────────────────
109
+
110
+ /**
111
+ * Load a wave pack (the shape of projects/salt-road/inputs/prompts/
112
+ * wave-v2.json: `{ formula_ref?, wave?, items: [{ id, seed, subject, ... }] }`)
113
+ * into a Map<id, item>. Falls back to a `subjects` array (the
114
+ * `sdlab generate` prompt-pack shape) if `items` is absent, so a wave file
115
+ * authored either way is still usable.
116
+ *
117
+ * Returns an empty index when wavePath is falsy — --wave is opt-in
118
+ * enrichment, not a requirement. A wavePath that IS given but cannot be
119
+ * read/parsed is a user error worth stopping for (they asked for that
120
+ * file specifically), not a silent "provenance unknown" fallback.
121
+ *
122
+ * @param {string|null|undefined} wavePath — absolute, already-validated path
123
+ * @returns {Promise<{index: Map<string,Object>, formulaRef: string|null, waveName: string|null}>}
124
+ */
125
+ export async function loadWaveIndex(wavePath) {
126
+ const index = new Map();
127
+ if (!wavePath) return { index, formulaRef: null, waveName: null };
128
+
129
+ if (!existsSync(wavePath)) {
130
+ throw inputError(
131
+ 'INPUT_WAVE_NOT_FOUND',
132
+ `--wave file not found: ${wavePath}`,
133
+ 'Check the path. Wave files live under inputs/prompts/*.json — see projects/salt-road/inputs/prompts/wave-v2.json for the shape.'
134
+ );
135
+ }
136
+
137
+ let raw;
138
+ try {
139
+ raw = JSON.parse(await readFile(wavePath, 'utf-8'));
140
+ } catch (err) {
141
+ throw inputError(
142
+ 'INPUT_INVALID_JSON',
143
+ `Failed to parse --wave file ${wavePath}: ${err.message}`,
144
+ 'Check the file for JSON syntax errors.'
145
+ );
146
+ }
147
+
148
+ const items = Array.isArray(raw.items) ? raw.items
149
+ : Array.isArray(raw.subjects) ? raw.subjects
150
+ : null;
151
+
152
+ if (!items) {
153
+ warn(`ingest: --wave file ${wavePath} has no "items" or "subjects" array — no entries will be attached.`);
154
+ return { index, formulaRef: raw.formula_ref || null, waveName: raw.wave || raw.name || null };
155
+ }
156
+
157
+ for (const item of items) {
158
+ if (item && typeof item.id === 'string') index.set(item.id, item);
159
+ }
160
+
161
+ return { index, formulaRef: raw.formula_ref || null, waveName: raw.wave || raw.name || null };
162
+ }
163
+
164
+ /**
165
+ * Load a receipts.jsonl (the shape of projects/salt-road/inbox/generated/
166
+ * set-v2-2026-07-30/receipts.jsonl: one `{id, status, prompt_id, seed, secs}`
167
+ * object per line) into a Map<id, receipt>.
168
+ *
169
+ * Best-effort by design: a missing file yields an empty Map (this is
170
+ * opportunistic enrichment co-located with the images, not a requirement),
171
+ * and a malformed INDIVIDUAL line is skipped with a warning rather than
172
+ * failing the whole load — one corrupt trailing line from an interrupted
173
+ * write must not blank out every other real receipt in the file.
174
+ *
175
+ * @param {string} receiptsPath — absolute path
176
+ * @returns {Promise<Map<string,Object>>}
177
+ */
178
+ export async function loadReceiptsIndex(receiptsPath) {
179
+ const index = new Map();
180
+ if (!receiptsPath || !existsSync(receiptsPath)) return index;
181
+
182
+ const raw = await readFile(receiptsPath, 'utf-8');
183
+ const lines = raw.split('\n');
184
+ for (let i = 0; i < lines.length; i++) {
185
+ const trimmed = lines[i].trim();
186
+ if (!trimmed) continue;
187
+ try {
188
+ const entry = JSON.parse(trimmed);
189
+ if (entry && typeof entry.id === 'string') index.set(entry.id, entry);
190
+ } catch (err) {
191
+ warn(`ingest: ${receiptsPath}:${i + 1} is not valid JSON — line skipped (${err.message})`);
192
+ }
193
+ }
194
+ return index;
195
+ }
196
+
197
+ // ─── Provenance builder ─────────────────────────────────────────────
198
+
199
+ /**
200
+ * Build the provenance block for one ingested image. Additive-only: a
201
+ * field is set exactly when a real source (the --wave entry or the
202
+ * receipts.jsonl entry for this id) supplied it. Nothing here is ever
203
+ * synthesized, interpolated, or defaulted from another record.
204
+ */
205
+ function buildExternalProvenance({ file, sourcePath, waveEntry, waveMeta, receiptEntry, receiptsRelPath }) {
206
+ const prov = {
207
+ source: 'external',
208
+ ingested_at: new Date().toISOString(),
209
+ original_filename: file,
210
+ original_path: sourcePath,
211
+ };
212
+
213
+ const notes = [];
214
+
215
+ if (waveEntry) {
216
+ prov.wave_file = waveMeta.waveRelPath;
217
+ if (waveMeta.waveName) prov.wave_name = waveMeta.waveName;
218
+ if (waveMeta.formulaRef) {
219
+ prov.formula_ref = waveMeta.formulaRef;
220
+ notes.push(
221
+ 'preamble_ref/negative_ref are keys into the wave\'s formula_ref file, not literal prompt/negative text.'
222
+ );
223
+ }
224
+ if (waveEntry.seed !== undefined) prov.seed = waveEntry.seed;
225
+ const promptText = waveEntry.subject ?? waveEntry.prompt;
226
+ if (promptText !== undefined) prov.prompt = promptText;
227
+ if (waveEntry.preamble !== undefined) prov.preamble_ref = waveEntry.preamble;
228
+ if (waveEntry.neg !== undefined) prov.negative_ref = waveEntry.neg;
229
+ if (waveEntry.negExtra !== undefined) prov.negative_extra = waveEntry.negExtra;
230
+ if (waveEntry.cn !== undefined) prov.controlnet = waveEntry.cn;
231
+ if (waveEntry.enriched_by !== undefined) prov.enriched_by = waveEntry.enriched_by;
232
+ } else if (waveMeta.wavePath) {
233
+ notes.push(`No entry for this id was found in --wave file ${waveMeta.waveRelPath}.`);
234
+ }
235
+
236
+ if (receiptEntry) {
237
+ prov.receipts_file = receiptsRelPath;
238
+ if (receiptEntry.prompt_id !== undefined) prov.prompt_id = receiptEntry.prompt_id;
239
+ // Wave seed wins if both are present and disagree; a receipt seed is
240
+ // still useful when the wave lacked one — never overwritten if already set.
241
+ if (receiptEntry.seed !== undefined && prov.seed === undefined) prov.seed = receiptEntry.seed;
242
+ if (receiptEntry.secs !== undefined) prov.generation_seconds = receiptEntry.secs;
243
+ if (receiptEntry.status !== undefined) prov.receipt_status = receiptEntry.status;
244
+ }
245
+
246
+ prov.generation_provenance_known = Boolean(waveEntry || receiptEntry);
247
+ if (!waveEntry && !receiptEntry) {
248
+ notes.push(
249
+ waveMeta.wavePath
250
+ ? 'No matching wave or receipts.jsonl entry for this id — generation details are unknown.'
251
+ : 'No --wave file supplied and no receipts.jsonl found alongside the source images — generation details are unknown.'
252
+ );
253
+ }
254
+ if (notes.length) prov.provenance_note = notes.join(' ');
255
+
256
+ return prov;
257
+ }
258
+
259
+ // ─── Main entry point ────────────────────────────────────────────────
260
+
261
+ /**
262
+ * Ingest a directory of pre-existing images into a project as bare,
263
+ * uncurated candidate records ready for `sdlab curate`.
264
+ *
265
+ * @param {string} projectRoot — absolute project root (already resolved by the caller)
266
+ * @param {string} sourceDir — absolute path to a directory of images (already validated by the caller)
267
+ * @param {Object} [options]
268
+ * @param {string} [options.wavePath] — absolute, already-validated path to a wave JSON file
269
+ * @param {boolean} [options.dryRun]
270
+ * @returns {Promise<{created: string[], skipped: string[], rejected: {file:string, reason:string}[], receiptsFile: string|null}>}
271
+ */
272
+ export async function ingestDirectory(projectRoot, sourceDir, options = {}) {
273
+ const dryRun = Boolean(options.dryRun);
274
+
275
+ if (!existsSync(sourceDir)) {
276
+ throw inputError(
277
+ 'INPUT_SOURCE_NOT_FOUND',
278
+ `Source directory not found: ${sourceDir}`,
279
+ 'Check the path passed to "sdlab ingest <dir>".'
280
+ );
281
+ }
282
+
283
+ const recordsDir = join(projectRoot, 'records');
284
+ const candidatesDir = join(projectRoot, 'outputs', 'candidates');
285
+
286
+ // Wave + receipts are loaded up front, once, before touching any output
287
+ // directory — reading is safe under --dry-run (no mutation).
288
+ const wavePath = options.wavePath || null;
289
+ const { index: waveIndex, formulaRef, waveName } = await loadWaveIndex(wavePath);
290
+ const waveMeta = {
291
+ wavePath,
292
+ waveRelPath: wavePath ? toPortablePath(relative(projectRoot, wavePath)) : null,
293
+ formulaRef,
294
+ waveName,
295
+ };
296
+
297
+ const receiptsPath = join(sourceDir, RECEIPTS_FILENAME);
298
+ const receiptsIndex = await loadReceiptsIndex(receiptsPath);
299
+ const receiptsFound = receiptsIndex.size > 0;
300
+ const receiptsRelPath = receiptsFound ? toPortablePath(relative(projectRoot, receiptsPath)) : null;
301
+ if (receiptsFound) {
302
+ verbose(`ingest: loaded ${receiptsIndex.size} receipt(s) from ${receiptsPath}`);
303
+ }
304
+
305
+ // SDL-M11 discipline: a --dry-run preview must be side-effect-free. Only
306
+ // create output directories on a real run.
307
+ if (!dryRun) {
308
+ await mkdir(recordsDir, { recursive: true });
309
+ await mkdir(candidatesDir, { recursive: true });
310
+ }
311
+
312
+ const entries = (await readdir(sourceDir)).sort();
313
+ const imageFiles = entries.filter((f) => IMAGE_EXTENSIONS.has(extname(f).toLowerCase()));
314
+
315
+ const created = [];
316
+ const skipped = [];
317
+ const rejected = [];
318
+
319
+ for (const file of imageFiles) {
320
+ const ext = extname(file);
321
+ const stem = basename(file, ext);
322
+
323
+ // SDL-H5-style boundary: a filename becoming a record id (and later a
324
+ // path segment for the copied image / record file) is a path-traversal
325
+ // vector. Reuses the exact allowlist lib/export.js already applies at
326
+ // every other record-id-becomes-a-filename boundary in this codebase.
327
+ // Unlike that adapter-layer call site (which hard-throws because an
328
+ // unsafe id THERE means a pipeline-internal invariant already broke),
329
+ // this is a first-touch scan of an arbitrary, externally-named
330
+ // directory — one badly-named file (spaces, unicode, an accidental
331
+ // "..") is the normal case, not an attack. It is rejected and reported
332
+ // so the other 99 files in the same directory still get ingested; the
333
+ // vector is closed identically either way, since no path is ever built
334
+ // from a name that fails this check.
335
+ try {
336
+ assertSafeId(stem, `filename "${file}" in ${sourceDir}`);
337
+ } catch (err) {
338
+ rejected.push({ file, reason: err.message });
339
+ continue;
340
+ }
341
+
342
+ const id = stem;
343
+ const recordPath = join(recordsDir, `${id}.json`);
344
+
345
+ if (existsSync(recordPath)) {
346
+ skipped.push(id);
347
+ continue;
348
+ }
349
+
350
+ if (dryRun) {
351
+ // Preview only — no stat(), no copy, no write. Report what WOULD be
352
+ // created without touching the filesystem beyond the read-only
353
+ // listing/existence checks already done above.
354
+ created.push(id);
355
+ continue;
356
+ }
357
+
358
+ const sourcePath = join(sourceDir, file);
359
+ const destPath = join(candidatesDir, file);
360
+ const assetPath = toPortablePath(join('outputs', 'candidates', file));
361
+
362
+ const fileStat = await stat(sourcePath);
363
+
364
+ if (!existsSync(destPath)) {
365
+ await atomicCopyFile(sourcePath, destPath);
366
+ }
367
+
368
+ const waveEntry = waveIndex.get(id) || null;
369
+ const receiptEntry = receiptsIndex.get(id) || null;
370
+ const provenance = buildExternalProvenance({
371
+ file,
372
+ sourcePath,
373
+ waveEntry,
374
+ waveMeta,
375
+ receiptEntry,
376
+ receiptsRelPath,
377
+ });
378
+
379
+ const record = buildBaseRecord(
380
+ id,
381
+ assetPath,
382
+ { width: null, height: null, bytes: fileStat.size },
383
+ provenance
384
+ );
385
+
386
+ await atomicWriteFile(recordPath, JSON.stringify(record, null, 2) + '\n');
387
+ created.push(id);
388
+ }
389
+
390
+ return {
391
+ created,
392
+ skipped,
393
+ rejected,
394
+ receiptsFile: receiptsFound ? receiptsRelPath : null,
395
+ };
396
+ }
@@ -0,0 +1,220 @@
1
+ /**
2
+ * judgment-provenance.js — tell a judgment somebody MADE from one a script MINTED.
3
+ *
4
+ * Law 2 of this product is "inclusion is explainable": every record carries a
5
+ * reason it was included or excluded. That law is only worth anything if the
6
+ * reason describes the actual image.
7
+ *
8
+ * It currently doesn't, at scale. Measured on `projects/star-freight`
9
+ * (1,182 records) on 2026-08-04:
10
+ *
11
+ * reviewer records distinct explanations max reuse
12
+ * bulk_curate_v2 566 ) 82 across both x144
13
+ * bulk_curate_v1 322 )
14
+ * wave25-script 60 8 x32
15
+ * human:mike 110 110 x1
16
+ * (no judgment) 124 — —
17
+ *
18
+ * 948 of 1,182 records — 80% — carry a rationale that was selected by matching
19
+ * the record id against a prefix regex, not by looking at the image. One string
20
+ * ("On-style gritty space costume. Faction-appropriate material vocabulary and
21
+ * wear level…") appears verbatim on 144 different images. The three scripts
22
+ * that wrote them (`bulk-curate-wave2-5.js`, `bulk-curate-waves11-18.js`,
23
+ * `curate-wave25.js`) were deleted in this repo's Stage A health pass for
24
+ * exactly this reason — but deleting the tool did not retract its output, and
25
+ * that output still flows into eligibility, snapshots, splits, exports and
26
+ * training packages.
27
+ *
28
+ * The `human:mike` row is the control that makes this a finding rather than a
29
+ * guess: 110 records, 110 distinct explanations, nothing reused. Genuine
30
+ * curation in this corpus looks like one rationale per image.
31
+ *
32
+ * ─────────────────────────────────────────────────────────────────────────
33
+ * THIS MODULE READS. IT NEVER WRITES.
34
+ *
35
+ * The Director's ruling (2026-08-04) was "mark, don't touch": make the
36
+ * distinction visible everywhere it matters, and leave the records alone. So
37
+ * there is no migration here, no backfilled flag, no rewritten judgment. The
38
+ * signal is derived at read time from data already on disk, which also means
39
+ * it cannot drift out of sync with the records and needs no re-run when they
40
+ * change.
41
+ * ─────────────────────────────────────────────────────────────────────────
42
+ */
43
+
44
+ /**
45
+ * Reviewer ids known to have been written by a bulk script rather than a
46
+ * person. Matched against `judgment.reviewer`.
47
+ *
48
+ * Deliberately a conservative allowlist of OBSERVED offenders, not a clever
49
+ * heuristic like /script|bot|auto/. A false positive here defames a real
50
+ * human judgment, which is worse than missing one: the corpus-level reuse
51
+ * statistic below catches anything this list doesn't, without needing to
52
+ * guess from a name.
53
+ */
54
+ export const SCRIPT_REVIEWER_PATTERNS = Object.freeze([
55
+ /^bulk_curate(_v\d+)?$/i,
56
+ /^wave\d+-script$/i,
57
+ ]);
58
+
59
+ /** A judgment is machine-minted, human-made, or absent. */
60
+ export const JUDGMENT_ORIGIN = Object.freeze({
61
+ HUMAN: 'human',
62
+ SCRIPT: 'script',
63
+ UNKNOWN: 'unknown',
64
+ NONE: 'none',
65
+ });
66
+
67
+ /**
68
+ * Classify a single record's judgment origin from its reviewer id.
69
+ *
70
+ * @param {object} record
71
+ * @returns {'human'|'script'|'unknown'|'none'}
72
+ */
73
+ export function classifyJudgmentOrigin(record) {
74
+ const judgment = record?.judgment;
75
+ if (!judgment || typeof judgment !== 'object') return JUDGMENT_ORIGIN.NONE;
76
+
77
+ const reviewer = judgment.reviewer;
78
+ if (typeof reviewer !== 'string' || reviewer.trim() === '') {
79
+ return JUDGMENT_ORIGIN.UNKNOWN;
80
+ }
81
+ if (SCRIPT_REVIEWER_PATTERNS.some((re) => re.test(reviewer))) {
82
+ return JUDGMENT_ORIGIN.SCRIPT;
83
+ }
84
+ // `human:<name>` is this corpus's convention for a person.
85
+ if (/^human(:|$)/i.test(reviewer)) return JUDGMENT_ORIGIN.HUMAN;
86
+
87
+ // A reviewer id we don't recognise. NOT assumed human — that assumption is
88
+ // how 948 fabricated rationales passed as curation for months.
89
+ return JUDGMENT_ORIGIN.UNKNOWN;
90
+ }
91
+
92
+ /**
93
+ * Corpus-level rationale-reuse audit.
94
+ *
95
+ * This is the evidence the reviewer-name check cannot give you, and it is the
96
+ * part that generalises: a rationale reused verbatim across many images did
97
+ * not describe any one of them, whoever or whatever wrote it. It catches a
98
+ * future bulk script under a reviewer id nobody has added to the list above.
99
+ *
100
+ * `reuseThreshold` is the count at which sharing stops being coincidence.
101
+ * Two records can legitimately share a terse reason ("out of focus"); 144
102
+ * cannot.
103
+ *
104
+ * @param {object[]} records
105
+ * @param {{reuseThreshold?: number}} [opts]
106
+ */
107
+ export function auditJudgmentProvenance(records, opts = {}) {
108
+ const reuseThreshold = opts.reuseThreshold ?? 3;
109
+
110
+ const byOrigin = { human: 0, script: 0, unknown: 0, none: 0 };
111
+ const byReviewer = new Map();
112
+ const byExplanation = new Map();
113
+
114
+ for (const record of records) {
115
+ const origin = classifyJudgmentOrigin(record);
116
+ byOrigin[origin]++;
117
+
118
+ const judgment = record?.judgment;
119
+ if (!judgment) continue;
120
+
121
+ const reviewer = typeof judgment.reviewer === 'string' ? judgment.reviewer : '(unset)';
122
+ byReviewer.set(reviewer, (byReviewer.get(reviewer) ?? 0) + 1);
123
+
124
+ const explanation = judgment.explanation;
125
+ if (typeof explanation === 'string' && explanation.trim() !== '') {
126
+ const key = explanation.trim();
127
+ if (!byExplanation.has(key)) byExplanation.set(key, []);
128
+ byExplanation.get(key).push(record.id);
129
+ }
130
+ }
131
+
132
+ const reused = [...byExplanation.entries()]
133
+ .filter(([, ids]) => ids.length >= reuseThreshold)
134
+ .map(([explanation, ids]) => ({
135
+ explanation,
136
+ count: ids.length,
137
+ sample_record_ids: ids.slice(0, 5),
138
+ }))
139
+ .sort((a, b) => b.count - a.count);
140
+
141
+ const judgedCount = byOrigin.human + byOrigin.script + byOrigin.unknown;
142
+ const reusedRecordCount = reused.reduce((sum, r) => sum + r.count, 0);
143
+
144
+ return {
145
+ total_records: records.length,
146
+ judged: judgedCount,
147
+ by_origin: byOrigin,
148
+ by_reviewer: Object.fromEntries([...byReviewer.entries()].sort((a, b) => b[1] - a[1])),
149
+ // How many judged records carry a rationale that is not unique to them.
150
+ // This is the honest headline number — it does not depend on recognising
151
+ // a reviewer id.
152
+ reused_rationale_records: reusedRecordCount,
153
+ reused_rationale_ratio: judgedCount > 0 ? reusedRecordCount / judgedCount : 0,
154
+ reuse_threshold: reuseThreshold,
155
+ // Worst offenders first, capped — this is for an operator to read, not a
156
+ // full dump of a 1,182-record corpus.
157
+ top_reused: reused.slice(0, 10),
158
+ // True when any judged record's rationale is shared at or above the
159
+ // threshold. Consumers gate their wording on this rather than re-deriving.
160
+ has_reused_rationales: reusedRecordCount > 0,
161
+ };
162
+ }
163
+
164
+ /**
165
+ * One-line human summary for CLI output. Returns null when there is nothing
166
+ * to report, so a clean corpus prints nothing rather than a reassuring banner
167
+ * nobody reads.
168
+ *
169
+ * @param {ReturnType<typeof auditJudgmentProvenance>} audit
170
+ * @returns {string|null}
171
+ */
172
+ export function formatProvenanceWarning(audit) {
173
+ // ONLY a script-minted judgment is asserted as a Law 2 problem.
174
+ //
175
+ // Reuse alone is NOT evidence of fabrication, and an earlier revision of
176
+ // this function got that wrong. Measured on `projects/tallow-fen`: 54 of
177
+ // 293 rationales are reused, and they are legitimate — two strings, ×28 and
178
+ // ×26, each accurately describing ONE systematic defect across exactly the
179
+ // images that have it ("style on-target but the style descriptor's
180
+ // lantern-light noun literalized"). Reviewers there are `human:mike` and
181
+ // `claude:fable-5`; zero scripts. That is good curation of a bad wave.
182
+ //
183
+ // The heuristic cannot separate "144 images share a canned category blurb
184
+ // chosen by id-prefix" from "28 images share a true description of one
185
+ // shared flaw" — so it must not claim to. A gate that fires when nothing is
186
+ // wrong teaches its reader to ignore it, which is the same defect class as
187
+ // a gate that cannot fire, pointed the other way.
188
+ if (audit.by_origin.script === 0) return null;
189
+
190
+ const pct = Math.round((audit.by_origin.script / Math.max(audit.judged, 1)) * 100);
191
+ let msg =
192
+ `Judgment provenance: ${audit.by_origin.script} of ${audit.judged} judgments (${pct}%) ` +
193
+ `were written by a bulk script, not a reviewer. Those rationales were selected by ` +
194
+ `matching the record id against a pattern — they do not describe the image. ` +
195
+ `Treat them as uncurated for Law 2 purposes.`;
196
+
197
+ if (audit.reused_rationale_records > 0) {
198
+ msg += ` (${audit.reused_rationale_records} rationales are also reused verbatim across records.)`;
199
+ }
200
+ return msg;
201
+ }
202
+
203
+ /**
204
+ * Neutral, non-accusatory reuse note — for corpora with no script reviewer but
205
+ * heavy rationale reuse. Reuse there is usually a real systematic defect
206
+ * described accurately across a wave, which is worth SEEING but is not a
207
+ * finding. Returns null when there is nothing to say.
208
+ *
209
+ * @param {ReturnType<typeof auditJudgmentProvenance>} audit
210
+ * @returns {string|null}
211
+ */
212
+ export function formatReuseNote(audit) {
213
+ if (audit.by_origin.script > 0) return null; // the warning above covers it
214
+ if (audit.reused_rationale_records === 0) return null;
215
+
216
+ const pct = Math.round(audit.reused_rationale_ratio * 100);
217
+ return `${audit.reused_rationale_records} of ${audit.judged} rationales (${pct}%) are shared ` +
218
+ `across records — often one systematic defect described across the wave it affected. ` +
219
+ `Worth a glance, not necessarily a problem.`;
220
+ }