@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.es.md +117 -55
- package/README.fr.md +125 -65
- package/README.hi.md +117 -68
- package/README.it.md +119 -58
- package/README.ja.md +124 -74
- package/README.md +73 -7
- package/README.pt-BR.md +128 -63
- package/README.zh.md +116 -65
- package/bin/sdlab.js +100 -4
- package/lib/adapters/ai-toolkit.js +224 -35
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +94 -11
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/asset-ingest.js +575 -0
- package/lib/asset-masks.js +323 -0
- package/lib/asset-source.js +1176 -0
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-author.js +350 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/canon-review-gate.js +205 -0
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/colab-launcher-guards.js +285 -0
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +16 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +194 -6
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/lane1-render.js +668 -0
- package/lib/measure.js +422 -0
- package/lib/memory-store.js +663 -0
- package/lib/model-licences.js +305 -0
- package/lib/npy-meta.js +130 -0
- package/lib/paths.js +196 -7
- package/lib/png-meta.js +512 -0
- package/lib/records.js +54 -8
- package/lib/reingest-selected.js +113 -17
- package/lib/reingest.js +27 -3
- package/lib/run-manifest.js +159 -25
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/train-runs.js +214 -0
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +3 -0
- package/lib/verify-chain.js +367 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +4 -3
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/asset-ingest.js +160 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-draft.js +82 -0
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +31 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
- package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
- package/scripts/colab/colab_window.sh +587 -0
- package/scripts/colab/lane1_render.py +970 -0
- package/scripts/colab/launch_guards.py +564 -0
- package/scripts/colab/launch_lane1_stub.sh +309 -0
- package/scripts/colab/launch_lane3_stub.sh +366 -0
- package/scripts/colab/remote_runner.py +59 -0
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +182 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +86 -17
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/verify.js +67 -0
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
package/scripts/generate.js
CHANGED
|
@@ -14,14 +14,16 @@ import { writeFile, mkdir } from "node:fs/promises";
|
|
|
14
14
|
import { existsSync } from "node:fs";
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { parseArgs, getProjectName } from "../lib/args.js";
|
|
17
|
-
import {
|
|
17
|
+
import { getProjectRoot, resolveSafeProjectPath } from "../lib/paths.js";
|
|
18
18
|
import { readJsonFile, loadProjectMeta, resolveGpuModel } from "../lib/config.js";
|
|
19
19
|
import { runtimeError, inputError, handleCliError } from "../lib/errors.js";
|
|
20
20
|
import { comfyHealth, submitAndWait, downloadImage } from "../lib/comfyui.js";
|
|
21
21
|
import { pickOutputImage } from "../lib/comfyui-output.js";
|
|
22
|
-
import { buildQwenGraph, QWEN_DEFAULTS } from "../lib/adapters/comfyui-workflows.js";
|
|
22
|
+
import { buildQwenGraph, QWEN_DEFAULTS, resolveQwenLoras } from "../lib/adapters/comfyui-workflows.js";
|
|
23
23
|
import { buildPinning, SEED_POLICIES, createHashCache, flushHashCache } from "../lib/run-manifest.js";
|
|
24
|
-
import {
|
|
24
|
+
import { STUDIO_OWNER, STUDIO_LICENCE, buildDefaultRights } from "../lib/records.js";
|
|
25
|
+
import { resolveCanonLink } from "../lib/generated-provenance.js";
|
|
26
|
+
import { info, result, warn } from "../lib/log.js";
|
|
25
27
|
|
|
26
28
|
/**
|
|
27
29
|
* Format milliseconds as a human ETA string like "6m 12s" or "42s".
|
|
@@ -179,6 +181,34 @@ function buildWorkflow(prompt, negativePrompt, checkpoint, loras, seed, steps, c
|
|
|
179
181
|
return { nodes, saveNodeId: saveId };
|
|
180
182
|
}
|
|
181
183
|
|
|
184
|
+
/**
|
|
185
|
+
* Coordinator addendum (same session as H4/H5): every domain template's
|
|
186
|
+
* example-wave.json ships subject prompts with a literal `[DESCRIBE ...]`
|
|
187
|
+
* placeholder (e.g. "[DESCRIBE YOUR HERO — head and shoulders, ...]") that
|
|
188
|
+
* the user is meant to replace before generating. `sdlab init`'s own
|
|
189
|
+
* printed next-steps run generate before "edit the pack" is ever a
|
|
190
|
+
* mandatory step, so following them literally on an unedited pack burns a
|
|
191
|
+
* GPU wave on template text with no feedback until the images come back
|
|
192
|
+
* looking like a literal instruction string.
|
|
193
|
+
*
|
|
194
|
+
* All 5 domain templates use this exact "[DESCRIBE " shape (verified across
|
|
195
|
+
* templates/domains/*\/inputs/prompts/example-wave.json), so the check is
|
|
196
|
+
* narrow and specific to it — it does NOT flag ordinary SD prompt-editing
|
|
197
|
+
* bracket syntax such as "[word]" or "[from:to:0.5]", neither of which
|
|
198
|
+
* starts with the literal word DESCRIBE.
|
|
199
|
+
*/
|
|
200
|
+
const PLACEHOLDER_PROMPT_RE = /\[DESCRIBE\b/;
|
|
201
|
+
|
|
202
|
+
function findPlaceholderPrompts(jobs) {
|
|
203
|
+
const hits = [];
|
|
204
|
+
for (const job of jobs) {
|
|
205
|
+
if (typeof job.prompt === 'string' && PLACEHOLDER_PROMPT_RE.test(job.prompt)) {
|
|
206
|
+
hits.push(job.assetId);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return hits;
|
|
210
|
+
}
|
|
211
|
+
|
|
182
212
|
/**
|
|
183
213
|
* Detect the input schema and normalize to a unified shape:
|
|
184
214
|
* { lane, style_prefix, negative, defaults, base, jobs:[{assetId, subjectId, variationId, prompt, seed, loras}] }
|
|
@@ -198,7 +228,7 @@ function buildWorkflow(prompt, negativePrompt, checkpoint, loras, seed, steps, c
|
|
|
198
228
|
* `base` resolves: input.base → input.defaults.base → project.json defaults.base
|
|
199
229
|
* → "sdxl". `"qwen-image"` routes to the non-anime Qwen graph.
|
|
200
230
|
*/
|
|
201
|
-
function normalizeGenerationInput(raw, projectMeta) {
|
|
231
|
+
export function normalizeGenerationInput(raw, projectMeta) {
|
|
202
232
|
if (!Array.isArray(raw.subjects) || raw.subjects.length === 0) {
|
|
203
233
|
throw inputError(
|
|
204
234
|
'INPUT_MISSING_FIELDS',
|
|
@@ -232,10 +262,11 @@ function normalizeGenerationInput(raw, projectMeta) {
|
|
|
232
262
|
jobs.push({
|
|
233
263
|
assetId: `${subject.id}_${variation.id}`,
|
|
234
264
|
subjectId: subject.id,
|
|
265
|
+
entryId: subject.entry_id || subject.id,
|
|
235
266
|
variationId: variation.id,
|
|
236
267
|
prompt: compose(subject.prompt),
|
|
237
268
|
seed: baseSeed + k + (variation.seed_offset || 0),
|
|
238
|
-
loras: variation.loras || [],
|
|
269
|
+
loras: variation.loras || subject.loras || [],
|
|
239
270
|
});
|
|
240
271
|
k++;
|
|
241
272
|
}
|
|
@@ -247,14 +278,16 @@ function normalizeGenerationInput(raw, projectMeta) {
|
|
|
247
278
|
const negative = raw.negative ?? defaults.negative ?? '';
|
|
248
279
|
for (const subject of raw.subjects) {
|
|
249
280
|
const nvar = Number.isFinite(subject.variations) ? subject.variations : 1;
|
|
281
|
+
const subjectLoras = subject.loras || defaults.loras || raw.loras || [];
|
|
250
282
|
for (let v = 0; v < nvar; v++) {
|
|
251
283
|
jobs.push({
|
|
252
284
|
assetId: `${subject.id}_v${v}`,
|
|
253
285
|
subjectId: subject.id,
|
|
286
|
+
entryId: subject.entry_id || subject.id,
|
|
254
287
|
variationId: `v${v}`,
|
|
255
288
|
prompt: compose(subject.prompt),
|
|
256
289
|
seed: baseSeed + k,
|
|
257
|
-
loras:
|
|
290
|
+
loras: subjectLoras,
|
|
258
291
|
});
|
|
259
292
|
k++;
|
|
260
293
|
}
|
|
@@ -264,7 +297,7 @@ function normalizeGenerationInput(raw, projectMeta) {
|
|
|
264
297
|
|
|
265
298
|
export async function run(argv = process.argv.slice(2)) {
|
|
266
299
|
const projectName = getProjectName(argv);
|
|
267
|
-
const GAME_ROOT =
|
|
300
|
+
const GAME_ROOT = getProjectRoot(projectName);
|
|
268
301
|
const COMFY_URL = process.env.COMFY_URL || "http://127.0.0.1:8188";
|
|
269
302
|
|
|
270
303
|
const parsed = parseArgs(argv, {
|
|
@@ -323,6 +356,17 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
323
356
|
console.log(` Total candidates: ${pack.jobs.length}`);
|
|
324
357
|
console.log("");
|
|
325
358
|
|
|
359
|
+
// Coordinator addendum: catch unedited template text before it burns GPU
|
|
360
|
+
// time (or, under --dry-run, before the user thinks the pack is ready).
|
|
361
|
+
const placeholderHits = findPlaceholderPrompts(pack.jobs);
|
|
362
|
+
if (placeholderHits.length > 0) {
|
|
363
|
+
warn(
|
|
364
|
+
`${placeholderHits.length} of ${pack.jobs.length} prompt(s) still contain unedited "[DESCRIBE ...]" template ` +
|
|
365
|
+
`text (e.g. "${placeholderHits[0]}"). Edit the prompt pack before generating for real — otherwise you'll spend ` +
|
|
366
|
+
`GPU time rendering placeholder instructions instead of your subject.`
|
|
367
|
+
);
|
|
368
|
+
}
|
|
369
|
+
|
|
326
370
|
if (!dryRun) {
|
|
327
371
|
const online = await comfyHealth(COMFY_URL);
|
|
328
372
|
if (!online) {
|
|
@@ -331,8 +375,12 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
331
375
|
console.log("\x1b[32m✓\x1b[0m ComfyUI online");
|
|
332
376
|
}
|
|
333
377
|
|
|
334
|
-
|
|
335
|
-
|
|
378
|
+
// SDL-M11: only create output dirs on a real run — a --dry-run preview must
|
|
379
|
+
// be side-effect-free (README.md promises "preview the effect first").
|
|
380
|
+
if (!dryRun) {
|
|
381
|
+
await mkdir(join(GAME_ROOT, "outputs/candidates"), { recursive: true });
|
|
382
|
+
await mkdir(join(GAME_ROOT, "records"), { recursive: true });
|
|
383
|
+
}
|
|
336
384
|
|
|
337
385
|
let generated = 0;
|
|
338
386
|
let errors = 0;
|
|
@@ -341,7 +389,10 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
341
389
|
const runStartMs = Date.now();
|
|
342
390
|
|
|
343
391
|
for (const job of pack.jobs) {
|
|
344
|
-
const { assetId, prompt: fullPrompt, seed, loras } = job;
|
|
392
|
+
const { assetId, entryId, prompt: fullPrompt, seed, loras } = job;
|
|
393
|
+
const effectiveLoras = isQwen
|
|
394
|
+
? resolveQwenLoras({ projectDefaults: d, loras })
|
|
395
|
+
: loras;
|
|
345
396
|
|
|
346
397
|
// --resume: skip jobs whose record + image are both already on disk.
|
|
347
398
|
// Seeds are precomputed per job index, so skipping never shifts later seeds.
|
|
@@ -356,7 +407,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
356
407
|
}
|
|
357
408
|
}
|
|
358
409
|
|
|
359
|
-
console.log(` [${generated + 1}/${totalExpected}] ${assetId} (seed: ${seed}, loras: ${
|
|
410
|
+
console.log(` [${generated + 1}/${totalExpected}] ${assetId} (seed: ${seed}, loras: ${effectiveLoras.length})`);
|
|
360
411
|
|
|
361
412
|
if (dryRun) {
|
|
362
413
|
generated++;
|
|
@@ -381,12 +432,12 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
381
432
|
vae: d.vae || QWEN_DEFAULTS.vae,
|
|
382
433
|
clipType: d.clip_type || QWEN_DEFAULTS.clip_type,
|
|
383
434
|
weightDtype: d.weight_dtype || QWEN_DEFAULTS.weight_dtype,
|
|
384
|
-
loras:
|
|
435
|
+
loras: effectiveLoras,
|
|
385
436
|
filenamePrefix: 'sdl',
|
|
386
437
|
})
|
|
387
438
|
: buildWorkflow(
|
|
388
439
|
fullPrompt, pack.negative,
|
|
389
|
-
d.checkpoint,
|
|
440
|
+
d.checkpoint, effectiveLoras, seed,
|
|
390
441
|
resolved.steps, resolved.cfg, resolved.sampler, resolved.scheduler,
|
|
391
442
|
resolved.width, resolved.height,
|
|
392
443
|
);
|
|
@@ -409,9 +460,16 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
409
460
|
const destPath = `outputs/candidates/${assetId}.png`;
|
|
410
461
|
await writeFile(join(GAME_ROOT, destPath), imgData);
|
|
411
462
|
|
|
463
|
+
const canonLink = resolveCanonLink(GAME_ROOT, {
|
|
464
|
+
canonSha: raw.canon_sha || null,
|
|
465
|
+
entryId,
|
|
466
|
+
});
|
|
467
|
+
|
|
412
468
|
// Write provenance record (base-aware: Qwen records unet/clip/vae/shift,
|
|
413
469
|
// SDXL records checkpoint/loras).
|
|
414
470
|
const provenance = {
|
|
471
|
+
owner: STUDIO_OWNER,
|
|
472
|
+
licence: STUDIO_LICENCE,
|
|
415
473
|
workflow_id: pack.lane,
|
|
416
474
|
workflow_version: "1.0.0",
|
|
417
475
|
base,
|
|
@@ -427,16 +485,23 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
427
485
|
generation_time_ms: elapsed,
|
|
428
486
|
gpu_model: gpuModel,
|
|
429
487
|
batch_index: generated,
|
|
488
|
+
...(canonLink
|
|
489
|
+
? {
|
|
490
|
+
canon_sha: canonLink.canon_sha,
|
|
491
|
+
entry_id: canonLink.entry_id,
|
|
492
|
+
entry_sha256: canonLink.entry_sha256,
|
|
493
|
+
}
|
|
494
|
+
: {}),
|
|
430
495
|
};
|
|
431
496
|
if (isQwen) {
|
|
432
497
|
provenance.unet = d.unet || QWEN_DEFAULTS.unet;
|
|
433
498
|
provenance.clip = d.clip || QWEN_DEFAULTS.clip;
|
|
434
499
|
provenance.vae = d.vae || QWEN_DEFAULTS.vae;
|
|
435
500
|
provenance.shift = d.shift ?? QWEN_DEFAULTS.shift;
|
|
436
|
-
provenance.loras =
|
|
501
|
+
provenance.loras = effectiveLoras;
|
|
437
502
|
} else {
|
|
438
503
|
provenance.checkpoint = d.checkpoint;
|
|
439
|
-
provenance.loras =
|
|
504
|
+
provenance.loras = effectiveLoras;
|
|
440
505
|
}
|
|
441
506
|
|
|
442
507
|
// PIN_PER_STEP: pin the exact graph + model/LoRA content identity + seed
|
|
@@ -447,21 +512,25 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
447
512
|
models: isQwen
|
|
448
513
|
? { unet: provenance.unet, clip: provenance.clip, vae: provenance.vae }
|
|
449
514
|
: { checkpoint: d.checkpoint },
|
|
450
|
-
loras:
|
|
515
|
+
loras: effectiveLoras,
|
|
451
516
|
seedPolicy: SEED_POLICIES.BASE_INCREMENT,
|
|
452
517
|
hashModels,
|
|
453
518
|
cache: hashCache,
|
|
454
519
|
});
|
|
520
|
+
provenance.comfy_workflow_sha = provenance.pinning.comfy_workflow_sha;
|
|
455
521
|
|
|
456
522
|
const record = {
|
|
457
523
|
id: assetId,
|
|
458
524
|
schema_version: "1.0.0",
|
|
459
525
|
created_at: new Date().toISOString(),
|
|
526
|
+
owner: STUDIO_OWNER,
|
|
527
|
+
licence: STUDIO_LICENCE,
|
|
528
|
+
rights: buildDefaultRights(),
|
|
460
529
|
asset_path: destPath,
|
|
461
530
|
image: { format: "png", width: resolved.width, height: resolved.height, bytes: imgData.length },
|
|
462
531
|
provenance,
|
|
463
532
|
judgment: null,
|
|
464
|
-
canon:
|
|
533
|
+
canon: canonLink,
|
|
465
534
|
iteration: null,
|
|
466
535
|
};
|
|
467
536
|
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { join } from 'node:path';
|
|
13
13
|
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
14
|
-
import {
|
|
14
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
15
15
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
16
16
|
import { listTrainingManifests } from '../lib/training-manifests.js';
|
|
17
17
|
import { buildImplementationPack, listImplementationPacks, loadImplementationPack } from '../lib/implementation-packs.js';
|
|
@@ -26,7 +26,7 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
26
26
|
});
|
|
27
27
|
|
|
28
28
|
const projectName = flags.project || getProjectName(argv);
|
|
29
|
-
const projectRoot =
|
|
29
|
+
const projectRoot = getProjectRoot(projectName);
|
|
30
30
|
const subcommand = positionals.find(a => !a.startsWith('impl-') && !a.startsWith('tm-')) || 'list';
|
|
31
31
|
|
|
32
32
|
if (subcommand === 'build') {
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* ingest.js — Bring a directory of pre-existing images into the pipeline
|
|
5
|
+
* as bare, uncurated candidate records.
|
|
6
|
+
*
|
|
7
|
+
* Closes the gap where generation happened OUTSIDE sdlab entirely (a hand
|
|
8
|
+
* batch, external Python against Comfy Cloud, salvage from somewhere else)
|
|
9
|
+
* and there was no CLI path to get those images into records/ for review.
|
|
10
|
+
* Unlike `sdlab reingest generated` (which requires a trained LoRA's
|
|
11
|
+
* training manifest) or `sdlab reingest selected` (which requires a
|
|
12
|
+
* selection id only the production loop can mint), this takes any folder
|
|
13
|
+
* of images and any project — no pipeline history required.
|
|
14
|
+
*
|
|
15
|
+
* Usage:
|
|
16
|
+
* sdlab ingest <dir> --project <name>
|
|
17
|
+
* sdlab ingest <dir> --project <name> --wave inputs/prompts/wave-v2.json
|
|
18
|
+
* sdlab ingest <dir> --project <name> --dry-run
|
|
19
|
+
* sdlab ingest <dir> --project <name> --json
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
23
|
+
import { REPO_ROOT, getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
|
|
24
|
+
import { inputError, handleCliError } from '../lib/errors.js';
|
|
25
|
+
import { ingestDirectory } from '../lib/ingest.js';
|
|
26
|
+
import { result } from '../lib/log.js';
|
|
27
|
+
|
|
28
|
+
const USAGE = `Usage: sdlab ingest <dir> --project <name> [--wave <wave.json>] [--dry-run] [--json]
|
|
29
|
+
|
|
30
|
+
Scan <dir> for images and create bare candidate records (judgment: null,
|
|
31
|
+
canon: null, provenance.source: "external") ready for "sdlab curate".
|
|
32
|
+
|
|
33
|
+
Positional:
|
|
34
|
+
dir Directory of images to ingest (png/jpg/jpeg/webp)
|
|
35
|
+
|
|
36
|
+
Flags:
|
|
37
|
+
--project <name> Project to operate on
|
|
38
|
+
--wave <path> Optional wave JSON (inputs/prompts/*.json shape) to
|
|
39
|
+
attach real generation provenance (seed, prompt, ...)
|
|
40
|
+
by matching each image's filename stem to an item id.
|
|
41
|
+
A sibling receipts.jsonl next to <dir> is picked up
|
|
42
|
+
automatically when present (prompt_id, seconds, status).
|
|
43
|
+
--dry-run Preview what would be created — no files written
|
|
44
|
+
--json Emit the result as JSON on stdout
|
|
45
|
+
|
|
46
|
+
Next steps after ingesting:
|
|
47
|
+
1. Review: sdlab curate <id> approved/rejected "explanation"
|
|
48
|
+
2. Bind: sdlab bind --project <name>`;
|
|
49
|
+
|
|
50
|
+
export async function run(argv = process.argv.slice(2)) {
|
|
51
|
+
if (argv.includes('--help') || argv.includes('-h')) {
|
|
52
|
+
console.log(USAGE);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const { flags, positionals } = parseArgs(argv, {
|
|
57
|
+
flags: {
|
|
58
|
+
project: { type: 'string' },
|
|
59
|
+
wave: { type: 'string' },
|
|
60
|
+
'dry-run': { type: 'boolean', default: false },
|
|
61
|
+
json: { type: 'boolean', default: false },
|
|
62
|
+
},
|
|
63
|
+
deprecated: { game: 'project' },
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const dirArg = positionals[0];
|
|
67
|
+
if (!dirArg) {
|
|
68
|
+
throw inputError(
|
|
69
|
+
'INPUT_MISSING_ARGS',
|
|
70
|
+
'A source directory is required.',
|
|
71
|
+
USAGE
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const projectName = flags.project || getProjectName(argv);
|
|
76
|
+
const projectRoot = getProjectRoot(projectName);
|
|
77
|
+
const dryRun = flags['dry-run'] || flags.dryRun;
|
|
78
|
+
|
|
79
|
+
// Mirrors scripts/reingest.js's --source handling: the source directory
|
|
80
|
+
// of pre-existing images is allowed to live anywhere under the workspace
|
|
81
|
+
// root, not just inside this specific project (e.g. a shared staging
|
|
82
|
+
// inbox), while still refusing to escape the workspace entirely.
|
|
83
|
+
const safeSourceDir = resolveSafeProjectPath(projectRoot, dirArg, { baseRoot: REPO_ROOT, flagName: 'dir' });
|
|
84
|
+
|
|
85
|
+
// Mirrors scripts/generate.js's pack-path handling: a --wave file is
|
|
86
|
+
// expected to live inside THIS project (inputs/prompts/*.json), so its
|
|
87
|
+
// containment root defaults to the project root, not the whole workspace.
|
|
88
|
+
const safeWavePath = flags.wave
|
|
89
|
+
? resolveSafeProjectPath(projectRoot, flags.wave, { flagName: 'wave' })
|
|
90
|
+
: null;
|
|
91
|
+
|
|
92
|
+
if (!flags.json) {
|
|
93
|
+
console.log(`\x1b[1msdlab ingest\x1b[0m — ${projectName}`);
|
|
94
|
+
console.log(` Source: ${dirArg}`);
|
|
95
|
+
if (flags.wave) console.log(` Wave: ${flags.wave}`);
|
|
96
|
+
if (dryRun) console.log(` Mode: DRY RUN (no files written)`);
|
|
97
|
+
console.log('');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const { created, skipped, rejected, receiptsFile } = await ingestDirectory(projectRoot, safeSourceDir, {
|
|
101
|
+
wavePath: safeWavePath,
|
|
102
|
+
dryRun,
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
if (flags.json) {
|
|
106
|
+
console.log(JSON.stringify({ created, skipped, rejected, receiptsFile, dryRun }, null, 2));
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (receiptsFile) {
|
|
111
|
+
console.log(` \x1b[2mreceipts: ${receiptsFile}\x1b[0m`);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
console.log(` \x1b[32m✓\x1b[0m Created: ${created.length} record(s)${dryRun ? ' (dry run — nothing written)' : ''}`);
|
|
115
|
+
if (created.length > 0 && created.length <= 25) {
|
|
116
|
+
for (const id of created) console.log(` ✓ ${id}`);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (skipped.length > 0) {
|
|
120
|
+
console.log(` Skipped: ${skipped.length} (record already exists)`);
|
|
121
|
+
if (skipped.length <= 25) {
|
|
122
|
+
for (const id of skipped) console.log(` ○ ${id}`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (rejected.length > 0) {
|
|
127
|
+
console.log(` \x1b[33m⚠\x1b[0m Rejected: ${rejected.length} (unsafe id derived from filename — rename and re-run)`);
|
|
128
|
+
for (const r of rejected) console.log(` ✗ ${r.file}: ${r.reason}`);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (!dryRun && created.length > 0) {
|
|
132
|
+
console.log('');
|
|
133
|
+
result(`Candidates: outputs/candidates/`);
|
|
134
|
+
result(`Records: records/`);
|
|
135
|
+
console.log('');
|
|
136
|
+
console.log(' Next steps:');
|
|
137
|
+
console.log(' 1. Review: sdlab curate <id> approved/rejected "explanation"');
|
|
138
|
+
console.log(' 2. Bind: sdlab bind --project ' + projectName);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Direct execution guard
|
|
143
|
+
if (process.argv[1] && (process.argv[1].endsWith('ingest.js') || process.argv[1].endsWith('ingest'))) {
|
|
144
|
+
run().catch(handleCliError);
|
|
145
|
+
}
|
package/scripts/init.js
CHANGED
|
@@ -12,10 +12,16 @@
|
|
|
12
12
|
import { mkdir, readdir, readFile, writeFile, copyFile } from 'node:fs/promises';
|
|
13
13
|
import { join, basename } from 'node:path';
|
|
14
14
|
import { existsSync } from 'node:fs';
|
|
15
|
-
import { REPO_ROOT } from '../lib/paths.js';
|
|
15
|
+
import { REPO_ROOT, getPackageRoot } from '../lib/paths.js';
|
|
16
16
|
import { inputError, handleCliError } from '../lib/errors.js';
|
|
17
|
+
import { takeFlagValue } from '../lib/args.js';
|
|
17
18
|
|
|
18
|
-
|
|
19
|
+
// Two DIFFERENT roots — see getPackageRoot() in lib/paths.js.
|
|
20
|
+
// `templates/` ships inside the npm tarball, so it is package-relative;
|
|
21
|
+
// `projects/` is the user's own work, so it is workspace-relative. In a repo
|
|
22
|
+
// checkout these coincide, which is why resolving both against REPO_ROOT
|
|
23
|
+
// passed every in-repo test while breaking `init` for every installed user.
|
|
24
|
+
const TEMPLATES_DIR = join(getPackageRoot(), 'templates');
|
|
19
25
|
const PROJECTS_DIR = join(REPO_ROOT, 'projects');
|
|
20
26
|
|
|
21
27
|
const PROJECT_DIRS = [
|
|
@@ -64,11 +70,15 @@ async function listDomains() {
|
|
|
64
70
|
export async function run(argv = process.argv.slice(2)) {
|
|
65
71
|
const projectName = argv.find(a => !a.startsWith('--'));
|
|
66
72
|
|
|
67
|
-
// Parse --domain
|
|
73
|
+
// Parse --domain. H4: previously a hand-rolled `argv.indexOf('--domain')`
|
|
74
|
+
// that only recognized the space-separated form — `sdlab init foo
|
|
75
|
+
// --domain=game-art` silently fell through to the 'generic' default
|
|
76
|
+
// because no argv token was EXACTLY "--domain". takeFlagValue() (already
|
|
77
|
+
// hardened for both `--flag value` and `--flag=value`) closes that gap.
|
|
68
78
|
let domain = 'generic';
|
|
69
|
-
const
|
|
70
|
-
if (
|
|
71
|
-
domain =
|
|
79
|
+
const domainValue = takeFlagValue(argv, 'domain');
|
|
80
|
+
if (domainValue !== undefined) {
|
|
81
|
+
domain = domainValue;
|
|
72
82
|
}
|
|
73
83
|
|
|
74
84
|
if (!projectName) {
|
|
@@ -133,6 +143,34 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
133
143
|
}
|
|
134
144
|
console.log(` \x1b[32m✓\x1b[0m Created ${PROJECT_DIRS.length} directories`);
|
|
135
145
|
|
|
146
|
+
// SDL-M6: the generate → curate → reingest loop fills inbox/generated/ and
|
|
147
|
+
// the outputs/ subtrees with PNGs (see lib/reingest-selected.js), but this
|
|
148
|
+
// repo's OWN .gitignore (which happens to cover those paths in dev) is not
|
|
149
|
+
// in the npm `files` allow-list and ships to nobody. Without a per-project
|
|
150
|
+
// .gitignore, every downstream `sdlab init` user stages regenerable blobs
|
|
151
|
+
// by construction — measured at 55 MB of untracked PNGs in this repo before
|
|
152
|
+
// this fix. Never clobber a .gitignore the user already created by hand.
|
|
153
|
+
const gitignorePath = join(projectDir, '.gitignore');
|
|
154
|
+
if (!existsSync(gitignorePath)) {
|
|
155
|
+
const gitignoreLines = [
|
|
156
|
+
'# Generated by `sdlab init` — regenerable blobs for this project.',
|
|
157
|
+
'# Records/config JSON are tracked; only pixels and derived artifacts are not.',
|
|
158
|
+
'',
|
|
159
|
+
'inbox/**/*.png',
|
|
160
|
+
'inbox/**/*.jpg',
|
|
161
|
+
'inbox/**/*.jpeg',
|
|
162
|
+
'inbox/**/*.webp',
|
|
163
|
+
'outputs/**',
|
|
164
|
+
'exports/',
|
|
165
|
+
'snapshots/',
|
|
166
|
+
'splits/',
|
|
167
|
+
'canon-build/*/',
|
|
168
|
+
'',
|
|
169
|
+
];
|
|
170
|
+
await writeFile(gitignorePath, gitignoreLines.join('\n'));
|
|
171
|
+
console.log(` \x1b[32m✓\x1b[0m .gitignore (regenerable outputs/inbox/exports/snapshots/splits)`);
|
|
172
|
+
}
|
|
173
|
+
|
|
136
174
|
// Copy config files from domain template or generate generic ones
|
|
137
175
|
const configFiles = ['project.json', 'constitution.json', 'lanes.json', 'rubric.json', 'terminology.json'];
|
|
138
176
|
|
|
@@ -331,21 +369,43 @@ export async function run(argv = process.argv.slice(2)) {
|
|
|
331
369
|
console.log('');
|
|
332
370
|
console.log(`\x1b[32m✓\x1b[0m Project "${projectName}" initialized`);
|
|
333
371
|
console.log('');
|
|
372
|
+
// Two DIFFERENT path bases are printed in this block, for two DIFFERENT
|
|
373
|
+
// audiences — do not collapse them back into one helper:
|
|
374
|
+
// - "Edit <file>" / "Full quick-start" lines are opened by the user in
|
|
375
|
+
// an editor/file-browser from wherever they ran `init` (the workspace
|
|
376
|
+
// root), so they need workspace-relative paths: projects/<name>/....
|
|
377
|
+
// - The `sdlab generate <pack>` line's argument is consumed by
|
|
378
|
+
// generate.js via resolveSafeProjectPath(GAME_ROOT, packPath) — GAME_ROOT
|
|
379
|
+
// IS projects/<name>, so that path must be PROJECT-relative
|
|
380
|
+
// (inputs/prompts/...), not workspace-relative. Printing the
|
|
381
|
+
// workspace-relative form here doubles the path
|
|
382
|
+
// (projects/<name>/projects/<name>/...) and generate.js 404s with
|
|
383
|
+
// INPUT_FILE_NOT_FOUND. Regression introduced in 27c42c6 while fixing
|
|
384
|
+
// the sibling "Edit" lines (which legitimately needed the workspace-
|
|
385
|
+
// relative switch) — see tests/cli-scripts/init-generate-path.test.js.
|
|
386
|
+
const rel = (...segments) => ['projects', projectName, ...segments].join('/');
|
|
387
|
+
const projectRel = (...segments) => segments.join('/');
|
|
388
|
+
|
|
334
389
|
console.log('Next steps:');
|
|
335
390
|
let step = 1;
|
|
336
391
|
if (wroteCanon['constitution.md']) {
|
|
337
|
-
console.log(` ${step++}. Edit ${
|
|
392
|
+
console.log(` ${step++}. Edit ${rel('canon', 'constitution.md')}`);
|
|
393
|
+
console.log(` Describe your visual rules in prose — this is the canon the pipeline binds to.`);
|
|
338
394
|
}
|
|
339
|
-
console.log(` ${step++}. Edit ${
|
|
395
|
+
console.log(` ${step++}. Edit ${rel('constitution.json')}`);
|
|
396
|
+
console.log(` Encode those rules as machine-checkable entries for canon-bind.`);
|
|
340
397
|
if (wroteExamplePack) {
|
|
341
|
-
console.log(` ${step++}.
|
|
398
|
+
console.log(` ${step++}. Generate your first candidates:`);
|
|
399
|
+
console.log(` sdlab generate ${projectRel('inputs', 'prompts', 'example-wave.json')} --project ${projectName}`);
|
|
342
400
|
} else {
|
|
343
|
-
console.log(` ${step++}.
|
|
344
|
-
console.log(` ${step++}.
|
|
401
|
+
console.log(` ${step++}. Add a prompt pack under ${rel('inputs', 'prompts')}/`);
|
|
402
|
+
console.log(` ${step++}. Generate your first candidates:`);
|
|
403
|
+
console.log(` sdlab generate <pack> --project ${projectName}`);
|
|
345
404
|
}
|
|
346
|
-
console.log(` ${step++}.
|
|
405
|
+
console.log(` ${step++}. Check the project is wired correctly:`);
|
|
406
|
+
console.log(` sdlab project doctor --project ${projectName}`);
|
|
347
407
|
console.log('');
|
|
348
|
-
console.log(`
|
|
408
|
+
console.log(`Full quick-start: ${rel('README.md')}`);
|
|
349
409
|
}
|
|
350
410
|
|
|
351
411
|
// Direct execution guard
|