@mcptoolshop/style-dataset-lab 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -1
- package/README.es.md +40 -19
- package/README.fr.md +74 -53
- package/README.hi.md +67 -46
- package/README.it.md +42 -21
- package/README.ja.md +44 -23
- package/README.md +25 -4
- package/README.pt-BR.md +42 -21
- package/README.zh.md +66 -45
- package/bin/sdlab.js +51 -4
- package/lib/adapters/ai-toolkit.js +37 -5
- package/lib/adapters/comfyui-runner.js +114 -27
- package/lib/adapters/comfyui-workflows.js +10 -2
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +82 -3
- package/lib/critique-render.js +45 -0
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +69 -16
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +16 -1
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/measure.js +422 -0
- package/lib/paths.js +196 -7
- package/lib/reingest-selected.js +98 -16
- package/lib/run-manifest.js +394 -0
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +121 -16
- package/lib/split.js +364 -48
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +67 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +3 -2
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +27 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +48 -8
- package/scripts/curate.js +23 -2
- package/scripts/doctor.js +181 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +72 -5
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +334 -13
- package/scripts/reingest.js +2 -2
- package/scripts/run-generate.js +3 -0
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
package/bin/sdlab.js
CHANGED
|
@@ -18,6 +18,21 @@ const COMMANDS = {
|
|
|
18
18
|
'generate:controlnet': '../scripts/generate-controlnet.js',
|
|
19
19
|
'generate:ipadapter': '../scripts/generate-ipadapter.js',
|
|
20
20
|
'curate': '../scripts/curate.js',
|
|
21
|
+
// Bring images that were generated OUTSIDE sdlab into the pipeline. The
|
|
22
|
+
// documented production loop (brief→run→critique→batch→select→reingest) has
|
|
23
|
+
// never run on any project in this repo — real generation happens in
|
|
24
|
+
// external Python against Comfy Cloud, and until now those images had no
|
|
25
|
+
// CLI path in. `reingest generated` needs a trained LoRA's manifest;
|
|
26
|
+
// `reingest selected` needs a selection only the loop can mint.
|
|
27
|
+
'ingest': '../scripts/ingest.js',
|
|
28
|
+
// Contact sheet over ANY directory of candidates. The renderer already
|
|
29
|
+
// existed (lib/batch-sheet-render.js) but was reachable only via
|
|
30
|
+
// `batch sheet`, which needs a batch manifest nothing has ever produced —
|
|
31
|
+
// so five hand-rolled PIL contact-sheet scripts grew up in projects/ instead.
|
|
32
|
+
'sheet': '../scripts/sheet.js',
|
|
33
|
+
// Deterministic pixel measurement. Nothing in this tool opened an image
|
|
34
|
+
// before this: critique scores brief TEXT, not pixels.
|
|
35
|
+
'measure': '../scripts/measure.js',
|
|
21
36
|
'compare': '../scripts/compare.js',
|
|
22
37
|
'bind': '../scripts/canon-bind.js',
|
|
23
38
|
'canon-bind': '../scripts/canon-bind.js',
|
|
@@ -362,7 +377,10 @@ function printHelp() {
|
|
|
362
377
|
console.log(' generate:identity Generate named-subject identity images');
|
|
363
378
|
console.log(' generate:controlnet ControlNet-guided structural generation');
|
|
364
379
|
console.log(' generate:ipadapter IP-Adapter reference-guided generation');
|
|
380
|
+
console.log(' ingest <dir> Ingest externally-generated images as candidates');
|
|
365
381
|
console.log(' curate Move candidate to approved/rejected/borderline');
|
|
382
|
+
console.log(' sheet [<dir>] Render an HTML contact sheet for visual review');
|
|
383
|
+
console.log(' measure <target> Measure candidate pixels (palette, texture)');
|
|
366
384
|
console.log(' compare Record pairwise A-vs-B comparison');
|
|
367
385
|
console.log(' canon-bind Bind approved records to constitution rules');
|
|
368
386
|
console.log(' ("bind" is a short alias for canon-bind)');
|
|
@@ -445,15 +463,19 @@ function printHelp() {
|
|
|
445
463
|
console.log('Examples:');
|
|
446
464
|
console.log(' sdlab init my-project --domain character-design');
|
|
447
465
|
console.log(' sdlab generate --help');
|
|
448
|
-
console.log(' sdlab snapshot create --project
|
|
449
|
-
console.log(' sdlab split build --dry-run --project
|
|
450
|
-
console.log(' sdlab training-package build --debug --project
|
|
466
|
+
console.log(' sdlab snapshot create --project my-project');
|
|
467
|
+
console.log(' sdlab split build --dry-run --project my-project');
|
|
468
|
+
console.log(' sdlab training-package build --debug --project my-project');
|
|
451
469
|
}
|
|
452
470
|
|
|
453
471
|
// All top-level command tokens users can type (including two-word heads).
|
|
454
472
|
function allKnownCommands() {
|
|
455
473
|
const oneWord = Object.keys(COMMANDS);
|
|
456
|
-
|
|
474
|
+
// SDL-L8: 'canon' is a real two-word namespace (see CANON_COMMANDS / the
|
|
475
|
+
// `namespaces` table in main()) but was missing from this list, so a typo
|
|
476
|
+
// like "sdlab cannon build" had no candidate to suggest against — every
|
|
477
|
+
// other namespace head got a "Did you mean ...?" hint, canon did not.
|
|
478
|
+
const heads = ['project', 'workflow', 'brief', 'run', 'critique', 'refine', 'batch', 'select', 'selection', 'canon'];
|
|
457
479
|
return [...new Set([...oneWord, ...heads])];
|
|
458
480
|
}
|
|
459
481
|
|
|
@@ -529,6 +551,31 @@ async function main() {
|
|
|
529
551
|
}
|
|
530
552
|
}
|
|
531
553
|
|
|
554
|
+
// SDL-L2: a bare or invalid namespace verb ("sdlab workflow", "sdlab batch
|
|
555
|
+
// bogus"). These heads only exist as two-word commands — falling through to
|
|
556
|
+
// the generic COMMANDS[command] lookup below used to hand `command` itself
|
|
557
|
+
// to findClosest() against a corpus that includes that same head, so the
|
|
558
|
+
// suggester matched itself at distance 0 and printed "Unknown command:
|
|
559
|
+
// workflow" / 'Did you mean "workflow"?' — a hint that echoes the very
|
|
560
|
+
// thing the user just typed. List the real subcommands instead.
|
|
561
|
+
for (const { head, table } of namespaces) {
|
|
562
|
+
if (command === head) {
|
|
563
|
+
const subs = Object.keys(table);
|
|
564
|
+
if (!args[1] || args[1] === '--help' || args[1] === '-h') {
|
|
565
|
+
console.log(`sdlab ${head} <${subs.join('|')}>\n`);
|
|
566
|
+
console.log('Subcommands:');
|
|
567
|
+
for (const s of subs) console.log(` ${s}`);
|
|
568
|
+
console.log(`\nRun "sdlab ${head} <subcommand> --help" for details.`);
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
571
|
+
const suggestion = findClosest(args[1], subs);
|
|
572
|
+
const hint = suggestion
|
|
573
|
+
? `Did you mean "sdlab ${head} ${suggestion}"?`
|
|
574
|
+
: `Try: ${subs.map(s => `sdlab ${head} ${s}`).join(', ')}`;
|
|
575
|
+
throw inputError('INPUT_UNKNOWN_SUBCOMMAND', `Unknown ${head} subcommand: ${args[1]}`, hint);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
532
579
|
// "critique" — bare or "critique show"
|
|
533
580
|
if (command === 'critique') {
|
|
534
581
|
if (args[1] && CRITIQUE_COMMANDS[args[1]]) {
|
|
@@ -22,6 +22,8 @@ import YAML from 'yaml';
|
|
|
22
22
|
import { deriveStyleTrigger } from '../captions.js';
|
|
23
23
|
import { groupRowsByPartition } from '../rows.js';
|
|
24
24
|
import { inputError } from '../errors.js';
|
|
25
|
+
import { assertSafeAssetPath, assertSafeId } from '../export.js';
|
|
26
|
+
import { warn } from '../log.js';
|
|
25
27
|
|
|
26
28
|
/**
|
|
27
29
|
* Build training package in ai-toolkit format from canonical rows.
|
|
@@ -38,6 +40,11 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
38
40
|
const partitioned = groupRowsByPartition(rows || []);
|
|
39
41
|
|
|
40
42
|
let imageCount = 0;
|
|
43
|
+
// SDL-C5: per-row failure accounting (export.js DB-002 shape) — see
|
|
44
|
+
// generic-image-caption.js for the full rationale. A caught copy/symlink
|
|
45
|
+
// error must not silently drop the image, caption sidecar, AND metadata
|
|
46
|
+
// row together with zero logging.
|
|
47
|
+
const imagesFailed = [];
|
|
41
48
|
|
|
42
49
|
for (const [partition, partitionRows] of Object.entries(partitioned)) {
|
|
43
50
|
const datasetDir = join(packageDir, 'dataset', partition);
|
|
@@ -48,10 +55,21 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
48
55
|
const metadataRows = [];
|
|
49
56
|
|
|
50
57
|
for (const row of partitionRows) {
|
|
51
|
-
if (!row.asset_path)
|
|
58
|
+
if (!row.asset_path) {
|
|
59
|
+
imagesFailed.push({ record_id: row.entity_id, partition, reason: 'missing_asset_path' });
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
52
62
|
|
|
53
|
-
|
|
54
|
-
|
|
63
|
+
// SDL-H5 / SDL-H6: defense-in-depth — see generic-image-caption.js
|
|
64
|
+
// for the full rationale. Throws (hard fail) on traversal/unsafe id;
|
|
65
|
+
// that is a security boundary, not an operational hiccup.
|
|
66
|
+
assertSafeId(row.entity_id, `row.entity_id in ai-toolkit adapter (partition "${partition}")`);
|
|
67
|
+
const sourcePath = assertSafeAssetPath(row.asset_path, projectRoot, row.entity_id);
|
|
68
|
+
|
|
69
|
+
if (!existsSync(sourcePath)) {
|
|
70
|
+
imagesFailed.push({ record_id: row.entity_id, partition, reason: 'source_missing' });
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
55
73
|
|
|
56
74
|
const sourceBasename = row.asset_path.split(/[\\/]/).pop();
|
|
57
75
|
const extMatch = sourceBasename.match(/\.[^.]+$/);
|
|
@@ -67,7 +85,13 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
67
85
|
await symlink(relative(datasetDir, sourcePath), destPath);
|
|
68
86
|
}
|
|
69
87
|
imageCount++;
|
|
70
|
-
} catch {
|
|
88
|
+
} catch (err) {
|
|
89
|
+
imagesFailed.push({
|
|
90
|
+
record_id: row.entity_id,
|
|
91
|
+
partition,
|
|
92
|
+
reason: copy ? 'copy_failed' : 'link_failed',
|
|
93
|
+
detail: err.message,
|
|
94
|
+
});
|
|
71
95
|
continue;
|
|
72
96
|
}
|
|
73
97
|
|
|
@@ -91,13 +115,21 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
91
115
|
);
|
|
92
116
|
}
|
|
93
117
|
|
|
118
|
+
if (imagesFailed.length > 0) {
|
|
119
|
+
const sample = imagesFailed.slice(0, 10).map(f => `${f.record_id} (${f.reason})`);
|
|
120
|
+
warn(
|
|
121
|
+
`ai-toolkit adapter: ${imagesFailed.length} of ${imageCount + imagesFailed.length} rows failed to place` +
|
|
122
|
+
(imagesFailed.length > 10 ? ` — first 10: ${sample.join(', ')}, ...` : ` — ${sample.join(', ')}`),
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
94
126
|
const aiToolkitConfig = buildAiToolkitConfig({ profile, manifest });
|
|
95
127
|
await writeFile(
|
|
96
128
|
join(packageDir, 'ai-toolkit-config.yaml'),
|
|
97
129
|
YAML.stringify(aiToolkitConfig)
|
|
98
130
|
);
|
|
99
131
|
|
|
100
|
-
return { imageCount };
|
|
132
|
+
return { imageCount, imagesFailed };
|
|
101
133
|
}
|
|
102
134
|
|
|
103
135
|
/**
|
|
@@ -7,14 +7,34 @@
|
|
|
7
7
|
|
|
8
8
|
import { writeFile, mkdir } from 'node:fs/promises';
|
|
9
9
|
import { join } from 'node:path';
|
|
10
|
-
import {
|
|
11
|
-
import { buildWorkflowGraph, resolveTemplateId } from './comfyui-workflows.js';
|
|
12
|
-
import { buildSeedPlan, prepareRunDir, saveRunManifest } from '../runtime-runs.js';
|
|
10
|
+
import { comfyHealthDetailed, submitAndWait, downloadImage } from '../comfyui.js';
|
|
11
|
+
import { buildWorkflowGraph, resolveTemplateId, QWEN_DEFAULTS } from './comfyui-workflows.js';
|
|
12
|
+
import { buildSeedPlan, prepareRunDir, saveRunManifest, checkpointRunManifest } from '../runtime-runs.js';
|
|
13
13
|
import { loadProjectConfig } from '../config.js';
|
|
14
14
|
import { pickOutputImage } from '../comfyui-output.js';
|
|
15
|
+
import { buildPinning, seedPolicyFromMode, createHashCache, flushHashCache } from '../run-manifest.js';
|
|
16
|
+
import { SCHEMA_VERSION } from '../snapshot.js';
|
|
15
17
|
import { log, info, warn } from '../log.js';
|
|
16
18
|
import { runtimeError } from '../errors.js';
|
|
17
19
|
|
|
20
|
+
// B04: comfyHealthDetailed()'s `reason` maps to an ACTIONABLE hint, instead
|
|
21
|
+
// of every failure mode sharing the same "start ComfyUI" advice — which is
|
|
22
|
+
// simply wrong for an `http-<5xx>` (ComfyUI is up and answering) and useless
|
|
23
|
+
// for `dns`/`refused` against a misconfigured URL.
|
|
24
|
+
const HEALTH_REASON_HINTS = {
|
|
25
|
+
timeout: 'ComfyUI accepted the connection but never responded — it may be loading a custom node or a large model. Wait and retry, or check its console for a stuck step.',
|
|
26
|
+
refused: 'Start ComfyUI, or check COMFY_URL / project defaults.comfy_url — the connection was actively refused.',
|
|
27
|
+
dns: 'COMFY_URL / project defaults.comfy_url does not resolve — check for a typo in the hostname.',
|
|
28
|
+
unknown: 'Start ComfyUI or set COMFY_URL / project defaults.comfy_url.',
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
function healthReasonHint(reason) {
|
|
32
|
+
if (reason && reason.startsWith('http-')) {
|
|
33
|
+
return `ComfyUI responded with ${reason.slice('http-'.length)} — it is reachable but reported an error; check its console/logs.`;
|
|
34
|
+
}
|
|
35
|
+
return HEALTH_REASON_HINTS[reason] || HEALTH_REASON_HINTS.unknown;
|
|
36
|
+
}
|
|
37
|
+
|
|
18
38
|
/**
|
|
19
39
|
* Execute a brief: build graphs, submit to ComfyUI, capture outputs.
|
|
20
40
|
*
|
|
@@ -26,7 +46,7 @@ import { runtimeError } from '../errors.js';
|
|
|
26
46
|
* @param {boolean} [opts.dryRun] — if true, prepare run dir but do not submit
|
|
27
47
|
* @returns {Promise<Object>} — manifest
|
|
28
48
|
*/
|
|
29
|
-
export async function executeRun({ projectRoot, projectName, brief, baseSeed, dryRun = false }) {
|
|
49
|
+
export async function executeRun({ projectRoot, projectName, brief, baseSeed, dryRun = false, hashModels = false }) {
|
|
30
50
|
const config = loadProjectConfig(projectRoot);
|
|
31
51
|
const defaults = config.meta?.defaults ?? {};
|
|
32
52
|
const comfyUrl = defaults.comfy_url || process.env.COMFY_URL || 'http://127.0.0.1:8188';
|
|
@@ -48,12 +68,16 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
|
|
|
48
68
|
|
|
49
69
|
// Health check
|
|
50
70
|
if (!dryRun) {
|
|
51
|
-
|
|
52
|
-
|
|
71
|
+
// B04: use the reason-carrying check so a 503-while-loading, a refused
|
|
72
|
+
// connection, a DNS typo, and a genuine timeout each get their OWN
|
|
73
|
+
// actionable hint instead of the identical (and sometimes factually
|
|
74
|
+
// wrong) "ComfyUI not reachable — start ComfyUI" message.
|
|
75
|
+
const health = await comfyHealthDetailed(comfyUrl);
|
|
76
|
+
if (!health.ok) {
|
|
53
77
|
throw runtimeError(
|
|
54
78
|
'COMFYUI_OFFLINE',
|
|
55
|
-
`ComfyUI not reachable at ${comfyUrl}`,
|
|
56
|
-
|
|
79
|
+
`ComfyUI not reachable at ${comfyUrl} (${health.reason})`,
|
|
80
|
+
healthReasonHint(health.reason)
|
|
57
81
|
);
|
|
58
82
|
}
|
|
59
83
|
info('ComfyUI online');
|
|
@@ -61,6 +85,43 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
|
|
|
61
85
|
|
|
62
86
|
const outputs = [];
|
|
63
87
|
const errors = [];
|
|
88
|
+
// B01: fixed at run start and reused by every mid-loop checkpoint AND the
|
|
89
|
+
// final manifest, so `created_at` never drifts between an incremental
|
|
90
|
+
// snapshot and the finished manifest for the same run.
|
|
91
|
+
const createdAt = new Date().toISOString();
|
|
92
|
+
|
|
93
|
+
// B01: checkpointRunManifest() existed (with a docstring saying exactly
|
|
94
|
+
// this) but had zero production callers — comfyui-runner.js only called
|
|
95
|
+
// saveRunManifest() once, after the whole loop. A run killed partway
|
|
96
|
+
// through wrote NO manifest at all, so listRuns() silently skipped the
|
|
97
|
+
// directory entirely: no id, no `run show`, no resume. This snapshot
|
|
98
|
+
// mirrors the final manifest's shape (minus `pinning`, computed only
|
|
99
|
+
// after the loop) so a partial manifest.json is readable by the same
|
|
100
|
+
// code that reads a finished one.
|
|
101
|
+
function buildManifestSnapshot() {
|
|
102
|
+
return {
|
|
103
|
+
run_id: runId,
|
|
104
|
+
schema_version: SCHEMA_VERSION,
|
|
105
|
+
brief_id: brief.brief_id,
|
|
106
|
+
project_id: brief.project_id,
|
|
107
|
+
workflow_template_id: templateId,
|
|
108
|
+
adapter_target: 'comfyui',
|
|
109
|
+
output_mode: outputMode,
|
|
110
|
+
output_count: outputCount,
|
|
111
|
+
created_at: createdAt,
|
|
112
|
+
seed_plan: seedPlan,
|
|
113
|
+
comfy_url: comfyUrl,
|
|
114
|
+
checkpoint: defaults.checkpoint,
|
|
115
|
+
loras: defaults.loras || [],
|
|
116
|
+
pinning: null,
|
|
117
|
+
runtime_plan: brief.runtime_plan,
|
|
118
|
+
outputs,
|
|
119
|
+
errors,
|
|
120
|
+
dry_run: dryRun,
|
|
121
|
+
success_count: outputs.filter(o => o.status === 'ok' || o.status === 'dry_run').length,
|
|
122
|
+
error_count: errors.length,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
64
125
|
|
|
65
126
|
for (let i = 0; i < outputCount; i++) {
|
|
66
127
|
const seed = seedPlan.seeds[i];
|
|
@@ -75,6 +136,7 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
|
|
|
75
136
|
status: 'dry_run',
|
|
76
137
|
filename: `${imageIndex}.png`,
|
|
77
138
|
});
|
|
139
|
+
await checkpointRunManifest(runDir, buildManifestSnapshot());
|
|
78
140
|
continue;
|
|
79
141
|
}
|
|
80
142
|
|
|
@@ -104,6 +166,7 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
|
|
|
104
166
|
if (!picked) {
|
|
105
167
|
warn(` No output image for seed ${seed}`);
|
|
106
168
|
outputs.push({ index: i, seed, status: 'no_output', elapsed_ms: elapsed });
|
|
169
|
+
await checkpointRunManifest(runDir, buildManifestSnapshot());
|
|
107
170
|
continue;
|
|
108
171
|
}
|
|
109
172
|
|
|
@@ -136,28 +199,52 @@ export async function executeRun({ projectRoot, projectName, brief, baseSeed, dr
|
|
|
136
199
|
elapsed_ms: elapsed,
|
|
137
200
|
});
|
|
138
201
|
}
|
|
202
|
+
|
|
203
|
+
// B01: checkpoint after every completed output (success OR error) so a
|
|
204
|
+
// crash/interruption mid-loop leaves a partial, readable manifest.json
|
|
205
|
+
// instead of an orphan directory — this is the call the docstring on
|
|
206
|
+
// checkpointRunManifest() always described but nothing ever made.
|
|
207
|
+
await checkpointRunManifest(runDir, buildManifestSnapshot());
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// PIN_PER_STEP: pin the exact graph + model/LoRA content identity + seed
|
|
211
|
+
// policy via the shared contract (lib/run-manifest.js). One run = one prompt
|
|
212
|
+
// across N seeds, so the pin is wave-level (per-item seed is normalized out of
|
|
213
|
+
// comfy_workflow_sha and kept in seed_plan). Best-effort — a build/hash failure
|
|
214
|
+
// degrades to null pinning rather than failing the run.
|
|
215
|
+
let pinning = null;
|
|
216
|
+
try {
|
|
217
|
+
const isQwen = String(defaults.base || '').toLowerCase() === 'qwen-image';
|
|
218
|
+
const pinModels = isQwen
|
|
219
|
+
? {
|
|
220
|
+
unet: defaults.unet || QWEN_DEFAULTS.unet,
|
|
221
|
+
clip: defaults.clip || QWEN_DEFAULTS.clip,
|
|
222
|
+
vae: defaults.vae || QWEN_DEFAULTS.vae,
|
|
223
|
+
}
|
|
224
|
+
: { checkpoint: defaults.checkpoint };
|
|
225
|
+
const pinCache = hashModels ? createHashCache() : undefined;
|
|
226
|
+
const { nodes: pinNodes } = buildWorkflowGraph({ brief, projectDefaults: defaults, seed: seedPlan.seeds[0] ?? 0 });
|
|
227
|
+
pinning = buildPinning({
|
|
228
|
+
graph: pinNodes,
|
|
229
|
+
models: pinModels,
|
|
230
|
+
loras: defaults.loras || [],
|
|
231
|
+
seedPolicy: seedPolicyFromMode(seedMode),
|
|
232
|
+
hashModels,
|
|
233
|
+
cache: pinCache,
|
|
234
|
+
});
|
|
235
|
+
flushHashCache(pinCache);
|
|
236
|
+
} catch (err) {
|
|
237
|
+
warn(`pinning skipped: ${err.message}`);
|
|
139
238
|
}
|
|
140
239
|
|
|
141
|
-
// Build manifest
|
|
240
|
+
// Build manifest — reuses buildManifestSnapshot() (same helper the
|
|
241
|
+
// mid-loop checkpoints use) so the finished manifest and every
|
|
242
|
+
// incremental checkpoint before it are byte-for-byte the same shape,
|
|
243
|
+
// just with `pinning` filled in and no `incremental`/`last_checkpoint_at`
|
|
244
|
+
// stamp (that stamp is what lets a reader tell "done" from "still going").
|
|
142
245
|
const manifest = {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
project_id: brief.project_id,
|
|
146
|
-
workflow_template_id: templateId,
|
|
147
|
-
adapter_target: 'comfyui',
|
|
148
|
-
output_mode: outputMode,
|
|
149
|
-
output_count: outputCount,
|
|
150
|
-
created_at: new Date().toISOString(),
|
|
151
|
-
seed_plan: seedPlan,
|
|
152
|
-
comfy_url: comfyUrl,
|
|
153
|
-
checkpoint: defaults.checkpoint,
|
|
154
|
-
loras: defaults.loras || [],
|
|
155
|
-
runtime_plan: brief.runtime_plan,
|
|
156
|
-
outputs,
|
|
157
|
-
errors,
|
|
158
|
-
dry_run: dryRun,
|
|
159
|
-
success_count: outputs.filter(o => o.status === 'ok' || o.status === 'dry_run').length,
|
|
160
|
-
error_count: errors.length,
|
|
246
|
+
...buildManifestSnapshot(),
|
|
247
|
+
pinning,
|
|
161
248
|
};
|
|
162
249
|
|
|
163
250
|
await saveRunManifest(runDir, manifest);
|
|
@@ -33,8 +33,16 @@ export function listWorkflowTemplates() {
|
|
|
33
33
|
|
|
34
34
|
/**
|
|
35
35
|
* Resolve a template for a given output mode.
|
|
36
|
-
*
|
|
37
|
-
*
|
|
36
|
+
*
|
|
37
|
+
* Returns the first template declaring `outputMode` in its `compatible_modes`,
|
|
38
|
+
* and falls back to `'txt2img-standard'` when none matches — it does NOT
|
|
39
|
+
* throw. (The docstring claimed a throw until 2026-08-04; a caller written
|
|
40
|
+
* against that claim would guard a branch that cannot fire and would miss the
|
|
41
|
+
* silent-default case, where an unrecognised mode renders through the standard
|
|
42
|
+
* SDXL graph instead of failing loudly.)
|
|
43
|
+
*
|
|
44
|
+
* The fallback is also what makes an install with no `runtime/comfyui/`
|
|
45
|
+
* templates degrade rather than crash — see `getRuntimeDir()` in lib/paths.js.
|
|
38
46
|
*/
|
|
39
47
|
export function resolveTemplateId(outputMode) {
|
|
40
48
|
const templates = listWorkflowTemplates();
|
|
@@ -16,6 +16,8 @@ import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
|
|
|
16
16
|
import { join, relative } from 'node:path';
|
|
17
17
|
import { existsSync } from 'node:fs';
|
|
18
18
|
import { groupRowsByPartition } from '../rows.js';
|
|
19
|
+
import { assertSafeAssetPath, assertSafeId } from '../export.js';
|
|
20
|
+
import { warn } from '../log.js';
|
|
19
21
|
|
|
20
22
|
/**
|
|
21
23
|
* Build training package in diffusers LoRA format from canonical rows.
|
|
@@ -33,6 +35,11 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
33
35
|
const partitioned = groupRowsByPartition(rows || []);
|
|
34
36
|
|
|
35
37
|
let imageCount = 0;
|
|
38
|
+
// SDL-C5: per-row failure accounting (export.js DB-002 shape) — see
|
|
39
|
+
// generic-image-caption.js for the full rationale. A caught copy/symlink
|
|
40
|
+
// error must not silently drop the image, caption sidecar, AND metadata
|
|
41
|
+
// row together with zero logging.
|
|
42
|
+
const imagesFailed = [];
|
|
36
43
|
|
|
37
44
|
for (const [partition, partitionRows] of Object.entries(partitioned)) {
|
|
38
45
|
const datasetDir = join(packageDir, 'dataset', partition);
|
|
@@ -43,10 +50,21 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
43
50
|
const metadataRows = [];
|
|
44
51
|
|
|
45
52
|
for (const row of partitionRows) {
|
|
46
|
-
if (!row.asset_path)
|
|
53
|
+
if (!row.asset_path) {
|
|
54
|
+
imagesFailed.push({ record_id: row.entity_id, partition, reason: 'missing_asset_path' });
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
47
57
|
|
|
48
|
-
|
|
49
|
-
|
|
58
|
+
// SDL-H5 / SDL-H6: defense-in-depth — see generic-image-caption.js
|
|
59
|
+
// for the full rationale. Throws (hard fail) on traversal/unsafe id;
|
|
60
|
+
// that is a security boundary, not an operational hiccup.
|
|
61
|
+
assertSafeId(row.entity_id, `row.entity_id in diffusers-lora adapter (partition "${partition}")`);
|
|
62
|
+
const sourcePath = assertSafeAssetPath(row.asset_path, projectRoot, row.entity_id);
|
|
63
|
+
|
|
64
|
+
if (!existsSync(sourcePath)) {
|
|
65
|
+
imagesFailed.push({ record_id: row.entity_id, partition, reason: 'source_missing' });
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
50
68
|
|
|
51
69
|
const sourceBasename = row.asset_path.split(/[\\/]/).pop();
|
|
52
70
|
const extMatch = sourceBasename.match(/\.[^.]+$/);
|
|
@@ -62,7 +80,13 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
62
80
|
await symlink(relative(datasetDir, sourcePath), destPath);
|
|
63
81
|
}
|
|
64
82
|
imageCount++;
|
|
65
|
-
} catch {
|
|
83
|
+
} catch (err) {
|
|
84
|
+
imagesFailed.push({
|
|
85
|
+
record_id: row.entity_id,
|
|
86
|
+
partition,
|
|
87
|
+
reason: copy ? 'copy_failed' : 'link_failed',
|
|
88
|
+
detail: err.message,
|
|
89
|
+
});
|
|
66
90
|
continue;
|
|
67
91
|
}
|
|
68
92
|
|
|
@@ -77,7 +101,15 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
77
101
|
);
|
|
78
102
|
}
|
|
79
103
|
|
|
80
|
-
|
|
104
|
+
if (imagesFailed.length > 0) {
|
|
105
|
+
const sample = imagesFailed.slice(0, 10).map(f => `${f.record_id} (${f.reason})`);
|
|
106
|
+
warn(
|
|
107
|
+
`diffusers-lora adapter: ${imagesFailed.length} of ${imageCount + imagesFailed.length} rows failed to place` +
|
|
108
|
+
(imagesFailed.length > 10 ? ` — first 10: ${sample.join(', ')}, ...` : ` — ${sample.join(', ')}`),
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return { imageCount, imagesFailed };
|
|
81
113
|
}
|
|
82
114
|
|
|
83
115
|
function buildMetadataRow(row, partition, filename, stem) {
|
|
@@ -14,11 +14,20 @@ import { writeFile, mkdir, symlink, copyFile } from 'node:fs/promises';
|
|
|
14
14
|
import { join, relative } from 'node:path';
|
|
15
15
|
import { existsSync } from 'node:fs';
|
|
16
16
|
import { groupRowsByPartition } from '../rows.js';
|
|
17
|
+
import { assertSafeAssetPath, assertSafeId } from '../export.js';
|
|
18
|
+
import { warn } from '../log.js';
|
|
17
19
|
|
|
18
20
|
export async function buildPackage({ packageDir, rows, profile, manifest, config, projectRoot, copy }) {
|
|
19
21
|
const partitioned = groupRowsByPartition(rows || []);
|
|
20
22
|
|
|
21
23
|
let imageCount = 0;
|
|
24
|
+
// SDL-C5: per-row failure accounting (export.js DB-002 shape). The old
|
|
25
|
+
// bare `catch { continue; }` silently dropped the image, the caption
|
|
26
|
+
// sidecar, AND the metadata row together with no logging — on Windows
|
|
27
|
+
// without Developer Mode this fired on every row, so imageCount stayed
|
|
28
|
+
// 0 and the build still reported success. Every drop is now accounted
|
|
29
|
+
// for and returned to the caller.
|
|
30
|
+
const imagesFailed = [];
|
|
22
31
|
|
|
23
32
|
for (const [partition, partitionRows] of Object.entries(partitioned)) {
|
|
24
33
|
const datasetDir = join(packageDir, 'dataset', partition);
|
|
@@ -29,10 +38,25 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
29
38
|
const metadataRows = [];
|
|
30
39
|
|
|
31
40
|
for (const row of partitionRows) {
|
|
32
|
-
if (!row.asset_path)
|
|
41
|
+
if (!row.asset_path) {
|
|
42
|
+
imagesFailed.push({ record_id: row.entity_id, partition, reason: 'missing_asset_path' });
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
33
45
|
|
|
34
|
-
|
|
35
|
-
|
|
46
|
+
// SDL-H5 / SDL-H6: defense-in-depth. buildTrainingPackage validates
|
|
47
|
+
// every row before dispatching here, but this adapter is also
|
|
48
|
+
// unit-tested (and documented in lib/rows.js as a future direct
|
|
49
|
+
// canon-build target) independent of that caller, so the guard
|
|
50
|
+
// lives here too. A traversal / unsafe id is a security boundary,
|
|
51
|
+
// not an operational hiccup — it throws (hard fail) rather than
|
|
52
|
+
// joining the soft imagesFailed accounting below.
|
|
53
|
+
assertSafeId(row.entity_id, `row.entity_id in generic-image-caption adapter (partition "${partition}")`);
|
|
54
|
+
const sourcePath = assertSafeAssetPath(row.asset_path, projectRoot, row.entity_id);
|
|
55
|
+
|
|
56
|
+
if (!existsSync(sourcePath)) {
|
|
57
|
+
imagesFailed.push({ record_id: row.entity_id, partition, reason: 'source_missing' });
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
36
60
|
|
|
37
61
|
const sourceBasename = row.asset_path.split(/[\\/]/).pop();
|
|
38
62
|
const extMatch = sourceBasename.match(/\.[^.]+$/);
|
|
@@ -47,7 +71,13 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
47
71
|
await symlink(relative(datasetDir, sourcePath), destPath);
|
|
48
72
|
}
|
|
49
73
|
imageCount++;
|
|
50
|
-
} catch {
|
|
74
|
+
} catch (err) {
|
|
75
|
+
imagesFailed.push({
|
|
76
|
+
record_id: row.entity_id,
|
|
77
|
+
partition,
|
|
78
|
+
reason: copy ? 'copy_failed' : 'link_failed',
|
|
79
|
+
detail: err.message,
|
|
80
|
+
});
|
|
51
81
|
continue;
|
|
52
82
|
}
|
|
53
83
|
|
|
@@ -67,5 +97,13 @@ export async function buildPackage({ packageDir, rows, profile, manifest, config
|
|
|
67
97
|
);
|
|
68
98
|
}
|
|
69
99
|
|
|
70
|
-
|
|
100
|
+
if (imagesFailed.length > 0) {
|
|
101
|
+
const sample = imagesFailed.slice(0, 10).map(f => `${f.record_id} (${f.reason})`);
|
|
102
|
+
warn(
|
|
103
|
+
`generic-image-caption adapter: ${imagesFailed.length} of ${imageCount + imagesFailed.length} rows failed to place` +
|
|
104
|
+
(imagesFailed.length > 10 ? ` — first 10: ${sample.join(', ')}, ...` : ` — ${sample.join(', ')}`),
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return { imageCount, imagesFailed };
|
|
71
109
|
}
|
package/lib/args.js
CHANGED
|
@@ -149,8 +149,23 @@ export function getProjectName(argv) {
|
|
|
149
149
|
for (let i = 0; i < argv.length; i++) {
|
|
150
150
|
const a = argv[i];
|
|
151
151
|
if (a === '--project' || a === '--game') {
|
|
152
|
+
const flagName = a === '--game' ? 'game' : 'project';
|
|
152
153
|
if (a === '--game') warnDeprecated('game', 'project');
|
|
153
|
-
|
|
154
|
+
const next = argv[i + 1];
|
|
155
|
+
// SDL-L1: mirror takeFlagValue's guard below — without it, a missing
|
|
156
|
+
// value silently swallowed the NEXT flag as the project name (e.g.
|
|
157
|
+
// `--project --dry-run` yielded the project name "--dry-run").
|
|
158
|
+
if (next === undefined) {
|
|
159
|
+
throw inputError('INPUT_MISSING_VALUE', `Flag --${flagName} requires a value.`);
|
|
160
|
+
}
|
|
161
|
+
if (next.startsWith('--')) {
|
|
162
|
+
throw inputError(
|
|
163
|
+
'INPUT_MISSING_VALUE',
|
|
164
|
+
`Flag --${flagName} is missing its value (got another flag: ${next}).`,
|
|
165
|
+
`Pass a value after --${flagName}.`
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
explicit = next;
|
|
154
169
|
break;
|
|
155
170
|
}
|
|
156
171
|
if (a.startsWith('--project=')) { explicit = a.slice('--project='.length); break; }
|
package/lib/batch-modes.js
CHANGED
|
@@ -27,25 +27,35 @@ export function getBatchModesDir(projectRoot) {
|
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
29
|
* List all batch modes for a project.
|
|
30
|
+
*
|
|
31
|
+
* B06: listWorkflowProfiles() (its structurally identical sibling in
|
|
32
|
+
* workflow-profiles.js) already skips a malformed file per-entry via
|
|
33
|
+
* try/catch. This function had no such guard, so one bad batch-mode file
|
|
34
|
+
* threw an uncaught SyntaxError out of Array.prototype.map and hid EVERY
|
|
35
|
+
* batch mode, not just the broken one.
|
|
30
36
|
*/
|
|
31
37
|
export function listBatchModes(projectRoot) {
|
|
32
38
|
const dir = getBatchModesDir(projectRoot);
|
|
33
39
|
if (!existsSync(dir)) return [];
|
|
34
40
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
41
|
+
const modes = [];
|
|
42
|
+
for (const f of readdirSync(dir).filter(f => f.endsWith('.json'))) {
|
|
43
|
+
try {
|
|
38
44
|
const data = JSON.parse(readFileSync(join(dir, f), 'utf-8'));
|
|
39
|
-
|
|
45
|
+
modes.push({
|
|
40
46
|
mode_id: data.mode_id,
|
|
41
47
|
label: data.label,
|
|
42
48
|
batch_type: data.batch_type,
|
|
43
49
|
base_workflow_id: data.base_workflow_id,
|
|
44
50
|
subject_mode: data.subject_mode || 'optional',
|
|
45
51
|
slot_count: data.variant_plan?.length || 0,
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
|
|
52
|
+
});
|
|
53
|
+
} catch {
|
|
54
|
+
// Skip malformed — matches listWorkflowProfiles' existing resilience.
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return modes.sort((a, b) => a.mode_id.localeCompare(b.mode_id));
|
|
49
59
|
}
|
|
50
60
|
|
|
51
61
|
/**
|
|
@@ -63,7 +73,18 @@ export function getBatchMode(projectRoot, modeId) {
|
|
|
63
73
|
);
|
|
64
74
|
}
|
|
65
75
|
|
|
66
|
-
|
|
76
|
+
// B06: JSON.parse used to run unguarded here, so a malformed mode file
|
|
77
|
+
// threw a raw, uncaught SyntaxError naming nothing. Wrapped in the same
|
|
78
|
+
// structured-error pattern lib/config.js's loadJsonFile already uses.
|
|
79
|
+
try {
|
|
80
|
+
return JSON.parse(readFileSync(filePath, 'utf-8'));
|
|
81
|
+
} catch (err) {
|
|
82
|
+
throw inputError(
|
|
83
|
+
'INPUT_INVALID_JSON',
|
|
84
|
+
`Failed to parse ${filePath}: ${err.message}`,
|
|
85
|
+
'Check the file for JSON syntax errors (trailing commas, unquoted keys, etc).'
|
|
86
|
+
);
|
|
87
|
+
}
|
|
67
88
|
}
|
|
68
89
|
|
|
69
90
|
/**
|