@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.es.md +117 -55
- package/README.fr.md +125 -65
- package/README.hi.md +117 -68
- package/README.it.md +119 -58
- package/README.ja.md +124 -74
- package/README.md +73 -7
- package/README.pt-BR.md +128 -63
- package/README.zh.md +116 -65
- package/bin/sdlab.js +100 -4
- package/lib/adapters/ai-toolkit.js +224 -35
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +94 -11
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/asset-ingest.js +575 -0
- package/lib/asset-masks.js +323 -0
- package/lib/asset-source.js +1176 -0
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-author.js +350 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/canon-review-gate.js +205 -0
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/colab-launcher-guards.js +285 -0
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +16 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +194 -6
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/lane1-render.js +668 -0
- package/lib/measure.js +422 -0
- package/lib/memory-store.js +663 -0
- package/lib/model-licences.js +305 -0
- package/lib/npy-meta.js +130 -0
- package/lib/paths.js +196 -7
- package/lib/png-meta.js +512 -0
- package/lib/records.js +54 -8
- package/lib/reingest-selected.js +113 -17
- package/lib/reingest.js +27 -3
- package/lib/run-manifest.js +159 -25
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/train-runs.js +214 -0
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +3 -0
- package/lib/verify-chain.js +367 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +4 -3
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/asset-ingest.js +160 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-draft.js +82 -0
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +31 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
- package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
- package/scripts/colab/colab_window.sh +587 -0
- package/scripts/colab/lane1_render.py +970 -0
- package/scripts/colab/launch_guards.py +564 -0
- package/scripts/colab/launch_lane1_stub.sh +309 -0
- package/scripts/colab/launch_lane3_stub.sh +366 -0
- package/scripts/colab/remote_runner.py +59 -0
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +182 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +86 -17
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/verify.js +67 -0
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* measure.js — Attach deterministic pixel measurements to candidate records.
|
|
5
|
+
*
|
|
6
|
+
* Until now nothing in this tool ever opened an image. `lib/critique-engine.js`
|
|
7
|
+
* builds its entire drift report from brief and workflow TEXT — keyword
|
|
8
|
+
* matching against prompt strings — and stamps every candidate
|
|
9
|
+
* `overall_fit: 'usable'` without looking at a single pixel. Every question
|
|
10
|
+
* about what an image actually looks like was answered by eyeballing it.
|
|
11
|
+
*
|
|
12
|
+
* The instruments here are a port, not an invention. They come from the
|
|
13
|
+
* measurement scripts written for the salt-road project
|
|
14
|
+
* (`projects/salt-road/inputs/prompts/measure-palette.py`, `measure-texture.py`),
|
|
15
|
+
* which were built and validated against real production waves and carry
|
|
16
|
+
* their own written rationale for why simpler proxies were rejected.
|
|
17
|
+
*
|
|
18
|
+
* MEASUREMENT, NOT VERDICT. This command attaches numbers. It never sets
|
|
19
|
+
* `judgment`, never sets `overall_fit`, never approves or rejects. That line
|
|
20
|
+
* is deliberate and load-bearing: this repo deleted three scripts in its
|
|
21
|
+
* Stage A health pass for minting judgments nobody made, and 948 of
|
|
22
|
+
* star-freight's 1,182 records still carry that damage. Numbers inform a
|
|
23
|
+
* human; they do not replace one.
|
|
24
|
+
*
|
|
25
|
+
* Usage:
|
|
26
|
+
* sdlab measure <dir-or-record-glob> --project <name>
|
|
27
|
+
* sdlab measure outputs/approved --project salt-road
|
|
28
|
+
* sdlab measure "styleset_p*" --project salt-road --anchors canon/anchors.json
|
|
29
|
+
* sdlab measure outputs/candidates --project salt-road --dry-run
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { parseArgs, getProjectName } from '../lib/args.js';
|
|
33
|
+
import { getProjectRoot } from '../lib/paths.js';
|
|
34
|
+
import { handleCliError } from '../lib/errors.js';
|
|
35
|
+
import { warn, result } from '../lib/log.js';
|
|
36
|
+
import {
|
|
37
|
+
assertMeasurementPrereqs,
|
|
38
|
+
resolveMeasureTargets,
|
|
39
|
+
loadAnchors,
|
|
40
|
+
runPythonMeasurement,
|
|
41
|
+
applyMeasurements,
|
|
42
|
+
} from '../lib/measure.js';
|
|
43
|
+
|
|
44
|
+
const USAGE = `Usage: sdlab measure <dir-or-record-glob> --project <name> [--anchors <path>] [--dry-run] [--json]
|
|
45
|
+
|
|
46
|
+
Measure candidate images (palette conformance, texture character) and attach
|
|
47
|
+
the numbers to their records. Measurement only — never sets a judgment.
|
|
48
|
+
|
|
49
|
+
Positional:
|
|
50
|
+
dir-or-record-glob A directory of images, or a record-id glob
|
|
51
|
+
(quote globs so your shell does not expand them)
|
|
52
|
+
|
|
53
|
+
Flags:
|
|
54
|
+
--project <name> Project to operate on
|
|
55
|
+
--anchors <path> Palette anchor definitions, relative to the project
|
|
56
|
+
root. Without it, palette measures that need anchors
|
|
57
|
+
are skipped rather than guessed.
|
|
58
|
+
--dry-run Measure and report, write nothing
|
|
59
|
+
--json Emit the raw result as JSON
|
|
60
|
+
|
|
61
|
+
Requires Python 3.9+ with Pillow, numpy and scipy (see scripts/measure_image.py).
|
|
62
|
+
Set SDLAB_PYTHON to pin an explicit interpreter.`;
|
|
63
|
+
|
|
64
|
+
export async function run(argv = process.argv.slice(2)) {
|
|
65
|
+
const { flags, positionals } = parseArgs(argv, {
|
|
66
|
+
flags: {
|
|
67
|
+
project: { type: 'string' },
|
|
68
|
+
anchors: { type: 'string' },
|
|
69
|
+
'dry-run': { type: 'boolean' },
|
|
70
|
+
json: { type: 'boolean' },
|
|
71
|
+
help: { type: 'boolean' },
|
|
72
|
+
},
|
|
73
|
+
deprecated: { game: 'project' },
|
|
74
|
+
allowUnknown: true,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
if (flags.help) {
|
|
78
|
+
console.log(USAGE);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const projectName = flags.project || getProjectName(argv);
|
|
83
|
+
const projectRoot = getProjectRoot(projectName);
|
|
84
|
+
const dryRun = Boolean(flags['dry-run']);
|
|
85
|
+
|
|
86
|
+
// Fail on a missing interpreter or package BEFORE resolving targets, so an
|
|
87
|
+
// operator without the Python side gets one clear actionable error rather
|
|
88
|
+
// than a successful scan followed by a confusing failure.
|
|
89
|
+
const python = assertMeasurementPrereqs();
|
|
90
|
+
|
|
91
|
+
const { targets, skipped } = resolveMeasureTargets(projectRoot, positionals[0]);
|
|
92
|
+
|
|
93
|
+
console.log(`\x1b[1msdlab measure\x1b[0m — ${projectName}`);
|
|
94
|
+
console.log(` Target: ${positionals[0]}`);
|
|
95
|
+
console.log(` Images: ${targets.length}${dryRun ? ' (dry run — nothing will be written)' : ''}`);
|
|
96
|
+
|
|
97
|
+
const anchorSpec = loadAnchors(flags.anchors, projectRoot);
|
|
98
|
+
if (anchorSpec.source) {
|
|
99
|
+
console.log(` Anchors: ${anchorSpec.anchors.length} from ${flags.anchors}`);
|
|
100
|
+
} else {
|
|
101
|
+
// Say so plainly. A palette measure with no anchors is not a failed
|
|
102
|
+
// measurement, but it is a narrower one, and silently returning fewer
|
|
103
|
+
// fields than the operator expected is how a tool loses trust.
|
|
104
|
+
console.log(` Anchors: none — pass --anchors <path> for palette-conformance measures`);
|
|
105
|
+
}
|
|
106
|
+
console.log('');
|
|
107
|
+
|
|
108
|
+
if (skipped.length > 0) {
|
|
109
|
+
warn(`${skipped.length} target(s) skipped: ${skipped.slice(0, 3).map((s) => `${s.id} (${s.reason})`).join(', ')}${skipped.length > 3 ? ', …' : ''}`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (targets.length === 0) {
|
|
113
|
+
console.log(' Nothing to measure.');
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const pyResult = runPythonMeasurement(python, {
|
|
118
|
+
images: targets.map((t) => ({ id: t.id, path: t.imagePath })),
|
|
119
|
+
anchors: anchorSpec.anchors,
|
|
120
|
+
hue_tolerance_deg: anchorSpec.hue_tolerance_deg,
|
|
121
|
+
sat_min: anchorSpec.sat_min,
|
|
122
|
+
val_min: anchorSpec.val_min,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const { applied, failed } = await applyMeasurements(targets, pyResult, {
|
|
126
|
+
anchorsSource: anchorSpec.source,
|
|
127
|
+
dryRun,
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
if (flags.json) {
|
|
131
|
+
result(JSON.stringify({ applied, failed, skipped }, null, 2));
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
console.log(` \x1b[32m✓\x1b[0m Measured: ${applied.length}`);
|
|
136
|
+
if (failed.length > 0) {
|
|
137
|
+
console.log(` \x1b[31m✗\x1b[0m Failed: ${failed.length}`);
|
|
138
|
+
for (const f of failed.slice(0, 5)) {
|
|
139
|
+
console.log(` ${f.id}: ${f.error}`);
|
|
140
|
+
}
|
|
141
|
+
if (failed.length > 5) console.log(` … and ${failed.length - 5} more`);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (applied.length > 0) {
|
|
145
|
+
// Show one real measurement rather than only a count — a number the
|
|
146
|
+
// operator can sanity-check against the image in front of them is worth
|
|
147
|
+
// more than a success tally.
|
|
148
|
+
const sample = applied[0];
|
|
149
|
+
const keys = Object.keys(sample.measurements || {}).filter((k) => !k.startsWith('_'));
|
|
150
|
+
if (keys.length > 0) {
|
|
151
|
+
console.log('');
|
|
152
|
+
console.log(` Sample (${sample.id}):`);
|
|
153
|
+
for (const k of keys.slice(0, 6)) {
|
|
154
|
+
const v = sample.measurements[k];
|
|
155
|
+
const shown = typeof v === 'number' ? v.toFixed(4) : JSON.stringify(v);
|
|
156
|
+
console.log(` ${k.padEnd(28)} ${shown}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
if (dryRun) {
|
|
162
|
+
console.log('');
|
|
163
|
+
console.log(' (dry run — no records were modified)');
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Direct execution guard
|
|
168
|
+
if (process.argv[1] && (process.argv[1].endsWith('measure.js') || process.argv[1].endsWith('measure'))) {
|
|
169
|
+
run().catch(handleCliError);
|
|
170
|
+
}
|
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""measure_image — numeric palette + texture measurement for candidate images.
|
|
3
|
+
|
|
4
|
+
MEASUREMENT, NOT VERDICT. Every number this script produces is a signal to
|
|
5
|
+
attach to a record — nothing here classifies, thresholds, or emits a
|
|
6
|
+
pass/fail/approved/rejected verdict. That judgment stays with a human (or a
|
|
7
|
+
future LLM judge). This repo deleted three scripts in Stage A for fabricating
|
|
8
|
+
judgments from keyword matching instead of looking at the pixels; this script
|
|
9
|
+
exists to give the human (or a future judge) real pixels to look at, not to
|
|
10
|
+
replace them.
|
|
11
|
+
|
|
12
|
+
PORTED FROM (read these for the full rationale before touching the math):
|
|
13
|
+
projects/salt-road/inputs/prompts/measure-palette.py
|
|
14
|
+
projects/salt-road/inputs/prompts/measure-texture.py
|
|
15
|
+
These are the Director-validated, production-used salt-road audit
|
|
16
|
+
instruments. This script ports their MEASUREMENT CORE (the per-image pixel
|
|
17
|
+
math) — not the one-off audit drivers those files also contain (sweep-
|
|
18
|
+
directory globbing, filename-regex config classification, cross-run
|
|
19
|
+
comparison tables). sdlab measure operates per-record; the salt-road scripts
|
|
20
|
+
audit a specific sweep. Both are left untouched by this port and remain
|
|
21
|
+
available for that project-specific work.
|
|
22
|
+
|
|
23
|
+
DELIBERATELY NOT PORTED: the DISTS/StyleLoss perceptual cross-check in
|
|
24
|
+
projects/salt-road/inputs/prompts/crosscheck-dists.py. That instrument needs
|
|
25
|
+
torch+piq — a GPU-class dependency this CLI does not take on. Briefly, for
|
|
26
|
+
context (see that file for the full citations): it uses DISTS (Ding, Ma, Wang
|
|
27
|
+
& Simoncelli 2020, arXiv:2004.07728, TPAMI — separates texture similarity
|
|
28
|
+
from structure similarity, tolerant of texture resampling) and StyleLoss
|
|
29
|
+
(Gatys et al. 2015, arXiv:1505.07376 — Gram-matrix distance over VGG
|
|
30
|
+
features, legitimate for comparing scenes with different content), and it
|
|
31
|
+
explicitly rejects FID (poor sample complexity at small n — Jayasumana 2024,
|
|
32
|
+
arXiv:2401.09603; Chong & Forsyth 2020, arXiv:1911.07023) and DreamSim
|
|
33
|
+
(engineered to look past low-level colour/texture — Fu 2023, arXiv:2306.09344
|
|
34
|
+
— which would be blind to the thing under test here). A future `--perceptual`
|
|
35
|
+
mode should start from that file, not from scratch.
|
|
36
|
+
|
|
37
|
+
Deliberate changes from the ported originals (see also the sdlab feature
|
|
38
|
+
report for the full list):
|
|
39
|
+
- The two hardcoded salt-road ochre anchors (measure-palette.py's
|
|
40
|
+
`#c9a877` / `#a86b4c`, with hand-picked disjoint hue bands 8-42/42-70)
|
|
41
|
+
become an N-anchor system driven by the caller's `anchors` list (project
|
|
42
|
+
canon has no numeric palette data to read this from automatically — see
|
|
43
|
+
the sdlab feature report). Each gated pixel is assigned to whichever
|
|
44
|
+
anchor's hue is circularly nearest, provided that distance is within
|
|
45
|
+
`hue_tolerance_deg`; otherwise it is "off_anchor". This is a
|
|
46
|
+
generalization, not a re-derivation: for salt-road's own two anchors
|
|
47
|
+
(hue ~19deg and ~33deg) at the default 20deg tolerance it reproduces
|
|
48
|
+
materially the same combined catchment the original hand-tuned 8-42deg
|
|
49
|
+
band covered, while working for any project's own anchor set.
|
|
50
|
+
- measure-texture.py's `yellow_pct`/`ochre_pct`/`green_pct` (a second,
|
|
51
|
+
duplicate hue-binning pass baked into the texture measurer so "every
|
|
52
|
+
image measured for texture is measured for colour in the same pass") is
|
|
53
|
+
replaced by calling the SAME hue/anchor code the palette measurer uses.
|
|
54
|
+
Same discipline (one image load, texture + colour measured together),
|
|
55
|
+
without a second hardcoded hue-band implementation living in two files.
|
|
56
|
+
- `facade_rep` (row-of-identical-facades autocorrelation) is intentionally
|
|
57
|
+
NOT ported. It is a composition/repetition detector conditioned on
|
|
58
|
+
subject ("only meaningful on wide exteriors... meaningless for an
|
|
59
|
+
interior or a close stone study" — measure-texture.py), not a palette or
|
|
60
|
+
texture measure in the sense this CLI is scoped to (palette + texture,
|
|
61
|
+
per the feature's approved scope). Left out; see the feature report.
|
|
62
|
+
- The vectorised numpy hue/sat/val computation is adopted from
|
|
63
|
+
measure-texture.py's `hue_bands()` (itself a numerically-verified,
|
|
64
|
+
vectorised reproduction of measure-palette.py's slower colorsys
|
|
65
|
+
per-pixel loop — see that function's docstring) rather than
|
|
66
|
+
reimplementing the slow loop a second time. One hue/sat/val
|
|
67
|
+
implementation, used by both this script's palette and texture passes.
|
|
68
|
+
|
|
69
|
+
python measure_image.py --request <request.json> --out <result.json>
|
|
70
|
+
|
|
71
|
+
Request JSON shape (written by lib/measure.js):
|
|
72
|
+
{
|
|
73
|
+
"images": [{"id": "...", "path": "..."}, ...],
|
|
74
|
+
"anchors": [{"name": "ochre-warm", "hex": "#c9a877"}, ...], # optional, [] or omitted is fine
|
|
75
|
+
"hue_tolerance_deg": 20, # optional
|
|
76
|
+
"sat_min": 0.22, # optional — ported default, see measure_palette()
|
|
77
|
+
"val_min": 0.18 # optional — ported default, see measure_palette()
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
Result JSON shape (written to --out):
|
|
81
|
+
{
|
|
82
|
+
"schema_version": "1.0.0", "tool": "measure_image.py", "generated_at": "...",
|
|
83
|
+
"results": [{"id", "path", "status": "ok", "image": {...}, "palette": {...}, "texture": {...}}],
|
|
84
|
+
"errors": [{"id", "path", "error"}],
|
|
85
|
+
"ok_count": N, "error_count": N
|
|
86
|
+
}
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
from __future__ import annotations
|
|
90
|
+
|
|
91
|
+
import argparse
|
|
92
|
+
import colorsys
|
|
93
|
+
import json
|
|
94
|
+
import math
|
|
95
|
+
import os
|
|
96
|
+
import re
|
|
97
|
+
import time
|
|
98
|
+
from datetime import datetime, timezone
|
|
99
|
+
from pathlib import Path
|
|
100
|
+
|
|
101
|
+
import numpy as np
|
|
102
|
+
from PIL import Image
|
|
103
|
+
from scipy.ndimage import gaussian_filter, sobel
|
|
104
|
+
|
|
105
|
+
MEASURE_SCHEMA_VERSION = "1.0.0"
|
|
106
|
+
|
|
107
|
+
# Ported verbatim from measure-palette.py's saturated-pixel gate: "ignore
|
|
108
|
+
# greys/darks — stone, tar, shadow". Comparability with that instrument's
|
|
109
|
+
# published numbers depends on these two values, so they are the default
|
|
110
|
+
# rather than something re-derived here.
|
|
111
|
+
DEFAULT_SAT_MIN = 0.22
|
|
112
|
+
DEFAULT_VAL_MIN = 0.18
|
|
113
|
+
|
|
114
|
+
# Not from either original (neither had a per-anchor tolerance concept — see
|
|
115
|
+
# module docstring). A deliberate new default: half-width, in degrees, of the
|
|
116
|
+
# circular-hue catchment around each anchor's own hue. Project owners tune
|
|
117
|
+
# this via the anchors JSON's "hue_tolerance_deg" field.
|
|
118
|
+
DEFAULT_HUE_TOLERANCE_DEG = 20.0
|
|
119
|
+
|
|
120
|
+
_HEX_RE = re.compile(r"^#[0-9a-fA-F]{6}$")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _utc_now_iso() -> str:
|
|
124
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.") + \
|
|
125
|
+
f"{datetime.now(timezone.utc).microsecond // 1000:03d}Z"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _json_safe(obj):
|
|
129
|
+
"""Replace non-finite floats with None, recursively.
|
|
130
|
+
|
|
131
|
+
Python's json.dumps emits bare `NaN` / `Infinity` for these — a
|
|
132
|
+
non-standard extension that RFC 8259 does not allow and that JavaScript's
|
|
133
|
+
JSON.parse rejects outright, taking the WHOLE result down with it, not
|
|
134
|
+
just the offending field. The Node caller (lib/measure.js) then reports
|
|
135
|
+
`Unexpected token 'N'` and the operator learns nothing about which image
|
|
136
|
+
or which measure produced it.
|
|
137
|
+
|
|
138
|
+
Non-finite values are legitimate here rather than a bug to prevent: a
|
|
139
|
+
perfectly flat image has zero luminance variance, so a measure that
|
|
140
|
+
normalizes by it is genuinely undefined. Found in Phase 9 by measuring
|
|
141
|
+
three solid-colour images, which every unit test had missed because they
|
|
142
|
+
all paired a flat image with a textured one.
|
|
143
|
+
|
|
144
|
+
`null` is the honest encoding of "this measure is undefined for this
|
|
145
|
+
image" — the same choice palette conformance already makes when no
|
|
146
|
+
anchors are supplied. It is emphatically not 0, which would read as a
|
|
147
|
+
real measurement of a real property.
|
|
148
|
+
"""
|
|
149
|
+
if isinstance(obj, float):
|
|
150
|
+
return obj if math.isfinite(obj) else None
|
|
151
|
+
if isinstance(obj, dict):
|
|
152
|
+
return {k: _json_safe(v) for k, v in obj.items()}
|
|
153
|
+
if isinstance(obj, (list, tuple)):
|
|
154
|
+
return [_json_safe(v) for v in obj]
|
|
155
|
+
return obj
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _atomic_write_json(path: Path, obj) -> None:
|
|
159
|
+
"""Temp file in the same dir + os.replace — the Python-side mirror of
|
|
160
|
+
lib/runtime-runs.js's atomicWriteJson (temp+rename): a crash mid-write
|
|
161
|
+
can never leave a half-written result.json in place of a good one.
|
|
162
|
+
os.replace (not os.rename) because it atomically overwrites the
|
|
163
|
+
destination on Windows too, matching Node's fs.rename semantics."""
|
|
164
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
165
|
+
tmp = path.parent / f"{path.name}.tmp-{os.getpid()}-{int(time.time() * 1000)}"
|
|
166
|
+
try:
|
|
167
|
+
# allow_nan=False makes a leaked non-finite value a loud Python-side
|
|
168
|
+
# ValueError here rather than a bare `NaN` token that only explodes
|
|
169
|
+
# later in the Node caller's JSON.parse, where the message names no
|
|
170
|
+
# image and no measure. _json_safe should have caught them all; this
|
|
171
|
+
# is the assertion that it did.
|
|
172
|
+
tmp.write_text(
|
|
173
|
+
json.dumps(_json_safe(obj), indent=2, allow_nan=False) + "\n",
|
|
174
|
+
encoding="utf-8",
|
|
175
|
+
)
|
|
176
|
+
os.replace(tmp, path)
|
|
177
|
+
except Exception:
|
|
178
|
+
try:
|
|
179
|
+
tmp.unlink()
|
|
180
|
+
except OSError:
|
|
181
|
+
pass
|
|
182
|
+
raise
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# ─── Anchor validation ─────────────────────────────────────────────────────
|
|
186
|
+
|
|
187
|
+
def _validate_anchors(anchors) -> None:
|
|
188
|
+
if anchors is None:
|
|
189
|
+
return
|
|
190
|
+
if not isinstance(anchors, list):
|
|
191
|
+
raise ValueError("anchors must be a list of {name, hex} objects")
|
|
192
|
+
for a in anchors:
|
|
193
|
+
if not isinstance(a, dict) or "name" not in a or "hex" not in a:
|
|
194
|
+
raise ValueError(f"invalid anchor entry (needs name + hex): {a!r}")
|
|
195
|
+
if not _HEX_RE.match(str(a["hex"])):
|
|
196
|
+
raise ValueError(
|
|
197
|
+
f"anchor {a.get('name')!r} has invalid hex color {a.get('hex')!r} (expected #rrggbb)"
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# ─── Shared hue/sat/val (ported from measure-texture.py's hue_bands) ──────
|
|
202
|
+
|
|
203
|
+
def _hsv_arrays(im: Image.Image, size=(240, 137)):
|
|
204
|
+
"""Vectorised hue(deg)/saturation/value over a resized copy.
|
|
205
|
+
|
|
206
|
+
Adopted from measure-texture.py's hue_bands() — itself a numerically
|
|
207
|
+
verified, vectorised reproduction of measure-palette.py's per-pixel
|
|
208
|
+
colorsys loop (see that function's docstring: "reproducing
|
|
209
|
+
measure-palette.py EXACTLY -- same 240x137 downscale, same s>=0.22 /
|
|
210
|
+
v>=0.18 gate ... Vectorised, but deliberately not 'improved'"). Returns
|
|
211
|
+
raw (hue_deg, sat, val) arrays instead of pre-binned percentages so the
|
|
212
|
+
caller can bucket by an arbitrary, project-supplied anchor list instead
|
|
213
|
+
of the two hardcoded salt-road bands.
|
|
214
|
+
|
|
215
|
+
The downscale is the same deliberate choice as the original: palette is
|
|
216
|
+
a low-frequency, gross-color-mass question, so shrinking first is fine
|
|
217
|
+
(contrast measure_texture below, which explicitly does NOT resize).
|
|
218
|
+
"""
|
|
219
|
+
a = np.asarray(im.convert("RGB").resize(size), dtype=np.float64) / 255.0
|
|
220
|
+
mx, mn = a.max(2), a.min(2)
|
|
221
|
+
d = mx - mn
|
|
222
|
+
s = np.where(mx > 0, d / np.maximum(mx, 1e-12), 0.0)
|
|
223
|
+
r, g, b = a[..., 0], a[..., 1], a[..., 2]
|
|
224
|
+
h = np.zeros_like(mx)
|
|
225
|
+
nz = d > 1e-12
|
|
226
|
+
with np.errstate(invalid="ignore"):
|
|
227
|
+
h = np.where(nz & (mx == r), ((g - b) / np.where(nz, d, 1)) % 6, h)
|
|
228
|
+
h = np.where(nz & (mx == g), (b - r) / np.where(nz, d, 1) + 2, h)
|
|
229
|
+
h = np.where(nz & (mx == b), (r - g) / np.where(nz, d, 1) + 4, h)
|
|
230
|
+
hd = (h * 60.0) % 360.0
|
|
231
|
+
return hd, s, mx # mx doubles as V in HSV
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _anchor_hue_deg(hex_color: str) -> float:
|
|
235
|
+
"""Ported verbatim from measure-palette.py's anchor_hue()."""
|
|
236
|
+
r, g, b = (int(hex_color[i:i + 2], 16) / 255 for i in (1, 3, 5))
|
|
237
|
+
return colorsys.rgb_to_hsv(r, g, b)[0] * 360.0
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _circular_hue_distance(a, b):
|
|
241
|
+
d = np.abs(a - b) % 360.0
|
|
242
|
+
return np.minimum(d, 360.0 - d)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
# ─── Palette measurement ────────────────────────────────────────────────
|
|
246
|
+
|
|
247
|
+
def measure_palette(im: Image.Image, anchors, hue_tolerance_deg=DEFAULT_HUE_TOLERANCE_DEG,
|
|
248
|
+
sat_min=DEFAULT_SAT_MIN, val_min=DEFAULT_VAL_MIN) -> dict:
|
|
249
|
+
"""Numeric palette signal. `anchors` is a (possibly empty) list of
|
|
250
|
+
{name, hex}; with no anchors, only the gate stats are returned (still
|
|
251
|
+
useful signal: how much of the frame is "colorful" at all, and its mean
|
|
252
|
+
saturation/value) — this lets sdlab measure run on a project with no
|
|
253
|
+
authored anchor file yet instead of refusing outright.
|
|
254
|
+
|
|
255
|
+
Percentages are relative to the GATED pixel count (pixels that passed
|
|
256
|
+
the saturation/value gate), not the whole frame — ported from
|
|
257
|
+
measure-palette.py's `tot = yellow + redochre + other` (a sum of gated
|
|
258
|
+
bins only). A sky or a shadow dominating raw pixel count would otherwise
|
|
259
|
+
dilute the signal this measure exists to read.
|
|
260
|
+
"""
|
|
261
|
+
hd, s, v = _hsv_arrays(im)
|
|
262
|
+
keep = (s >= sat_min) & (v >= val_min)
|
|
263
|
+
gated = int(keep.sum())
|
|
264
|
+
frame_total = int(hd.size)
|
|
265
|
+
|
|
266
|
+
result = {
|
|
267
|
+
"gated_pct": round(100.0 * gated / frame_total, 2) if frame_total else 0.0,
|
|
268
|
+
"mean_saturation": round(float(s[keep].mean()), 4) if gated else 0.0,
|
|
269
|
+
"mean_value": round(float(v[keep].mean()), 4) if gated else 0.0,
|
|
270
|
+
"hue_tolerance_deg": hue_tolerance_deg,
|
|
271
|
+
"anchors": {},
|
|
272
|
+
"off_anchor_pct": None,
|
|
273
|
+
}
|
|
274
|
+
if not anchors or gated == 0:
|
|
275
|
+
return result
|
|
276
|
+
|
|
277
|
+
hd_gated = hd[keep]
|
|
278
|
+
names = [a["name"] for a in anchors]
|
|
279
|
+
anchor_hues = [_anchor_hue_deg(a["hex"]) for a in anchors]
|
|
280
|
+
|
|
281
|
+
# Nearest-anchor assignment within tolerance — see module docstring for
|
|
282
|
+
# why this generalizes measure-palette.py's two hand-picked bands.
|
|
283
|
+
dists = np.stack([_circular_hue_distance(hd_gated, hue) for hue in anchor_hues], axis=0)
|
|
284
|
+
nearest_idx = np.argmin(dists, axis=0)
|
|
285
|
+
nearest_dist = np.min(dists, axis=0)
|
|
286
|
+
within = nearest_dist <= hue_tolerance_deg
|
|
287
|
+
|
|
288
|
+
for i, name in enumerate(names):
|
|
289
|
+
count = int(((nearest_idx == i) & within).sum())
|
|
290
|
+
result["anchors"][name] = round(100.0 * count / gated, 2)
|
|
291
|
+
result["off_anchor_pct"] = round(100.0 * int((~within).sum()) / gated, 2)
|
|
292
|
+
return result
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# ─── Texture measurement (ported from measure-texture.py's measure()) ────
|
|
296
|
+
|
|
297
|
+
def measure_texture(im: Image.Image) -> dict:
|
|
298
|
+
"""Numeric texture/impasto signal, ported from measure-texture.py's
|
|
299
|
+
measure(). Deliberately kept at NATIVE resolution — no resize, no
|
|
300
|
+
re-encode: per that file's docstring, "downscaling is the exact
|
|
301
|
+
operation that destroys the signal we are looking for". See that file
|
|
302
|
+
for the full rationale on why each measure exists (flat_hf is the
|
|
303
|
+
headline: impasto is texture that survives into nominally flat regions,
|
|
304
|
+
which is what separates "the surface is painted" from "the frame has a
|
|
305
|
+
lot of stuff in it").
|
|
306
|
+
|
|
307
|
+
Excludes facade_rep and the yellow/ochre/green hue bands — see the
|
|
308
|
+
module docstring's "Deliberate changes" section.
|
|
309
|
+
"""
|
|
310
|
+
if im.mode not in ("RGB", "RGBA"):
|
|
311
|
+
raise ValueError(f"unexpected image mode {im.mode!r} (expected RGB or RGBA)")
|
|
312
|
+
if im.width < 8 or im.height < 8:
|
|
313
|
+
# Not present in the original (real 1344x768 salt-road plates never
|
|
314
|
+
# underflowed the percentile windows below) — added defensively so a
|
|
315
|
+
# tiny/degenerate image fails with a clear message instead of a
|
|
316
|
+
# cryptic numpy exception from np.percentile on a near-empty array.
|
|
317
|
+
raise ValueError(f"image too small to measure ({im.width}x{im.height}); minimum 8x8")
|
|
318
|
+
|
|
319
|
+
a = np.asarray(im.convert("RGB"), dtype=np.float64)
|
|
320
|
+
lum = 0.2126 * a[..., 0] + 0.7152 * a[..., 1] + 0.0722 * a[..., 2]
|
|
321
|
+
|
|
322
|
+
# --- high-pass residual: everything finer than ~1.5px ---
|
|
323
|
+
hp = lum - gaussian_filter(lum, sigma=1.5)
|
|
324
|
+
|
|
325
|
+
# --- large-scale structure gradient: where are the object edges? ---
|
|
326
|
+
struct = gaussian_filter(lum, sigma=6.0)
|
|
327
|
+
gx, gy = sobel(struct, axis=1), sobel(struct, axis=0)
|
|
328
|
+
sgrad = np.hypot(gx, gy)
|
|
329
|
+
|
|
330
|
+
# flattest 40% of the frame by large-scale gradient
|
|
331
|
+
flat = sgrad <= np.percentile(sgrad, 40)
|
|
332
|
+
|
|
333
|
+
# --- laplacian variance ---
|
|
334
|
+
lap = (-4 * lum
|
|
335
|
+
+ np.roll(lum, 1, 0) + np.roll(lum, -1, 0)
|
|
336
|
+
+ np.roll(lum, 1, 1) + np.roll(lum, -1, 1))[1:-1, 1:-1]
|
|
337
|
+
|
|
338
|
+
# --- edge density on the raw luminance, fixed absolute threshold ---
|
|
339
|
+
ex, ey = sobel(lum, axis=1), sobel(lum, axis=0)
|
|
340
|
+
egrad = np.hypot(ex, ey) / 4.0 # sobel kernel gain
|
|
341
|
+
|
|
342
|
+
# --- acutance: how much do the strongest edges survive a sigma-2 blur? ---
|
|
343
|
+
blur = gaussian_filter(lum, 2.0)
|
|
344
|
+
bgrad = np.hypot(sobel(blur, axis=1), sobel(blur, axis=0)) / 4.0
|
|
345
|
+
strong = egrad >= np.percentile(egrad, 98)
|
|
346
|
+
edge_hard = float(egrad[strong].mean() / max(bgrad[strong].mean(), 1e-9)) if strong.any() else 0.0
|
|
347
|
+
|
|
348
|
+
# --- directionality of the fine residual inside the flat mask ---
|
|
349
|
+
hx, hy = sobel(hp, axis=1), sobel(hp, axis=0)
|
|
350
|
+
Jxx = gaussian_filter(hx * hx, 4.0)
|
|
351
|
+
Jyy = gaussian_filter(hy * hy, 4.0)
|
|
352
|
+
Jxy = gaussian_filter(hx * hy, 4.0)
|
|
353
|
+
disc = np.sqrt(((Jxx - Jyy) / 2.0) ** 2 + Jxy ** 2)
|
|
354
|
+
tr = (Jxx + Jyy) / 2.0
|
|
355
|
+
coh = 2.0 * disc / np.maximum(tr * 2.0, 1e-9)
|
|
356
|
+
|
|
357
|
+
# --- radial power spectrum ---
|
|
358
|
+
win = np.hanning(lum.shape[0])[:, None] * np.hanning(lum.shape[1])[None, :]
|
|
359
|
+
F = np.fft.fftshift(np.abs(np.fft.fft2((lum - lum.mean()) * win)) ** 2)
|
|
360
|
+
cy, cx = (np.array(F.shape) - 1) / 2.0
|
|
361
|
+
yy, xx = np.ogrid[:F.shape[0], :F.shape[1]]
|
|
362
|
+
r = np.hypot((yy - cy) / (F.shape[0] / 2.0), (xx - cx) / (F.shape[1] / 2.0))
|
|
363
|
+
tot = F.sum()
|
|
364
|
+
|
|
365
|
+
return {
|
|
366
|
+
"lap_var": round(float(lap.var()), 6),
|
|
367
|
+
"hf_ratio": round(float(hp.std() / max(lum.std(), 1e-9)), 6),
|
|
368
|
+
"flat_hf": round(float(hp[flat].std()), 6),
|
|
369
|
+
"edge_dens": round(float((egrad > 12.0).mean()), 6),
|
|
370
|
+
"band_hi": round(float(F[r > 0.30].sum() / tot), 6),
|
|
371
|
+
"band_mid": round(float(F[(r > 0.10) & (r <= 0.30)].sum() / tot), 6),
|
|
372
|
+
"edge_hard": round(edge_hard, 6),
|
|
373
|
+
"flat_coh": round(float(coh[flat].mean()), 6),
|
|
374
|
+
"lum_std": round(float(lum.std()), 6),
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
# ─── Per-image measurement + CLI driver ────────────────────────────────
|
|
379
|
+
|
|
380
|
+
def measure_one(path: str, anchors, hue_tolerance_deg, sat_min, val_min) -> dict:
|
|
381
|
+
"""Open the image ONCE and run both passes over it — mirrors
|
|
382
|
+
measure-texture.py's own discipline ("every image measured for texture
|
|
383
|
+
is measured for colour in the same pass"), generalized to palette+texture
|
|
384
|
+
instead of texture+yellow-band."""
|
|
385
|
+
with Image.open(path) as im:
|
|
386
|
+
im.load()
|
|
387
|
+
texture = measure_texture(im)
|
|
388
|
+
palette = measure_palette(im, anchors, hue_tolerance_deg, sat_min, val_min)
|
|
389
|
+
return {
|
|
390
|
+
"image": {"width": im.width, "height": im.height, "mode": im.mode},
|
|
391
|
+
"palette": palette,
|
|
392
|
+
"texture": texture,
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def main() -> None:
|
|
397
|
+
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
398
|
+
ap.add_argument("--request", required=True, type=Path)
|
|
399
|
+
ap.add_argument("--out", required=True, type=Path)
|
|
400
|
+
args = ap.parse_args()
|
|
401
|
+
|
|
402
|
+
try:
|
|
403
|
+
request = json.loads(args.request.read_text(encoding="utf-8"))
|
|
404
|
+
except FileNotFoundError:
|
|
405
|
+
raise SystemExit(f"request file not found: {args.request}")
|
|
406
|
+
except json.JSONDecodeError as exc:
|
|
407
|
+
raise SystemExit(f"request file is not valid JSON: {args.request}: {exc}")
|
|
408
|
+
|
|
409
|
+
images = request.get("images", [])
|
|
410
|
+
anchors = request.get("anchors") or []
|
|
411
|
+
hue_tolerance_deg = request.get("hue_tolerance_deg", DEFAULT_HUE_TOLERANCE_DEG)
|
|
412
|
+
sat_min = request.get("sat_min", DEFAULT_SAT_MIN)
|
|
413
|
+
val_min = request.get("val_min", DEFAULT_VAL_MIN)
|
|
414
|
+
|
|
415
|
+
try:
|
|
416
|
+
_validate_anchors(anchors)
|
|
417
|
+
except ValueError as exc:
|
|
418
|
+
raise SystemExit(f"invalid anchors in request: {exc}")
|
|
419
|
+
|
|
420
|
+
results = []
|
|
421
|
+
errors = []
|
|
422
|
+
for item in images:
|
|
423
|
+
iid = item.get("id")
|
|
424
|
+
path = item.get("path")
|
|
425
|
+
try:
|
|
426
|
+
measured = measure_one(path, anchors, hue_tolerance_deg, sat_min, val_min)
|
|
427
|
+
results.append({"id": iid, "path": path, "status": "ok", **measured})
|
|
428
|
+
print(f" {iid} -> ok", flush=True)
|
|
429
|
+
except Exception as exc: # noqa: BLE001 — one bad image must not sink the batch
|
|
430
|
+
errors.append({"id": iid, "path": path, "error": f"{type(exc).__name__}: {exc}"})
|
|
431
|
+
print(f" {iid} -> ERROR: {exc}", flush=True)
|
|
432
|
+
|
|
433
|
+
payload = {
|
|
434
|
+
"schema_version": MEASURE_SCHEMA_VERSION,
|
|
435
|
+
"tool": "measure_image.py",
|
|
436
|
+
"generated_at": _utc_now_iso(),
|
|
437
|
+
"results": results,
|
|
438
|
+
"errors": errors,
|
|
439
|
+
"ok_count": len(results),
|
|
440
|
+
"error_count": len(errors),
|
|
441
|
+
}
|
|
442
|
+
_atomic_write_json(args.out, payload)
|
|
443
|
+
|
|
444
|
+
summary = f"\n{len(results)}/{len(images)} image(s) measured -> {args.out}"
|
|
445
|
+
if errors:
|
|
446
|
+
summary += f" ({len(errors)} error(s) - see {args.out} for per-item detail)"
|
|
447
|
+
print(summary)
|
|
448
|
+
|
|
449
|
+
# Mirror qwen_generate.py's convention: a run where EVERY item failed
|
|
450
|
+
# exits non-zero (nothing to salvage); a partial failure still exits 0 —
|
|
451
|
+
# per-item errors are visible above and in the written result file, and
|
|
452
|
+
# individual item failure is tolerated everywhere else in this pipeline.
|
|
453
|
+
if images and len(errors) == len(images):
|
|
454
|
+
raise SystemExit(f"all {len(errors)} measurement attempt(s) failed - see {args.out} for per-item errors")
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
if __name__ == "__main__":
|
|
458
|
+
main()
|