@saifymatteo/pi-vision 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE.md +5 -0
- package/README.md +42 -2
- package/examples/vision-toolkit.example.json +12 -0
- package/package.json +36 -4
- package/skills/vision-tools/LICENSE +21 -0
- package/skills/vision-tools/SKILL.md +134 -0
- package/skills/vision-tools/agents/openai.yaml +4 -0
- package/skills/vision-tools/bin/crop +94 -0
- package/skills/vision-tools/bin/detect +18 -0
- package/skills/vision-tools/bin/glance +98 -0
- package/skills/vision-tools/bin/ground +18 -0
- package/skills/vision-tools/bin/trace +134 -0
- package/skills/vision-tools/bin/vision-doctor +168 -0
- package/skills/vision-tools/detect.py +56 -0
- package/skills/vision-tools/ground.py +216 -0
- package/skills/vision-tools/references/gui.md +93 -0
- package/skills/vision-tools/references/long-screenshot-ocr.md +103 -0
- package/skills/vision-tools/references/restore-graphic.md +92 -0
- package/skills/vision-tools/references/restore-structure.md +47 -0
- package/skills/vision-tools/references/restore-ui.md +200 -0
- package/skills/vision-tools/scripts/dominant_colors.py +224 -0
- package/skills/vision-tools/scripts/extract_fg.py +278 -0
- package/skills/vision-tools/scripts/html_shot.py +111 -0
- package/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
- package/skills/vision-tools/scripts/pixel_diff.py +88 -0
- package/skills/vision-tools/vision_client.py +581 -0
- package/src/index.ts +728 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Extracting or rebuilding a graphic asset
|
|
2
|
+
|
|
3
|
+
**When to use**: the deliverable is an isolated icon, logo, illustration,
|
|
4
|
+
decorative mark, or other visual asset as a transparent PNG or SVG. For a full
|
|
5
|
+
page or component, read `restore-ui.md` and use this workflow only for the
|
|
6
|
+
individual visual assets it identifies.
|
|
7
|
+
|
|
8
|
+
## Choose the least destructive representation
|
|
9
|
+
|
|
10
|
+
Use this order:
|
|
11
|
+
|
|
12
|
+
1. Reuse the original source asset if it already exists in the project.
|
|
13
|
+
2. Extract a transparent PNG from the reference when the visual only needs to
|
|
14
|
+
be displayed at the captured size. This preserves the original pixels and
|
|
15
|
+
is the default for screenshot-only brand artwork.
|
|
16
|
+
3. Trace an irregular, flat, high-contrast shape when SVG is required.
|
|
17
|
+
4. Hand-write simple editable geometry or stroke icons, using the trace or
|
|
18
|
+
pixel measurements as evidence rather than visual estimates.
|
|
19
|
+
|
|
20
|
+
## Extract a transparent PNG
|
|
21
|
+
|
|
22
|
+
The final extraction region must be tight around the target ink. A loose crop
|
|
23
|
+
that touches neighboring text, borders, or decoration may preserve those
|
|
24
|
+
pixels as foreground.
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# Known tight region
|
|
28
|
+
python3 scripts/extract_fg.py shot.png \
|
|
29
|
+
--region X1,Y1,X2,Y2 -o asset.png
|
|
30
|
+
|
|
31
|
+
# Dark or gray line art
|
|
32
|
+
python3 scripts/extract_fg.py shot.png \
|
|
33
|
+
--region X1,Y1,X2,Y2 --mode dark -o asset.png
|
|
34
|
+
|
|
35
|
+
# Center and enlarge a small visual before automatic extraction
|
|
36
|
+
crop shot.png --region X1,Y1,X2,Y2 --scale 4 -o work/asset4x.png
|
|
37
|
+
python3 scripts/extract_fg.py work/asset4x.png
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Use `--exclude-color '#RRGGBB'` when a connected background color would
|
|
41
|
+
otherwise be retained. If automatic centering is wrong, provide a tighter
|
|
42
|
+
region or a target box rather than repeatedly widening the crop.
|
|
43
|
+
|
|
44
|
+
Inspect the result on both light and dark backgrounds. Confirm complete ink,
|
|
45
|
+
clean alpha edges, and no neighboring fragments before reuse.
|
|
46
|
+
|
|
47
|
+
## Rebuild an SVG only when the deliverable needs one
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
trace asset.png -o asset.svg
|
|
51
|
+
trace asset.png --polygon -o asset.svg
|
|
52
|
+
trace shot.png --region X1,Y1,X2,Y2 -o asset.svg
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Ship traced paths directly for organic or irregular shapes. For rectangles,
|
|
56
|
+
circles, pills, and other simple editable geometry, use the trace as an
|
|
57
|
+
approximate outline and write the simpler SVG yourself. Filtering, upscaling
|
|
58
|
+
and curve fitting can change boundaries; measure pixels directly when exact
|
|
59
|
+
dimensions matter.
|
|
60
|
+
|
|
61
|
+
A small stroke icon often suits a hand-written SVG: tracing outlines both
|
|
62
|
+
sides of the raster stroke, while the desired asset is normally a centerline
|
|
63
|
+
path with `stroke` and `fill="none"`. If an outline would help, use
|
|
64
|
+
`trace --polygon` as a reference for endpoints and corners, then write and
|
|
65
|
+
render the clean centerline path.
|
|
66
|
+
|
|
67
|
+
When reusing traced paths:
|
|
68
|
+
|
|
69
|
+
- Composite transparent SVG output on the intended background before judging
|
|
70
|
+
it; transparent regions may appear black in some viewers.
|
|
71
|
+
- Copy each path's `transform` together with its `d` data. Dropping the
|
|
72
|
+
transform displaces otherwise correct geometry.
|
|
73
|
+
- Keep opposite-winding subpaths together so holes remain holes.
|
|
74
|
+
|
|
75
|
+
## Verify
|
|
76
|
+
|
|
77
|
+
Render the asset at its intended size and compare it with the same tight box
|
|
78
|
+
from the reference. Use an overlay or `scripts/pixel_diff.py` to locate missing
|
|
79
|
+
parts, contamination, wrong scale, or displaced paths. Judge the asset at its
|
|
80
|
+
actual delivery size as well as zoomed in; a technically detailed SVG that
|
|
81
|
+
looks worse at 16px is not an improvement.
|
|
82
|
+
|
|
83
|
+
## Boundaries
|
|
84
|
+
|
|
85
|
+
- Whole screenshots and photos do not trace usefully.
|
|
86
|
+
- Low-contrast art may disappear during binarization. Tighten the region,
|
|
87
|
+
increase scale, or invert light-on-dark input before reaching for color
|
|
88
|
+
tracing.
|
|
89
|
+
- Use color tracing only for genuinely multicolor artwork; antialiased single-
|
|
90
|
+
color input can otherwise fragment into many gray-level paths.
|
|
91
|
+
- If the visual contains live text, controls, or data-driven content, rebuild
|
|
92
|
+
those parts natively and use the extracted asset only for the static artwork.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Turning a visual diagram into structured code
|
|
2
|
+
|
|
3
|
+
**When to use**: the task is converting a sketch, flowchart, architecture
|
|
4
|
+
diagram, or whiteboard into Mermaid, Graphviz, JSON, or another structured
|
|
5
|
+
representation. The goal is semantic and topological fidelity, not matching
|
|
6
|
+
the source renderer pixel for pixel.
|
|
7
|
+
|
|
8
|
+
## Steps
|
|
9
|
+
|
|
10
|
+
1. Inspect the image natively with the main agent's image reader for diagram
|
|
11
|
+
type, reading direction, major groups, and visual conventions.
|
|
12
|
+
2. Inventory nodes and labels. Preserve visible spelling, punctuation, and
|
|
13
|
+
abbreviations verbatim; write `[unreadable]` instead of guessing.
|
|
14
|
+
3. For ambiguous arrows, connectors, labels, legends, or group boundaries,
|
|
15
|
+
crop the relevant region and read it natively. Use `detect`, `ground`, or
|
|
16
|
+
`glance` (including OCR) only when a specialist pass would resolve a
|
|
17
|
+
remaining question. Model boxes are estimates: they can select the wrong
|
|
18
|
+
object or omit substantial parts, so verify them against the image.
|
|
19
|
+
4. Build an explicit intermediate inventory before writing the output:
|
|
20
|
+
nodes, labels, groups, edges, directions, and edge labels.
|
|
21
|
+
5. Generate the Mermaid, Graphviz, or requested structure from that inventory.
|
|
22
|
+
|
|
23
|
+
## Verify structure, not pixels
|
|
24
|
+
|
|
25
|
+
Render the result, inventory it again, and compare these facts with the
|
|
26
|
+
reference:
|
|
27
|
+
|
|
28
|
+
- node count and label set
|
|
29
|
+
- group membership and nesting
|
|
30
|
+
- edge list and direction
|
|
31
|
+
- edge labels and branch conditions
|
|
32
|
+
- start/end or input/output roles
|
|
33
|
+
|
|
34
|
+
Pixel differences are expected because the target renderer chooses its own
|
|
35
|
+
fonts, spacing, and routing. A missing node or reversed arrow is a failure even
|
|
36
|
+
when the two images look broadly similar; a different curve or gap is not a
|
|
37
|
+
failure when the structure is correct.
|
|
38
|
+
|
|
39
|
+
## Boundaries
|
|
40
|
+
|
|
41
|
+
- Do not tidy labels, expand abbreviations, or silently repair wording unless
|
|
42
|
+
the user asks for editorial cleanup.
|
|
43
|
+
- Do not infer an edge from proximity alone. If its endpoint or direction is
|
|
44
|
+
unclear, zoom into the connector and mark unresolved facts explicitly.
|
|
45
|
+
- If the user wants a visual SVG copy rather than structured code, follow
|
|
46
|
+
`restore-graphic.md` instead. `trace` is approximate vectorization, not
|
|
47
|
+
exact measurement or a reliable source of graph topology.
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# Rebuilding a UI from a reference image
|
|
2
|
+
|
|
3
|
+
**When to use**: the task is turning a screenshot or design image into a
|
|
4
|
+
page, component, or application UI in HTML/CSS or the project's existing
|
|
5
|
+
frontend stack. Checking an existing implementation against its reference is
|
|
6
|
+
the same workflow entered at Verify. For an isolated icon, logo, or
|
|
7
|
+
illustration, read `restore-graphic.md`; for Mermaid, Graphviz, or another
|
|
8
|
+
structured diagram, read `restore-structure.md`.
|
|
9
|
+
|
|
10
|
+
## Choose the restore mode
|
|
11
|
+
|
|
12
|
+
- Use **fast restore mode** when the user asks for a quick, rough,
|
|
13
|
+
approximate, prototype, or first-pass reconstruction, or explicitly values
|
|
14
|
+
speed over fidelity. Its target is a recognizable screenshot in about three
|
|
15
|
+
minutes when the project already runs.
|
|
16
|
+
- Use the **standard restore workflow** below when the user asks for close,
|
|
17
|
+
precise, pixel-level, or production-ready alignment, or does not opt into a
|
|
18
|
+
faster approximation.
|
|
19
|
+
|
|
20
|
+
## Fast restore mode: first screenshot in about three minutes
|
|
21
|
+
|
|
22
|
+
Fast mode preserves the page's hierarchy, major regions, visible text, and
|
|
23
|
+
primary state. It deliberately approximates fine spacing, exact colors,
|
|
24
|
+
typography, shadows, decorative details, and icon geometry.
|
|
25
|
+
|
|
26
|
+
### Hard limits
|
|
27
|
+
|
|
28
|
+
1. Inspect the existing frontend stack, component library, icon set, source
|
|
29
|
+
assets, and design tokens before writing code, but stop searching as soon as
|
|
30
|
+
a usable local primitive is found.
|
|
31
|
+
2. Inspect the reference natively with the main agent's image reader for
|
|
32
|
+
hierarchy, major regions, visible text, and state. Crop and read a region
|
|
33
|
+
natively only when a material detail remains unresolved; skip exhaustive
|
|
34
|
+
region-by-region inventory. Normally one or two inspection rounds suffice.
|
|
35
|
+
3. Do not use `trace`, foreground extraction, repeated color sampling, or
|
|
36
|
+
iterative `pixel_diff` work in fast mode. Those are fidelity tools and will
|
|
37
|
+
consume the delivery window.
|
|
38
|
+
|
|
39
|
+
### Build the approximation
|
|
40
|
+
|
|
41
|
+
1. Implement the largest layout regions first, then visible text, primary
|
|
42
|
+
controls, and the most important state. Ignore details that are only visible
|
|
43
|
+
when zoomed in.
|
|
44
|
+
2. Reuse the project's existing components and CSS tokens. If its frontend or
|
|
45
|
+
icon library contains a reasonably similar component or icon, use it
|
|
46
|
+
directly instead of recreating the reference. Prefer an approximate library
|
|
47
|
+
icon over cropping, tracing, or hand-drawing a new one.
|
|
48
|
+
3. Use nearby existing palette tokens or visually similar CSS values. Exact
|
|
49
|
+
sampled hex values, gradients, subtle borders, and shadow opacity are out of
|
|
50
|
+
scope unless one of them defines the whole composition.
|
|
51
|
+
4. Keep text and controls native, selectable, and interactive. Fast mode relaxes
|
|
52
|
+
visual fidelity, not basic UI behavior.
|
|
53
|
+
|
|
54
|
+
### Render once, fix once, deliver
|
|
55
|
+
|
|
56
|
+
1. Render the target viewport with `scripts/html_shot.py` or the project's
|
|
57
|
+
existing browser setup.
|
|
58
|
+
2. Inspect the screenshot once. If there is an obvious structural failure such
|
|
59
|
+
as a missing major region, broken wrapping, or a wildly wrong scale, make
|
|
60
|
+
one focused correction and render once more.
|
|
61
|
+
3. Deliver the screenshot. Stop instead of spending the remaining time on
|
|
62
|
+
small color, icon, font, shadow, radius, or spacing differences.
|
|
63
|
+
|
|
64
|
+
A practical time box is roughly 30 seconds for project and native image
|
|
65
|
+
inspection, 90 seconds for implementation, and the remaining minute for startup,
|
|
66
|
+
rendering, one correction, and screenshot delivery. Dependency installation or
|
|
67
|
+
a project that does not already run may extend that target; do not compensate
|
|
68
|
+
by silently switching back to a long precision loop.
|
|
69
|
+
|
|
70
|
+
## Standard restore workflow
|
|
71
|
+
|
|
72
|
+
### Core strategy: code-native UI plus screenshot-backed visuals
|
|
73
|
+
|
|
74
|
+
Do not choose one reconstruction mode for the whole page. Classify each
|
|
75
|
+
element separately. Most finished pages should combine both kinds:
|
|
76
|
+
|
|
77
|
+
| Kind | Build it this way | Typical examples |
|
|
78
|
+
|---|---|---|
|
|
79
|
+
| **Code-native component** | Reuse the project's component library or build it with semantic HTML, CSS, and simple SVG primitives | layout, text, buttons, inputs, cards, tabs, dividers, backgrounds, simple geometric badges |
|
|
80
|
+
| **Screenshot-backed visual** | Extract the original pixels, remove the background when needed, and place the result as an image asset | logos, proprietary icons, illustrations, decorative artwork, textures, complex static marks |
|
|
81
|
+
|
|
82
|
+
Use this decision order for every element:
|
|
83
|
+
|
|
84
|
+
1. **Reuse an existing component or source asset from the project.** Search
|
|
85
|
+
before recreating anything.
|
|
86
|
+
2. **Use a code-native component** when the element contains text, accepts
|
|
87
|
+
input, changes state, must respond to layout, or is simple to express with
|
|
88
|
+
the project's normal primitives.
|
|
89
|
+
3. **Use a screenshot-backed visual** when it is static, visually distinctive,
|
|
90
|
+
expensive to redraw, and no source asset exists.
|
|
91
|
+
4. Rebuild a vector only when the user requires SVG/editability, the visual
|
|
92
|
+
must scale beyond the screenshot resolution, or extraction cannot isolate
|
|
93
|
+
it cleanly. Follow `restore-graphic.md` for that branch. `trace` provides
|
|
94
|
+
approximate vectorization, not exact geometric measurement.
|
|
95
|
+
|
|
96
|
+
Never flatten text, controls, or a large layout block into a screenshot. Never
|
|
97
|
+
redraw extractable brand artwork merely to make the implementation look more
|
|
98
|
+
"native". For a complex widget, keep the shell, text, and interaction native;
|
|
99
|
+
use an extracted image only for its static decorative layer.
|
|
100
|
+
|
|
101
|
+
## Steps
|
|
102
|
+
|
|
103
|
+
### 1. Inspect the reference and target project
|
|
104
|
+
|
|
105
|
+
Read the reference directly at its current path. Copy it to durable work
|
|
106
|
+
storage only when it must persist beyond the temporary file's lifetime.
|
|
107
|
+
Inspect the existing stack, component library, fonts, icons, images, and
|
|
108
|
+
design tokens before writing replacements.
|
|
109
|
+
|
|
110
|
+
### 2. Establish the coordinate system
|
|
111
|
+
|
|
112
|
+
Record the reference image dimensions and the target viewport. Screenshots may
|
|
113
|
+
be HiDPI: `detect`, `ground`, and `crop` report image pixels, while CSS uses
|
|
114
|
+
logical pixels. Derive the scale from the actual image and viewport dimensions;
|
|
115
|
+
do not assume a fixed 2x ratio.
|
|
116
|
+
|
|
117
|
+
### 3. Inventory the UI outside-in
|
|
118
|
+
|
|
119
|
+
1. Inspect the image natively for hierarchy, component types, visible text,
|
|
120
|
+
state, and styling; list the major regions outside-in.
|
|
121
|
+
2. For unresolved details, crop the relevant region and read it natively.
|
|
122
|
+
3. Use specialist tools only for a remaining concrete need: `detect` for an
|
|
123
|
+
inventory, `glance` for a second opinion or OCR, `ground` for a target box.
|
|
124
|
+
|
|
125
|
+
Model boxes are estimates and can identify the wrong element or miss large
|
|
126
|
+
parts of it. Verify them against the pixels before using them for crops or
|
|
127
|
+
layout; they are not measured boundaries.
|
|
128
|
+
|
|
129
|
+
### 4. Classify before implementing
|
|
130
|
+
|
|
131
|
+
Make a short element list with one of these decisions: reuse existing source,
|
|
132
|
+
code-native, screenshot-backed, or hybrid. This prevents two costly mistakes:
|
|
133
|
+
rebuilding complex artwork from scratch and turning functional UI into static
|
|
134
|
+
image patches.
|
|
135
|
+
|
|
136
|
+
### 5. Extract screenshot-backed visuals
|
|
137
|
+
|
|
138
|
+
Use a padded crop only to inspect and center the target. Before foreground
|
|
139
|
+
extraction, tighten the final region to the visual's own ink; adjacent text or
|
|
140
|
+
rules inside the region can become foreground components too.
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
crop shot.png --region X1,Y1,X2,Y2 --scale 4 -o work/icon4x.png
|
|
144
|
+
python3 scripts/extract_fg.py work/icon4x.png
|
|
145
|
+
|
|
146
|
+
# Or extract directly from a known tight region:
|
|
147
|
+
python3 scripts/extract_fg.py shot.png \
|
|
148
|
+
--region X1,Y1,X2,Y2 -o work/icon.png
|
|
149
|
+
python3 scripts/extract_fg.py shot.png \
|
|
150
|
+
--region X1,Y1,X2,Y2 --mode dark -o work/logo.png
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Inspect the transparent result before use. Confirm that the whole visual is
|
|
154
|
+
present, no neighboring text or border leaked in, and the alpha edge remains
|
|
155
|
+
clean. When extracting several assets, inspect them together in a contact
|
|
156
|
+
sheet instead of checking them from filenames alone.
|
|
157
|
+
|
|
158
|
+
### 6. Implement the native frame
|
|
159
|
+
|
|
160
|
+
- Follow the project's existing framework and component patterns.
|
|
161
|
+
- Build content, interaction, responsive layout, borders, shadows, and simple
|
|
162
|
+
shapes natively.
|
|
163
|
+
- Use `scripts/dominant_colors.py` when exact background, text, or accent
|
|
164
|
+
values matter; visual inspection can name a color but cannot measure its
|
|
165
|
+
numeric value.
|
|
166
|
+
- Place extracted assets with explicit logical dimensions. Preserve their
|
|
167
|
+
aspect ratio and avoid baking surrounding whitespace into the asset.
|
|
168
|
+
- Match structure and proportions before tuning small spacing.
|
|
169
|
+
|
|
170
|
+
## Verify and converge by impact
|
|
171
|
+
|
|
172
|
+
For an existing implementation, start here.
|
|
173
|
+
|
|
174
|
+
1. Render at the same logical viewport with `scripts/html_shot.py` or the
|
|
175
|
+
project's browser test setup.
|
|
176
|
+
2. Compare the render and reference at the same dimensions. Inspect them side
|
|
177
|
+
by side; use `scripts/pixel_diff.py` to locate differences that are hard to
|
|
178
|
+
spot or explain.
|
|
179
|
+
3. Fix material discrepancies: missing or wrong content, incorrect hierarchy,
|
|
180
|
+
broken wrapping, visibly wrong alignment or scale, wrong component state,
|
|
181
|
+
distorted assets, and clearly different primary colors.
|
|
182
|
+
4. Re-render after a meaningful fix. Continue while the comparison exposes a
|
|
183
|
+
user-visible defect; stop when the remaining difference is font
|
|
184
|
+
rasterization, antialiasing, subpixel placement, or another imperceptible
|
|
185
|
+
rendering detail.
|
|
186
|
+
|
|
187
|
+
`pixel_diff` is a locator, not the acceptance target. Do not keep iterating only
|
|
188
|
+
to lower its aggregate percentage, and do not promise zero-difference output
|
|
189
|
+
from a screenshot without a separately scoped browser, font, viewport, and
|
|
190
|
+
rendering environment.
|
|
191
|
+
|
|
192
|
+
## Boundaries
|
|
193
|
+
|
|
194
|
+
- Do not ship a screenshot of the whole page as the implementation.
|
|
195
|
+
- Do not use screenshot-backed text or controls that must remain selectable,
|
|
196
|
+
accessible, interactive, localized, or responsive.
|
|
197
|
+
- Do not stretch an extracted bitmap beyond the resolution supported by the
|
|
198
|
+
reference. Obtain a source asset or rebuild a vector when it must scale.
|
|
199
|
+
- Always compare against the original reference, never only against an earlier
|
|
200
|
+
version of your own render.
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""dominant_colors: a region's significant colours, and the exact value among a candidate palette.
|
|
3
|
+
|
|
4
|
+
Two halves of one job. First you extract the region's palette -- downsample,
|
|
5
|
+
quantize, merge near-duplicates -- so you can see which colours are actually
|
|
6
|
+
there and how much of the region each one owns. That share histogram is the
|
|
7
|
+
role map: a background takes a big share, an accent a small one.
|
|
8
|
+
|
|
9
|
+
A vision model (glance) names those clusters in prose ("white", "light gray",
|
|
10
|
+
"orange") but not their values. So for the value itself you supply the
|
|
11
|
+
candidate palette your label implies ("light gray" -> #F9FAFA / #F5F5F5 /
|
|
12
|
+
#F3F3F3 / #EDEDED) and this script decides from the pixels: each candidate is
|
|
13
|
+
scored by a distance filter over the region's pixels, and the best one wins.
|
|
14
|
+
The label comes from the model; the number always comes from here.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
from PIL import Image, ImageChops
|
|
24
|
+
except ImportError:
|
|
25
|
+
Image = None
|
|
26
|
+
ImageChops = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_rgb(path: Path) -> "Image.Image":
|
|
30
|
+
# Composite transparency on white the way a viewer would; unflattened
|
|
31
|
+
# alpha reads as black and skews every colour statistic.
|
|
32
|
+
image = Image.open(path)
|
|
33
|
+
if image.mode in ("RGBA", "LA", "P"):
|
|
34
|
+
image = image.convert("RGBA")
|
|
35
|
+
canvas = Image.new("RGB", image.size, "white")
|
|
36
|
+
canvas.paste(image, mask=image.split()[-1])
|
|
37
|
+
image = canvas
|
|
38
|
+
else:
|
|
39
|
+
image = image.convert("RGB")
|
|
40
|
+
return image
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def parse_region(region: str, width: int, height: int) -> tuple[int, int, int, int]:
|
|
44
|
+
try:
|
|
45
|
+
x1, y1, x2, y2 = (int(v) for v in region.split(","))
|
|
46
|
+
except ValueError:
|
|
47
|
+
raise ValueError("--region expects four integers: X1,Y1,X2,Y2 (pixels)") from None
|
|
48
|
+
box = (max(0, min(x1, x2)), max(0, min(y1, y2)),
|
|
49
|
+
min(width, max(x1, x2)), min(height, max(y1, y2)))
|
|
50
|
+
if box[2] <= box[0] or box[3] <= box[1]:
|
|
51
|
+
raise ValueError(f"--region {region} is empty after clamping to {width}x{height}")
|
|
52
|
+
return box
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def hex_of(rgb: tuple[int, int, int]) -> str:
|
|
56
|
+
return "#{:02X}{:02X}{:02X}".format(*rgb)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def parse_hex(text: str) -> tuple[int, int, int]:
|
|
60
|
+
value = text.strip().lstrip("#")
|
|
61
|
+
if len(value) != 6:
|
|
62
|
+
raise ValueError(f"invalid colour {text!r}: expected #RRGGBB")
|
|
63
|
+
return (int(value[0:2], 16), int(value[2:4], 16), int(value[4:6], 16))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def chebyshev(a: tuple[int, int, int], b: tuple[int, int, int]) -> int:
|
|
67
|
+
return max(abs(x - y) for x, y in zip(a, b))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def bar(share: float, max_share: float, width: int = 20) -> str:
|
|
71
|
+
fill = round(share / max_share * width) if max_share else 0
|
|
72
|
+
return "#" * fill
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _Cluster:
|
|
76
|
+
__slots__ = ("rgb", "count")
|
|
77
|
+
|
|
78
|
+
def __init__(self, rgb: tuple[int, int, int], count: int) -> None:
|
|
79
|
+
self.rgb = rgb
|
|
80
|
+
self.count = count
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def extract(image: "Image.Image", box: tuple[int, int, int, int], top: int,
|
|
84
|
+
quantize_k: int, max_pixels: int, merge_tol: int) -> list[_Cluster]:
|
|
85
|
+
"""Downsample, quantize, merge near-duplicates; return clusters sorted by count."""
|
|
86
|
+
crop = image.crop(box)
|
|
87
|
+
width, height = crop.size
|
|
88
|
+
scale = min(1.0, max_pixels / max(width, height))
|
|
89
|
+
if scale < 1.0:
|
|
90
|
+
crop = crop.resize((max(1, round(width * scale)), max(1, round(height * scale))), Image.LANCZOS)
|
|
91
|
+
quantized = crop.quantize(colors=quantize_k, method=Image.MEDIANCUT)
|
|
92
|
+
palette = quantized.getpalette()
|
|
93
|
+
clusters: list[_Cluster] = []
|
|
94
|
+
for count, index in sorted(quantized.getcolors(maxcolors=quantize_k), reverse=True):
|
|
95
|
+
rgb = (palette[index * 3], palette[index * 3 + 1], palette[index * 3 + 2])
|
|
96
|
+
for existing in clusters:
|
|
97
|
+
if chebyshev(rgb, existing.rgb) <= merge_tol:
|
|
98
|
+
total = existing.count + count
|
|
99
|
+
existing.rgb = tuple(
|
|
100
|
+
round((existing.rgb[i] * existing.count + rgb[i] * count) / total)
|
|
101
|
+
for i in range(3))
|
|
102
|
+
existing.count = total
|
|
103
|
+
break
|
|
104
|
+
else:
|
|
105
|
+
clusters.append(_Cluster(rgb, count))
|
|
106
|
+
clusters.sort(key=lambda c: c.count, reverse=True)
|
|
107
|
+
return clusters
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def pick(image: "Image.Image", box: tuple[int, int, int, int], candidates: list[str],
|
|
111
|
+
tol: int) -> tuple[list[dict], dict, dict]:
|
|
112
|
+
"""Score each candidate by how close the region's pixels are to it."""
|
|
113
|
+
crop = image.crop(box)
|
|
114
|
+
width, height = crop.size
|
|
115
|
+
total = width * height
|
|
116
|
+
bands = crop.split()
|
|
117
|
+
rows = []
|
|
118
|
+
for text in candidates:
|
|
119
|
+
rgb = parse_hex(text)
|
|
120
|
+
farthest = None
|
|
121
|
+
for band, value in zip(bands, rgb):
|
|
122
|
+
diff = ImageChops.difference(band, Image.new("L", (width, height), value))
|
|
123
|
+
farthest = diff if farthest is None else ImageChops.lighter(farthest, diff)
|
|
124
|
+
histogram = farthest.histogram()
|
|
125
|
+
mean_distance = sum(i * count for i, count in enumerate(histogram)) / total
|
|
126
|
+
hard = sum(count for i, count in enumerate(histogram) if i <= tol)
|
|
127
|
+
# Soft support: pixels closer to the candidate weigh more, so a match
|
|
128
|
+
# is not a binary in/out -- a candidate exactly on the colour wins over
|
|
129
|
+
# a near neighbour that is also inside the tolerance.
|
|
130
|
+
weighted = sum(max(0, tol - i) * count for i, count in enumerate(histogram))
|
|
131
|
+
rows.append({"text": text, "rgb": rgb, "mean_distance": mean_distance,
|
|
132
|
+
"hard": hard, "weighted": weighted,
|
|
133
|
+
"share": hard / total * 100})
|
|
134
|
+
winner = max(rows, key=lambda r: (r["weighted"], r["hard"]))
|
|
135
|
+
closest = min(rows, key=lambda r: r["mean_distance"])
|
|
136
|
+
return rows, winner, closest
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def format_extract(clusters: list[_Cluster], top: int, box: tuple[int, int, int, int],
|
|
140
|
+
merge_tol: int) -> list[str]:
|
|
141
|
+
total = sum(c.count for c in clusters)
|
|
142
|
+
if not total:
|
|
143
|
+
return ["(region has no pixels)"]
|
|
144
|
+
width = box[2] - box[0]
|
|
145
|
+
height = box[3] - box[1]
|
|
146
|
+
max_share = max(c.count / total for c in clusters) * 100
|
|
147
|
+
lines = [f"region {box[0]},{box[1]},{box[2]},{box[3]} - {width}x{height} px",
|
|
148
|
+
f"top {top} of {len(clusters)} clusters (merged at distance <= {merge_tol}):"]
|
|
149
|
+
for cluster in clusters[:top]:
|
|
150
|
+
share = cluster.count / total * 100
|
|
151
|
+
lines.append(f"{hex_of(cluster.rgb)} {share:5.1f}% {bar(share, max_share)}")
|
|
152
|
+
return lines
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def format_pick(rows: list[dict], winner: dict, closest: dict,
|
|
156
|
+
box: tuple[int, int, int, int], tol: int) -> list[str]:
|
|
157
|
+
width = box[2] - box[0]
|
|
158
|
+
height = box[3] - box[1]
|
|
159
|
+
total = width * height
|
|
160
|
+
max_share = max(r["share"] for r in rows) or 1.0
|
|
161
|
+
max_wt = max(r["weighted"] for r in rows)
|
|
162
|
+
lines = [f"region {box[0]},{box[1]},{box[2]},{box[3]} - {width}x{height} px ({total} px sampled)",
|
|
163
|
+
"candidate share mean_d wt bar"]
|
|
164
|
+
for row in rows:
|
|
165
|
+
mark = "*" if row is winner else " "
|
|
166
|
+
wt = (row["weighted"] / max_wt * 100) if max_wt else 0.0
|
|
167
|
+
lines.append(f"{mark}{row['text']:<9} {row['share']:5.1f}% {row['mean_distance']:4.1f} "
|
|
168
|
+
f"{wt:4.0f}% "
|
|
169
|
+
f"{bar(row['share'], max_share)}")
|
|
170
|
+
if winner["hard"] == 0:
|
|
171
|
+
lines.append(f"note: no candidate is within distance <= {tol} of the region; "
|
|
172
|
+
f"closest by mean distance is {closest['text']}")
|
|
173
|
+
else:
|
|
174
|
+
lines.append(f"winner: {winner['text']} (* in table) - wt is soft-match closeness, "
|
|
175
|
+
f"so the winner need not have the highest share; "
|
|
176
|
+
f"{winner['share']:.1f}% of region pixels within distance <= {tol}")
|
|
177
|
+
return lines
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def main() -> None:
|
|
181
|
+
parser = argparse.ArgumentParser(
|
|
182
|
+
prog="dominant_colors",
|
|
183
|
+
description="Extract a region's significant colours, or pick the concrete value among a candidate palette",
|
|
184
|
+
)
|
|
185
|
+
parser.add_argument("image", type=Path, help="the image")
|
|
186
|
+
parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
|
|
187
|
+
help="restrict to this pixel box (e.g. from ground); default: whole image")
|
|
188
|
+
parser.add_argument("--candidates", metavar="LIST",
|
|
189
|
+
help="pick mode: comma-separated candidate palette, e.g. #F9FAFA,#F5F5F5,#F3F3F3,#EDEDED")
|
|
190
|
+
parser.add_argument("--top", type=int, default=5, help="how many significant clusters to show (default: 5)")
|
|
191
|
+
parser.add_argument("--quantize", type=int, default=16,
|
|
192
|
+
help="palette size before merging near-duplicates (default: 16)")
|
|
193
|
+
parser.add_argument("--max-pixels", type=int, default=96,
|
|
194
|
+
help="downsample the crop to at most this on the long side (default: 96)")
|
|
195
|
+
parser.add_argument("--merge-tol", type=int, default=8,
|
|
196
|
+
help="merge clusters whose centres are within this per-channel distance (default: 8)")
|
|
197
|
+
parser.add_argument("--tol", type=int, default=16,
|
|
198
|
+
help="pick mode: per-channel distance for 'matches this candidate' (default: 16)")
|
|
199
|
+
args = parser.parse_args()
|
|
200
|
+
|
|
201
|
+
if Image is None:
|
|
202
|
+
parser.exit(1, "dominant_colors: requires Pillow; install the optional dependency pillow first\n")
|
|
203
|
+
path = args.image.expanduser()
|
|
204
|
+
if not path.is_file():
|
|
205
|
+
parser.exit(1, f"dominant_colors: image not found: {path}\n")
|
|
206
|
+
try:
|
|
207
|
+
image = load_rgb(path)
|
|
208
|
+
width, height = image.size
|
|
209
|
+
box = parse_region(args.region, width, height) if args.region else (0, 0, width, height)
|
|
210
|
+
if args.candidates:
|
|
211
|
+
candidates = [candidate.strip() for candidate in args.candidates.split(",") if candidate.strip()]
|
|
212
|
+
if not candidates:
|
|
213
|
+
parser.exit(1, "dominant_colors: --candidates needs at least one #RRGGBB\n")
|
|
214
|
+
rows, winner, closest = pick(image, box, candidates, args.tol)
|
|
215
|
+
print("\n".join(format_pick(rows, winner, closest, box, args.tol)))
|
|
216
|
+
else:
|
|
217
|
+
clusters = extract(image, box, args.top, args.quantize, args.max_pixels, args.merge_tol)
|
|
218
|
+
print("\n".join(format_extract(clusters, args.top, box, args.merge_tol)))
|
|
219
|
+
except ValueError as exc:
|
|
220
|
+
parser.exit(1, f"dominant_colors: {exc}\n")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
if __name__ == "__main__":
|
|
224
|
+
main()
|