oda-engine 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -2
- package/bin/nowhere.mjs +352 -0
- package/lib/account.mjs +57 -0
- package/lib/agentModules.mjs +57 -0
- package/lib/devServer.mjs +351 -0
- package/lib/headless.mjs +231 -0
- package/lib/headlessPreview.mjs +212 -0
- package/lib/keys.mjs +517 -0
- package/lib/make.mjs +855 -0
- package/lib/mcp.mjs +537 -0
- package/lib/open.mjs +22 -0
- package/lib/preview.mjs +87 -0
- package/lib/previewRpc.mjs +134 -0
- package/lib/project.mjs +359 -0
- package/lib/publishFlow.mjs +90 -0
- package/lib/resources.mjs +50 -0
- package/lib/rules.mjs +60 -0
- package/lib/scaffold.mjs +42 -0
- package/lib/setup.mjs +59 -0
- package/lib/tools.mjs +668 -0
- package/package.json +51 -4
- package/template/game.config.json +9 -0
- package/template/scene.json +4 -0
- package/template/src/main.ts +11 -0
- package/vendor/DOCS.md +447 -0
- package/vendor/agent/appRun.mjs +520 -0
- package/vendor/agent/assets/brief.mjs +242 -0
- package/vendor/agent/assets/collider.mjs +531 -0
- package/vendor/agent/assets/environments/index.mjs +12 -0
- package/vendor/agent/assets/environments/worldlabs.mjs +161 -0
- package/vendor/agent/assets/fetch.mjs +168 -0
- package/vendor/agent/assets/footprint.mjs +13 -0
- package/vendor/agent/assets/gltf.mjs +547 -0
- package/vendor/agent/assets/hdr.mjs +269 -0
- package/vendor/agent/assets/imageOps.mjs +171 -0
- package/vendor/agent/assets/images/codex.mjs +307 -0
- package/vendor/agent/assets/images/fal.mjs +75 -0
- package/vendor/agent/assets/images/gemini.mjs +84 -0
- package/vendor/agent/assets/images/index.mjs +25 -0
- package/vendor/agent/assets/images/openai.mjs +75 -0
- package/vendor/agent/assets/images/stability.mjs +71 -0
- package/vendor/agent/assets/images/views.mjs +31 -0
- package/vendor/agent/assets/import.mjs +1352 -0
- package/vendor/agent/assets/inspect.mjs +465 -0
- package/vendor/agent/assets/jobs.mjs +196 -0
- package/vendor/agent/assets/ktx2.mjs +290 -0
- package/vendor/agent/assets/ledger.mjs +466 -0
- package/vendor/agent/assets/net.mjs +254 -0
- package/vendor/agent/assets/prices.json +39 -0
- package/vendor/agent/assets/providers/fal.mjs +115 -0
- package/vendor/agent/assets/providers/index.mjs +22 -0
- package/vendor/agent/assets/providers/meshy.mjs +119 -0
- package/vendor/agent/assets/providers/tripo.mjs +188 -0
- package/vendor/agent/assets/sources/ambientcg.mjs +129 -0
- package/vendor/agent/assets/sources/index.mjs +23 -0
- package/vendor/agent/assets/sources/polyhaven.mjs +237 -0
- package/vendor/agent/assets/sources/polypizza.mjs +121 -0
- package/vendor/agent/assets/splat.mjs +631 -0
- package/vendor/agent/assets/targets.mjs +122 -0
- package/vendor/agent/assets/tile.mjs +261 -0
- package/vendor/agent/assets/unzip.mjs +180 -0
- package/vendor/agent/blender/cleanup.py +360 -0
- package/vendor/agent/blender/convert.py +167 -0
- package/vendor/agent/blender/find.mjs +192 -0
- package/vendor/agent/blender/nw_export.py +104 -0
- package/vendor/agent/blender/nw_rubric.py +426 -0
- package/vendor/agent/blender/run.mjs +193 -0
- package/vendor/agent/blender/runner.py +252 -0
- package/vendor/agent/blender/sheet.py +743 -0
- package/vendor/agent/depth.mjs +134 -0
- package/vendor/agent/director/board.mjs +451 -0
- package/vendor/agent/director/contracts.mjs +663 -0
- package/vendor/agent/director/director.mjs +2365 -0
- package/vendor/agent/director/estimate.mjs +200 -0
- package/vendor/agent/director/gate.mjs +802 -0
- package/vendor/agent/director/limits.json +16 -0
- package/vendor/agent/director/methods.mjs +457 -0
- package/vendor/agent/director/policy.json +33 -0
- package/vendor/agent/director/prompts.mjs +244 -0
- package/vendor/agent/director/reading.mjs +341 -0
- package/vendor/agent/director/router.mjs +184 -0
- package/vendor/agent/director/seams.mjs +243 -0
- package/vendor/agent/director/session.mjs +643 -0
- package/vendor/agent/director/store.mjs +119 -0
- package/vendor/agent/director/taskGate.mjs +110 -0
- package/vendor/agent/director/teamLedger.mjs +118 -0
- package/vendor/agent/director/triage.mjs +63 -0
- package/vendor/agent/downloads.cjs +363 -0
- package/vendor/agent/flow.mjs +78 -0
- package/vendor/agent/guard.mjs +613 -0
- package/vendor/agent/host.mjs +1736 -0
- package/vendor/agent/language.mjs +58 -0
- package/vendor/agent/ledger.mjs +360 -0
- package/vendor/agent/locale.cjs +186 -0
- package/vendor/agent/loop/capacity.mjs +74 -0
- package/vendor/agent/loop/compose.mjs +127 -0
- package/vendor/agent/loop/gate.mjs +205 -0
- package/vendor/agent/loop/judge.mjs +189 -0
- package/vendor/agent/loop/levels.mjs +273 -0
- package/vendor/agent/loop/state.mjs +358 -0
- package/vendor/agent/notes.cjs +278 -0
- package/vendor/agent/pace.mjs +258 -0
- package/vendor/agent/panels.mjs +431 -0
- package/vendor/agent/prompt.mjs +342 -0
- package/vendor/agent/reference/board.mjs +410 -0
- package/vendor/agent/reference/compose.mjs +364 -0
- package/vendor/agent/reference/decode.mjs +56 -0
- package/vendor/agent/reference/facts.mjs +1874 -0
- package/vendor/agent/reference/png.mjs +107 -0
- package/vendor/agent/reference/records.mjs +63 -0
- package/vendor/agent/reference/web.mjs +252 -0
- package/vendor/agent/roles/artist.mjs +69 -0
- package/vendor/agent/roles/builder.mjs +79 -0
- package/vendor/agent/roles/checks.mjs +223 -0
- package/vendor/agent/roles/common.mjs +188 -0
- package/vendor/agent/roles/critic.mjs +171 -0
- package/vendor/agent/roles/index.mjs +39 -0
- package/vendor/agent/roles/modeller.mjs +142 -0
- package/vendor/agent/roles/qa.mjs +58 -0
- package/vendor/agent/roles/rubrics.mjs +79 -0
- package/vendor/agent/roles/sound.mjs +69 -0
- package/vendor/agent/rounds.cjs +333 -0
- package/vendor/agent/shared.mjs +284 -0
- package/vendor/agent/stopGate.mjs +574 -0
- package/vendor/agent/subagents.mjs +129 -0
- package/vendor/agent/tools/assets.mjs +499 -0
- package/vendor/agent/tools/blender.mjs +754 -0
- package/vendor/agent/tools/cards.mjs +479 -0
- package/vendor/agent/tools/choices.mjs +479 -0
- package/vendor/agent/tools/compose.mjs +74 -0
- package/vendor/agent/tools/environment.mjs +488 -0
- package/vendor/agent/tools/eyes.mjs +733 -0
- package/vendor/agent/tools/generate.mjs +338 -0
- package/vendor/agent/tools/guide.mjs +46 -0
- package/vendor/agent/tools/harness.mjs +193 -0
- package/vendor/agent/tools/image.mjs +643 -0
- package/vendor/agent/tools/index.mjs +162 -0
- package/vendor/agent/tools/look.mjs +578 -0
- package/vendor/agent/tools/loop.mjs +858 -0
- package/vendor/agent/tools/others.mjs +123 -0
- package/vendor/agent/tools/pictures.mjs +535 -0
- package/vendor/agent/tools/reference.mjs +366 -0
- package/vendor/agent/transcripts.mjs +118 -0
- package/vendor/agent-loop.mjs +602 -0
- package/vendor/agent-plugin/.claude-plugin/plugin.json +5 -0
- package/vendor/agent-plugin/skills/art-direction/SKILL.md +82 -0
- package/vendor/agent-plugin/skills/asset-legality/SKILL.md +101 -0
- package/vendor/agent-plugin/skills/asset-sources/SKILL.md +111 -0
- package/vendor/agent-plugin/skills/blender-modeling/SKILL.md +97 -0
- package/vendor/agent-plugin/skills/look-dev/SKILL.md +104 -0
- package/vendor/agent-plugin/skills/playtesting/SKILL.md +74 -0
- package/vendor/agent-plugin/skills/reference-loop/SKILL.md +102 -0
- package/vendor/agent-plugin/skills/sound-design/SKILL.md +60 -0
- package/vendor/agent-plugin/skills/style/SKILL.md +35 -0
- package/vendor/agent-plugin/skills/tsl-r185/SKILL.md +107 -0
- package/vendor/auth.cjs +570 -0
- package/vendor/build.cjs +953 -0
- package/vendor/catalog.json +455 -0
- package/vendor/connectors.cjs +688 -0
- package/vendor/eyes.mjs +733 -0
- package/vendor/gameExports.json +1390 -0
- package/vendor/git.cjs +222 -0
- package/vendor/models.json +57 -0
- package/vendor/publish.cjs +1018 -0
- package/vendor/refBoard.cjs +301 -0
- package/vendor/schema/dist/assets.d.ts +335 -0
- package/vendor/schema/dist/assets.js +439 -0
- package/vendor/schema/dist/brief.d.ts +357 -0
- package/vendor/schema/dist/brief.js +302 -0
- package/vendor/schema/dist/common.d.ts +9 -0
- package/vendor/schema/dist/common.js +13 -0
- package/vendor/schema/dist/config.d.ts +406 -0
- package/vendor/schema/dist/config.js +355 -0
- package/vendor/schema/dist/contract.d.ts +599 -0
- package/vendor/schema/dist/contract.js +1146 -0
- package/vendor/schema/dist/index.d.ts +14 -0
- package/vendor/schema/dist/index.js +21 -0
- package/vendor/schema/dist/look.d.ts +463 -0
- package/vendor/schema/dist/look.js +681 -0
- package/vendor/schema/dist/lookStats.d.ts +230 -0
- package/vendor/schema/dist/lookStats.js +768 -0
- package/vendor/schema/dist/pilot.d.ts +32 -0
- package/vendor/schema/dist/pilot.js +44 -0
- package/vendor/schema/dist/publish.d.ts +214 -0
- package/vendor/schema/dist/publish.js +291 -0
- package/vendor/schema/dist/scene.d.ts +75 -0
- package/vendor/schema/dist/scene.js +120 -0
- package/vendor/schema/dist/skyLight.d.ts +141 -0
- package/vendor/schema/dist/skyLight.js +436 -0
- package/vendor/schema/dist/style.d.ts +131 -0
- package/vendor/schema/dist/style.js +155 -0
- package/vendor/schema/package.json +1 -0
- package/vendor/selectionPrompt.mjs +51 -0
- package/vendor/useTools.mjs +195 -0
- package/vendor/userMessage.mjs +26 -0
- package/web-dist/assets/GaussianSplatUtils-CB77cP9U.js +1 -0
- package/web-dist/assets/KTX2Loader-6OIWUF7s.js +2 -0
- package/web-dist/assets/SPZLoader-zYj0Alfv.js +1 -0
- package/web-dist/assets/gaussianSplat-CJTI1sEh.js +1 -0
- package/web-dist/assets/index-CcjyyQco.css +1 -0
- package/web-dist/assets/index-DMyJcoZJ.js +693 -0
- package/web-dist/assets/meshopt_decoder.module-BvwdjiD3.js +1 -0
- package/web-dist/assets/zstddec.module-CfFeQW7d.js +1 -0
- package/web-dist/index.html +34 -0
|
@@ -0,0 +1,1874 @@
|
|
|
1
|
+
// What a reference picture is, in numbers: its lettering, palette (with where
|
|
2
|
+
// each colour sits), value key, contrast, saturation, warmth, haze, the big
|
|
3
|
+
// masses and where the eye goes, the horizon, a guess at the camera, its
|
|
4
|
+
// true shape and whether it carries a frame. The director's interpretation
|
|
5
|
+
// cites these numbers for every palette, value, light and haze value of the
|
|
6
|
+
// brief and boxes each subject of the picture against the masses and the
|
|
7
|
+
// focal points, so a look and a form are derived from the picture rather
|
|
8
|
+
// than from a stock mood or a guess; and the same numbers for a shot of the
|
|
9
|
+
// game, side by side with the reference's, are the pixel half of the gate
|
|
10
|
+
// (compareMetrics).
|
|
11
|
+
//
|
|
12
|
+
// Lettering (a title, credits) is found as rows of glyphs and masked out,
|
|
13
|
+
// painted over with the colours around each box, before anything else is
|
|
14
|
+
// measured: its strokes are neither the art's edges, nor its palette, nor a
|
|
15
|
+
// horizon; and the measures of structure leave the boxes out altogether, so
|
|
16
|
+
// the smooth fill is no empty space and its rim no step either. A uniform
|
|
17
|
+
// border (a letterbox, a mat) is cropped, never the art.
|
|
18
|
+
// A reading that may be wrong says how sure it is (confidence); one under
|
|
19
|
+
// CONFIDENT may inform an assumption of the brief, never become a request
|
|
20
|
+
// element, a camera's horizonY or a critic criterion (advisoryOf,
|
|
21
|
+
// citesAdvisory).
|
|
22
|
+
//
|
|
23
|
+
// Pure arithmetic on RGBA bytes, with no three, no DOM and no clock: k-means
|
|
24
|
+
// starts from a seeded k-means++ and runs a fixed number of rounds, so the
|
|
25
|
+
// same picture always gives byte-identical facts. Colour is analysed at a
|
|
26
|
+
// long edge of 256 (COLOUR_EDGE), structure at 512 (STRUCTURE_EDGE) and
|
|
27
|
+
// lettering at up to 1024 (LETTER_EDGE).
|
|
28
|
+
//
|
|
29
|
+
// Every threshold below that is not a colour science constant is a proposal
|
|
30
|
+
// (orchestrator spec 6.1 and 6.2), to be tuned by the real-model experiments.
|
|
31
|
+
|
|
32
|
+
import { compareLook, deltaE2000, labToHex, lookStats, srgbToLab } from '../../schema/dist/index.js'
|
|
33
|
+
|
|
34
|
+
export const COLOUR_EDGE = 256
|
|
35
|
+
export const STRUCTURE_EDGE = 512
|
|
36
|
+
export const LETTER_EDGE = 1024
|
|
37
|
+
/** The one confidence rule (post-mortem gap 2): a reading below it may
|
|
38
|
+
* inform an assumption, never a request element, a camera's horizonY or a
|
|
39
|
+
* critic criterion. The gate holds the horizon to it as well. */
|
|
40
|
+
export const CONFIDENT = 0.5
|
|
41
|
+
|
|
42
|
+
const PALETTE_K = 8
|
|
43
|
+
const PALETTE_ROUNDS = 12
|
|
44
|
+
/** Clusters closer than this (deltaE2000) are one colour. */
|
|
45
|
+
const PALETTE_MERGE = 4
|
|
46
|
+
/** A cluster smaller than this share is folded into its nearest colour. */
|
|
47
|
+
const PALETTE_MIN_SHARE = 0.01
|
|
48
|
+
const SEED = 0x5eed1
|
|
49
|
+
|
|
50
|
+
const round = (v, digits = 4) => (Number.isFinite(v) ? Math.round(v * 10 ** digits) / 10 ** digits : 0)
|
|
51
|
+
const clamp = (v, lo, hi) => Math.min(hi, Math.max(lo, v))
|
|
52
|
+
const DEG = 180 / Math.PI
|
|
53
|
+
|
|
54
|
+
/** mulberry32: a small seeded generator, so the palette never depends on Math.random. */
|
|
55
|
+
function rng(seed) {
|
|
56
|
+
let a = seed >>> 0
|
|
57
|
+
return () => {
|
|
58
|
+
a = (a + 0x6d2b79f5) >>> 0
|
|
59
|
+
let t = a
|
|
60
|
+
t = Math.imul(t ^ (t >>> 15), t | 1)
|
|
61
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
|
|
62
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// ---- pictures ---------------------------------------------------------------
|
|
67
|
+
// A picture is { data, width, height }: RGBA bytes, top row first.
|
|
68
|
+
|
|
69
|
+
/** The picture at another size: box-averaged down, bilinear up. */
|
|
70
|
+
export function resample(pic, W, H) {
|
|
71
|
+
const { data, width, height } = pic
|
|
72
|
+
W = Math.max(1, Math.round(W))
|
|
73
|
+
H = Math.max(1, Math.round(H))
|
|
74
|
+
const out = new Uint8Array(W * H * 4)
|
|
75
|
+
if (W <= width && H <= height) {
|
|
76
|
+
for (let y = 0; y < H; y++) {
|
|
77
|
+
const y0 = Math.floor((y * height) / H)
|
|
78
|
+
const y1 = Math.max(y0 + 1, Math.floor(((y + 1) * height) / H))
|
|
79
|
+
for (let x = 0; x < W; x++) {
|
|
80
|
+
const x0 = Math.floor((x * width) / W)
|
|
81
|
+
const x1 = Math.max(x0 + 1, Math.floor(((x + 1) * width) / W))
|
|
82
|
+
let r = 0
|
|
83
|
+
let g = 0
|
|
84
|
+
let b = 0
|
|
85
|
+
let a = 0
|
|
86
|
+
for (let sy = y0; sy < y1; sy++) {
|
|
87
|
+
let o = (sy * width + x0) * 4
|
|
88
|
+
for (let sx = x0; sx < x1; sx++, o += 4) {
|
|
89
|
+
r += data[o]
|
|
90
|
+
g += data[o + 1]
|
|
91
|
+
b += data[o + 2]
|
|
92
|
+
a += data[o + 3]
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const n = (y1 - y0) * (x1 - x0)
|
|
96
|
+
const o = (y * W + x) * 4
|
|
97
|
+
out[o] = Math.round(r / n)
|
|
98
|
+
out[o + 1] = Math.round(g / n)
|
|
99
|
+
out[o + 2] = Math.round(b / n)
|
|
100
|
+
out[o + 3] = Math.round(a / n)
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return { data: out, width: W, height: H }
|
|
104
|
+
}
|
|
105
|
+
for (let y = 0; y < H; y++) {
|
|
106
|
+
const fy = Math.min(height - 1, Math.max(0, ((y + 0.5) * height) / H - 0.5))
|
|
107
|
+
const y0 = Math.floor(fy)
|
|
108
|
+
const y1 = Math.min(height - 1, y0 + 1)
|
|
109
|
+
const ty = fy - y0
|
|
110
|
+
for (let x = 0; x < W; x++) {
|
|
111
|
+
const fx = Math.min(width - 1, Math.max(0, ((x + 0.5) * width) / W - 0.5))
|
|
112
|
+
const x0 = Math.floor(fx)
|
|
113
|
+
const x1 = Math.min(width - 1, x0 + 1)
|
|
114
|
+
const tx = fx - x0
|
|
115
|
+
for (let c = 0; c < 4; c++) {
|
|
116
|
+
const p00 = data[(y0 * width + x0) * 4 + c]
|
|
117
|
+
const p01 = data[(y0 * width + x1) * 4 + c]
|
|
118
|
+
const p10 = data[(y1 * width + x0) * 4 + c]
|
|
119
|
+
const p11 = data[(y1 * width + x1) * 4 + c]
|
|
120
|
+
out[(y * W + x) * 4 + c] = Math.round((p00 * (1 - tx) + p01 * tx) * (1 - ty) + (p10 * (1 - tx) + p11 * tx) * ty)
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return { data: out, width: W, height: H }
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The picture with its long edge at most `edge` (never enlarged). */
|
|
128
|
+
export function fitEdge(pic, edge) {
|
|
129
|
+
const long = Math.max(pic.width, pic.height)
|
|
130
|
+
if (long <= edge) return pic
|
|
131
|
+
const k = edge / long
|
|
132
|
+
return resample(pic, pic.width * k, pic.height * k)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The part of a picture inside `crop` (fractions x0, y0, x1, y1). */
|
|
136
|
+
export function cropPicture(pic, crop) {
|
|
137
|
+
if (!crop) return pic
|
|
138
|
+
const [x0, y0, x1, y1] = crop.map((v) => clamp(Number(v) || 0, 0, 1))
|
|
139
|
+
const left = Math.floor(Math.min(x0, x1) * pic.width)
|
|
140
|
+
const top = Math.floor(Math.min(y0, y1) * pic.height)
|
|
141
|
+
const W = Math.max(1, Math.ceil(Math.max(x0, x1) * pic.width) - left)
|
|
142
|
+
const H = Math.max(1, Math.ceil(Math.max(y0, y1) * pic.height) - top)
|
|
143
|
+
const out = new Uint8Array(W * H * 4)
|
|
144
|
+
for (let y = 0; y < H; y++) {
|
|
145
|
+
const from = ((top + y) * pic.width + left) * 4
|
|
146
|
+
out.set(pic.data.subarray(from, from + W * 4), y * W * 4)
|
|
147
|
+
}
|
|
148
|
+
return { data: out, width: W, height: H }
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** The centre of a picture cut to `aspect` (width over height), so two
|
|
152
|
+
* pictures of different shapes are compared over the same frame. */
|
|
153
|
+
export function fitAspect(pic, aspect) {
|
|
154
|
+
const own = pic.width / pic.height
|
|
155
|
+
if (!(aspect > 0) || Math.abs(own - aspect) < 1e-3) return pic
|
|
156
|
+
if (own > aspect) {
|
|
157
|
+
const w = (aspect * pic.height) / pic.width
|
|
158
|
+
return cropPicture(pic, [(1 - w) / 2, 0, (1 + w) / 2, 1])
|
|
159
|
+
}
|
|
160
|
+
const h = pic.width / aspect / pic.height
|
|
161
|
+
return cropPicture(pic, [0, (1 - h) / 2, 1, (1 + h) / 2])
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** The nw_view aspect closest to a picture's width over height (tall is
|
|
165
|
+
* 9:16, square 1:1, wide 16:9; the cut points are their geometric means). */
|
|
166
|
+
export const shapeOf = (aspect) => (aspect < 0.75 ? 'tall' : aspect > 4 / 3 ? 'wide' : 'square')
|
|
167
|
+
|
|
168
|
+
// ---- lettering ---------------------------------------------------------------
|
|
169
|
+
|
|
170
|
+
/** Means of `v` over a (2r + 1) square around each pixel, from an integral image. */
|
|
171
|
+
function boxMean(v, w, h, r) {
|
|
172
|
+
const I = new Float64Array((w + 1) * (h + 1))
|
|
173
|
+
for (let y = 0; y < h; y++) {
|
|
174
|
+
let s = 0
|
|
175
|
+
for (let x = 0; x < w; x++) {
|
|
176
|
+
s += v[y * w + x]
|
|
177
|
+
I[(y + 1) * (w + 1) + x + 1] = I[y * (w + 1) + x + 1] + s
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const out = new Float64Array(w * h)
|
|
181
|
+
for (let y = 0; y < h; y++) {
|
|
182
|
+
const y0 = Math.max(0, y - r)
|
|
183
|
+
const y1 = Math.min(h, y + r + 1)
|
|
184
|
+
for (let x = 0; x < w; x++) {
|
|
185
|
+
const x0 = Math.max(0, x - r)
|
|
186
|
+
const x1 = Math.min(w, x + r + 1)
|
|
187
|
+
out[y * w + x] = (I[y1 * (w + 1) + x1] - I[y0 * (w + 1) + x1] - I[y1 * (w + 1) + x0] + I[y0 * (w + 1) + x0]) / ((y1 - y0) * (x1 - x0))
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return out
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const NEIGHBOURS = [
|
|
194
|
+
[1, 0],
|
|
195
|
+
[-1, 0],
|
|
196
|
+
[0, 1],
|
|
197
|
+
[0, -1],
|
|
198
|
+
]
|
|
199
|
+
/** With the corners too: a thin diagonal stroke of a small letter touches its neighbour only there. */
|
|
200
|
+
const NEIGHBOURS8 = [...NEIGHBOURS, [1, 1], [1, -1], [-1, 1], [-1, -1]]
|
|
201
|
+
|
|
202
|
+
/** The connected pixels from `start` that `joins(from, to)` lets in, as a
|
|
203
|
+
* glyph candidate (its box, size, outline and core lightness), or null
|
|
204
|
+
* past `most` pixels (a field, not a mark). `seen` is shared; `around`
|
|
205
|
+
* are the steps to a neighbour (its outline is counted on the four sides). */
|
|
206
|
+
function grow(start, w, h, L, seen, joins, most, around = NEIGHBOURS) {
|
|
207
|
+
const c = { n: 0, per: 0, x0: w, y0: h, x1: -1, y1: -1, members: [] }
|
|
208
|
+
const stack = [start]
|
|
209
|
+
seen[start] = 1
|
|
210
|
+
while (stack.length) {
|
|
211
|
+
const j = stack.pop()
|
|
212
|
+
const x = j % w
|
|
213
|
+
const y = (j - x) / w
|
|
214
|
+
c.members.push(j)
|
|
215
|
+
if (++c.n > most) {
|
|
216
|
+
while (stack.length) seen[stack.pop()] = 1
|
|
217
|
+
return null
|
|
218
|
+
}
|
|
219
|
+
if (x < c.x0) c.x0 = x
|
|
220
|
+
if (x > c.x1) c.x1 = x
|
|
221
|
+
if (y < c.y0) c.y0 = y
|
|
222
|
+
if (y > c.y1) c.y1 = y
|
|
223
|
+
let edge = false
|
|
224
|
+
for (const [dx, dy] of around) {
|
|
225
|
+
const nx = x + dx
|
|
226
|
+
const ny = y + dy
|
|
227
|
+
const side = dx === 0 || dy === 0
|
|
228
|
+
if (nx < 0 || ny < 0 || nx >= w || ny >= h) {
|
|
229
|
+
if (side) edge = true
|
|
230
|
+
continue
|
|
231
|
+
}
|
|
232
|
+
const k = ny * w + nx
|
|
233
|
+
if (!joins(j, k)) {
|
|
234
|
+
if (side) edge = true
|
|
235
|
+
continue
|
|
236
|
+
}
|
|
237
|
+
if (!seen[k]) {
|
|
238
|
+
seen[k] = 1
|
|
239
|
+
stack.push(k)
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
if (edge) c.per++
|
|
243
|
+
}
|
|
244
|
+
c.w = c.x1 - c.x0 + 1
|
|
245
|
+
c.h = c.y1 - c.y0 + 1
|
|
246
|
+
c.core = median(c.members.map((j) => L[j]))
|
|
247
|
+
return c
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const mad = (list) => {
|
|
251
|
+
const m = median(list)
|
|
252
|
+
return median(list.map((v) => Math.abs(v - m)))
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* The rows of glyphs at one scale. A glyph is a mark of one ink, shaped like
|
|
257
|
+
* strokes and 6 L* or more off what it stands on: either the pixels that
|
|
258
|
+
* stand out from the lightness around them by 18 L* or more (sharp small
|
|
259
|
+
* type), or a field of one flat colour (a large title over a light sky,
|
|
260
|
+
* where the contrast is low but the fill is flat). Glyphs of one ink, one
|
|
261
|
+
* height and one baseline, close in a row, make a line; a line is lettering
|
|
262
|
+
* when it has three marks and four letters or more, its baseline or cap
|
|
263
|
+
* line holds within 6% of its height, its heights within 20%, its ink
|
|
264
|
+
* within 4 L* (or 15% of its contrast), and its shapes differ in width or
|
|
265
|
+
* fill (letters, not a row of one repeated mark, as windows or a fence
|
|
266
|
+
* would be).
|
|
267
|
+
*/
|
|
268
|
+
function lettersAt(pic) {
|
|
269
|
+
const { width: w, height: h } = pic
|
|
270
|
+
const n = w * h
|
|
271
|
+
const L = new Float64Array(n)
|
|
272
|
+
const A = new Float64Array(n)
|
|
273
|
+
const B = new Float64Array(n)
|
|
274
|
+
for (let i = 0; i < n; i++) {
|
|
275
|
+
const c = srgbToLab(pic.data[i * 4], pic.data[i * 4 + 1], pic.data[i * 4 + 2])
|
|
276
|
+
L[i] = c[0]
|
|
277
|
+
A[i] = c[1]
|
|
278
|
+
B[i] = c[2]
|
|
279
|
+
}
|
|
280
|
+
const around = boxMean(L, w, h, Math.max(3, Math.round(0.02 * Math.max(w, h))))
|
|
281
|
+
const most = Math.max(64, Math.round(0.02 * n))
|
|
282
|
+
const marks = []
|
|
283
|
+
for (const sign of [1, -1]) {
|
|
284
|
+
const seen = new Uint8Array(n)
|
|
285
|
+
const ink = (i) => sign * (L[i] - around[i]) >= 18
|
|
286
|
+
for (let i = 0; i < n; i++) {
|
|
287
|
+
if (seen[i] || !ink(i)) continue
|
|
288
|
+
const c = grow(i, w, h, L, seen, (_, k) => ink(k), most, NEIGHBOURS8)
|
|
289
|
+
if (c) marks.push({ ...c, sign })
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
{
|
|
293
|
+
const seen = new Uint8Array(n)
|
|
294
|
+
const flat = (i, k) => Math.abs(L[i] - L[k]) + 0.5 * (Math.abs(A[i] - A[k]) + Math.abs(B[i] - B[k])) <= 3
|
|
295
|
+
for (let i = 0; i < n; i++) {
|
|
296
|
+
if (seen[i]) continue
|
|
297
|
+
const c = grow(i, w, h, L, seen, flat, most)
|
|
298
|
+
if (c && c.n >= 8) marks.push({ ...c, sign: 0 })
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
// What each mark stands on: the median of the ring two pixels outside its box.
|
|
302
|
+
for (const c of marks) {
|
|
303
|
+
const ring = []
|
|
304
|
+
for (let x = Math.max(0, c.x0 - 2); x <= Math.min(w - 1, c.x1 + 2); x++) for (const y of [c.y0 - 2, c.y1 + 2]) if (y >= 0 && y < h) ring.push(L[y * w + x])
|
|
305
|
+
for (let y = Math.max(0, c.y0 - 2); y <= Math.min(h - 1, c.y1 + 2); y++) for (const x of [c.x0 - 2, c.x1 + 2]) if (x >= 0 && x < w) ring.push(L[y * w + x])
|
|
306
|
+
c.contrast = c.core - median(ring)
|
|
307
|
+
if (c.sign === 0) c.sign = c.contrast > 0 ? 1 : -1
|
|
308
|
+
}
|
|
309
|
+
const glyphs = []
|
|
310
|
+
const owned = new Uint8Array(n)
|
|
311
|
+
for (const c of marks
|
|
312
|
+
.filter((c) => {
|
|
313
|
+
if (c.n < 8 || c.h < 6 || c.h > 0.2 * h || c.w > 4 * c.h || Math.abs(c.contrast) < 6) return false
|
|
314
|
+
// Strokes, not blobs: a solid mark only as a narrow bar (an I, a 1).
|
|
315
|
+
return (2 * c.n) / Math.max(1, c.per) <= 0.45 * c.h && (c.n / (c.w * c.h) <= 0.95 || c.w <= 0.4 * c.h)
|
|
316
|
+
})
|
|
317
|
+
.sort((a, b) => b.n - a.n || a.y0 - b.y0 || a.x0 - b.x0)) {
|
|
318
|
+
// The same mark found both ways (most of its pixels a kept mark's of its ink): the larger stays.
|
|
319
|
+
const code = c.sign > 0 ? 1 : 2
|
|
320
|
+
if (c.members.filter((j) => owned[j] === code).length > 0.5 * c.n) continue
|
|
321
|
+
for (const j of c.members) owned[j] = code
|
|
322
|
+
glyphs.push({ sign: c.sign, x0: c.x0, y0: c.y0, x1: c.x1, y1: c.y1, w: c.w, h: c.h, n: c.n, core: c.core, contrast: Math.abs(c.contrast) })
|
|
323
|
+
}
|
|
324
|
+
// Rows: glyphs of one ink side by side, sharing most of their height.
|
|
325
|
+
const parent = glyphs.map((_, i) => i)
|
|
326
|
+
const find = (i) => (parent[i] === i ? i : (parent[i] = find(parent[i])))
|
|
327
|
+
const byX = glyphs.map((_, i) => i).sort((a, b) => glyphs[a].x0 - glyphs[b].x0 || a - b)
|
|
328
|
+
for (let a = 0; a < byX.length; a++) {
|
|
329
|
+
const g = glyphs[byX[a]]
|
|
330
|
+
for (let b = a + 1; b < byX.length; b++) {
|
|
331
|
+
const o = glyphs[byX[b]]
|
|
332
|
+
if (o.x0 - g.x1 > 1.2 * Math.max(g.h, o.h)) break
|
|
333
|
+
if (o.sign !== g.sign || Math.max(g.h, o.h) > 1.6 * Math.min(g.h, o.h) || Math.abs(g.core - o.core) > 12) continue
|
|
334
|
+
if (Math.min(g.y1, o.y1) - Math.max(g.y0, o.y0) < 0.6 * Math.min(g.h, o.h)) continue
|
|
335
|
+
parent[find(byX[a])] = find(byX[b])
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
const rows = new Map()
|
|
339
|
+
glyphs.forEach((g, i) => {
|
|
340
|
+
const r = find(i)
|
|
341
|
+
if (!rows.has(r)) rows.set(r, [])
|
|
342
|
+
rows.get(r).push(g)
|
|
343
|
+
})
|
|
344
|
+
const lines = []
|
|
345
|
+
for (const gs of rows.values()) {
|
|
346
|
+
const height = median(gs.map((g) => g.h))
|
|
347
|
+
// A run of letters that touch is one wide mark: it counts as the letters in it.
|
|
348
|
+
const letters = gs.reduce((s, g) => s + Math.max(1, Math.round(g.w / (0.8 * height))), 0)
|
|
349
|
+
if (gs.length < 3 || letters < 4) continue
|
|
350
|
+
const line = mad(gs.map((g) => g.y1)) <= 0.06 * height || mad(gs.map((g) => g.y0)) <= 0.06 * height
|
|
351
|
+
const heights = mad(gs.map((g) => g.h)) <= 0.2 * height
|
|
352
|
+
// Small type's cores vary with its edges' blend, so the ink holds to a
|
|
353
|
+
// share of its contrast too.
|
|
354
|
+
const ink = mad(gs.map((g) => g.core)) <= Math.max(4, 0.15 * median(gs.map((g) => g.contrast)))
|
|
355
|
+
// Letters differ in their width or in how much of their box they fill
|
|
356
|
+
// (a monospaced face keeps the width); a repeated mark differs in neither.
|
|
357
|
+
const spread = (list) => {
|
|
358
|
+
const mean = list.reduce((s, v) => s + v, 0) / list.length
|
|
359
|
+
return Math.sqrt(list.reduce((s, v) => s + (v - mean) ** 2, 0) / list.length) / Math.max(1e-9, mean)
|
|
360
|
+
}
|
|
361
|
+
const varied = spread(gs.map((g) => g.w / g.h)) >= 0.1 || spread(gs.map((g) => g.n / (g.w * g.h))) >= 0.1
|
|
362
|
+
const box = [Math.min(...gs.map((g) => g.x0)), Math.min(...gs.map((g) => g.y0)), Math.max(...gs.map((g) => g.x1)) + 1, Math.max(...gs.map((g) => g.y1)) + 1]
|
|
363
|
+
if (!(line && heights && ink && varied && box[2] - box[0] >= 1.5 * height)) continue
|
|
364
|
+
lines.push({ box, height, letters, sign: gs[0].sign, core: median(gs.map((g) => g.core)) })
|
|
365
|
+
}
|
|
366
|
+
return lines
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/** Letters this tall (a fraction of the picture's height) or taller are a title's. */
|
|
370
|
+
const TITLE_HEIGHT = 0.03
|
|
371
|
+
/** A lone line needs this many letters: a few marks of one size in a row
|
|
372
|
+
* turn up in texture (foliage, rubble, a tiled roof), a sentence of them
|
|
373
|
+
* does not. */
|
|
374
|
+
const LONE_LETTERS = 8
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* The picture's lettering: each block of lines of glyphs (a title, a
|
|
378
|
+
* tagline, credits) as a box in fractions [x0, y0, x1, y1], padded by a
|
|
379
|
+
* quarter of its letters' height so the edges of the strokes are inside,
|
|
380
|
+
* with its letters' height (a fraction of the picture's), its ink (light or
|
|
381
|
+
* dark) and how many letters it holds. Read at up to LETTER_EDGE and again
|
|
382
|
+
* at a third of that, where a large title's letters are the size small type
|
|
383
|
+
* is. A block is lettering when it has two lines or more, or one line of
|
|
384
|
+
* LONE_LETTERS letters or more (a single short word is missed, rather than
|
|
385
|
+
* a patch of texture masked); every box found is masked, so the rule is the
|
|
386
|
+
* whole of how sure the reading is.
|
|
387
|
+
*/
|
|
388
|
+
export function letteringOf(pic) {
|
|
389
|
+
const base = fitEdge(pic, LETTER_EDGE)
|
|
390
|
+
const lines = []
|
|
391
|
+
for (const k of [1, 1 / 3]) {
|
|
392
|
+
const at = k === 1 ? base : resample(base, base.width * k, base.height * k)
|
|
393
|
+
for (const l of lettersAt(at)) lines.push({ ...l, box: [l.box[0] / at.width, l.box[1] / at.height, l.box[2] / at.width, l.box[3] / at.height], height: l.height / at.height })
|
|
394
|
+
}
|
|
395
|
+
// Lines one above the other in the same ink are one block; the same line
|
|
396
|
+
// read at two scales, and the counters of a line's letters (the other
|
|
397
|
+
// ink, inside it), are that line again, never another one.
|
|
398
|
+
const area = (b) => (b[2] - b[0]) * (b[3] - b[1])
|
|
399
|
+
const blocks = []
|
|
400
|
+
for (const l of lines.sort((a, b) => a.box[1] - b.box[1] || a.box[0] - b.box[0])) {
|
|
401
|
+
let again = false
|
|
402
|
+
const near = blocks.find((b) => {
|
|
403
|
+
const ix = Math.min(b.box[2], l.box[2]) - Math.max(b.box[0], l.box[0])
|
|
404
|
+
const iy = Math.min(b.box[3], l.box[3]) - Math.max(b.box[1], l.box[1])
|
|
405
|
+
if (ix > 0 && iy > 0 && ix * iy > 0.5 * Math.min(area(b.box), area(l.box))) return (again = true)
|
|
406
|
+
return ix > 0 && -iy <= 1.5 * Math.max(b.height, l.height) && b.sign === l.sign
|
|
407
|
+
})
|
|
408
|
+
if (!near) {
|
|
409
|
+
blocks.push({ ...l, lines: 1 })
|
|
410
|
+
continue
|
|
411
|
+
}
|
|
412
|
+
near.box = [Math.min(near.box[0], l.box[0]), Math.min(near.box[1], l.box[1]), Math.max(near.box[2], l.box[2]), Math.max(near.box[3], l.box[3])]
|
|
413
|
+
near.height = Math.max(near.height, l.height)
|
|
414
|
+
if (again) near.letters = Math.max(near.letters, l.letters)
|
|
415
|
+
else {
|
|
416
|
+
near.letters += l.letters
|
|
417
|
+
near.lines++
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
const aspect = base.width / base.height
|
|
421
|
+
return blocks
|
|
422
|
+
.filter((b) => b.lines > 1 || b.letters >= LONE_LETTERS)
|
|
423
|
+
.map((b) => {
|
|
424
|
+
const pad = 0.25 * b.height
|
|
425
|
+
return {
|
|
426
|
+
box: [clamp(b.box[0] - pad / aspect, 0, 1), clamp(b.box[1] - pad, 0, 1), clamp(b.box[2] + pad / aspect, 0, 1), clamp(b.box[3] + pad, 0, 1)].map((v) => round(v, 4)),
|
|
427
|
+
height: round(b.height, 4),
|
|
428
|
+
kind: b.height >= TITLE_HEIGHT ? 'title' : 'small',
|
|
429
|
+
ink: b.sign > 0 ? 'light' : 'dark',
|
|
430
|
+
letters: b.letters,
|
|
431
|
+
}
|
|
432
|
+
})
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* The picture with each box (fractions) painted over by the colours around
|
|
437
|
+
* it: every pixel inside takes the four sides of the box beyond its row and
|
|
438
|
+
* its column (each side smoothed along itself), each weighted by how near
|
|
439
|
+
* it is. What lies under lettering is not known, so a smooth fill from its
|
|
440
|
+
* surround is the least the measures can be misled by: no strokes, no
|
|
441
|
+
* edges, the colours the box stands in.
|
|
442
|
+
*/
|
|
443
|
+
export function maskBoxes(pic, boxes) {
|
|
444
|
+
if (!boxes?.length) return pic
|
|
445
|
+
const { width: w, height: h } = pic
|
|
446
|
+
const data = Uint8Array.from(pic.data)
|
|
447
|
+
for (const box of boxes) {
|
|
448
|
+
const x0 = clamp(Math.floor(box[0] * w), 0, w)
|
|
449
|
+
const y0 = clamp(Math.floor(box[1] * h), 0, h)
|
|
450
|
+
const x1 = clamp(Math.ceil(box[2] * w), 0, w)
|
|
451
|
+
const y1 = clamp(Math.ceil(box[3] * h), 0, h)
|
|
452
|
+
if (x1 <= x0 || y1 <= y0) continue
|
|
453
|
+
// A side's sample: the mean of the three pixels beyond it, where the picture has them.
|
|
454
|
+
const sample = (xs, ys) => {
|
|
455
|
+
const out = [0, 0, 0]
|
|
456
|
+
let k = 0
|
|
457
|
+
for (const y of ys) {
|
|
458
|
+
for (const x of xs) {
|
|
459
|
+
if (x < 0 || y < 0 || x >= w || y >= h) continue
|
|
460
|
+
const o = (y * w + x) * 4
|
|
461
|
+
out[0] += data[o]
|
|
462
|
+
out[1] += data[o + 1]
|
|
463
|
+
out[2] += data[o + 2]
|
|
464
|
+
k++
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
return k ? out.map((v) => v / k) : null
|
|
468
|
+
}
|
|
469
|
+
let left = []
|
|
470
|
+
let right = []
|
|
471
|
+
for (let y = y0; y < y1; y++) {
|
|
472
|
+
left.push(sample([x0 - 3, x0 - 2, x0 - 1], [y]))
|
|
473
|
+
right.push(sample([x1, x1 + 1, x1 + 2], [y]))
|
|
474
|
+
}
|
|
475
|
+
let top = []
|
|
476
|
+
let bottom = []
|
|
477
|
+
for (let x = x0; x < x1; x++) {
|
|
478
|
+
top.push(sample([x], [y0 - 3, y0 - 2, y0 - 1]))
|
|
479
|
+
bottom.push(sample([x], [y1, y1 + 1, y1 + 2]))
|
|
480
|
+
}
|
|
481
|
+
// Each side smoothed along itself over the box's short side, so the
|
|
482
|
+
// detail of the surround does not streak across the fill.
|
|
483
|
+
const soften = (list) => {
|
|
484
|
+
const r = Math.max(1, Math.round(Math.min(x1 - x0, y1 - y0) / 2))
|
|
485
|
+
return list.map((_, i) => {
|
|
486
|
+
const near = list.slice(Math.max(0, i - r), i + r + 1).filter(Boolean)
|
|
487
|
+
return near.length ? [0, 1, 2].map((k) => near.reduce((s, c) => s + c[k], 0) / near.length) : null
|
|
488
|
+
})
|
|
489
|
+
}
|
|
490
|
+
;[left, right, top, bottom] = [left, right, top, bottom].map(soften)
|
|
491
|
+
for (let y = y0; y < y1; y++) {
|
|
492
|
+
for (let x = x0; x < x1; x++) {
|
|
493
|
+
const sides = [
|
|
494
|
+
[left[y - y0], x - x0 + 1],
|
|
495
|
+
[right[y - y0], x1 - x],
|
|
496
|
+
[top[x - x0], y - y0 + 1],
|
|
497
|
+
[bottom[x - x0], y1 - y],
|
|
498
|
+
].filter(([s]) => s)
|
|
499
|
+
if (!sides.length) continue
|
|
500
|
+
let sw = 0
|
|
501
|
+
const c = [0, 0, 0]
|
|
502
|
+
for (const [s, d] of sides) {
|
|
503
|
+
const wt = 1 / d
|
|
504
|
+
sw += wt
|
|
505
|
+
for (let k = 0; k < 3; k++) c[k] += s[k] * wt
|
|
506
|
+
}
|
|
507
|
+
const o = (y * w + x) * 4
|
|
508
|
+
for (let k = 0; k < 3; k++) data[o + k] = Math.round(c[k] / sw)
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
return { data, width: w, height: h }
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
// ---- colour -----------------------------------------------------------------
|
|
516
|
+
|
|
517
|
+
const LINEAR = new Float64Array(256)
|
|
518
|
+
for (let i = 0; i < 256; i++) {
|
|
519
|
+
const c = i / 255
|
|
520
|
+
LINEAR[i] = c <= 0.04045 ? c / 12.92 : Math.pow((c + 0.055) / 1.055, 2.4)
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/** sRGB bytes to OKLab (Ottosson 2020): the space the palette clusters in,
|
|
524
|
+
* where a distance reads about the same for every hue. */
|
|
525
|
+
export function oklabOf(r, g, b) {
|
|
526
|
+
const R = LINEAR[r & 255]
|
|
527
|
+
const G = LINEAR[g & 255]
|
|
528
|
+
const B = LINEAR[b & 255]
|
|
529
|
+
const l = Math.cbrt(0.4122214708 * R + 0.5363325363 * G + 0.0514459929 * B)
|
|
530
|
+
const m = Math.cbrt(0.2119034982 * R + 0.6806995451 * G + 0.1073969566 * B)
|
|
531
|
+
const s = Math.cbrt(0.0883024619 * R + 0.2817188376 * G + 0.6299787005 * B)
|
|
532
|
+
return [
|
|
533
|
+
0.2104542553 * l + 0.793617785 * m - 0.0040720468 * s,
|
|
534
|
+
1.9779984951 * l - 2.428592205 * m + 0.4505937099 * s,
|
|
535
|
+
0.0259040371 * l + 0.7827717662 * m - 0.808675766 * s,
|
|
536
|
+
]
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Every pixel's CIE Lab, chroma and OKLab, in flat arrays. */
|
|
540
|
+
function analyse(pic) {
|
|
541
|
+
const n = pic.width * pic.height
|
|
542
|
+
const lab = new Float64Array(n * 3)
|
|
543
|
+
const ok = new Float64Array(n * 3)
|
|
544
|
+
const chroma = new Float64Array(n)
|
|
545
|
+
const d = pic.data
|
|
546
|
+
for (let i = 0; i < n; i++) {
|
|
547
|
+
const r = d[i * 4]
|
|
548
|
+
const g = d[i * 4 + 1]
|
|
549
|
+
const b = d[i * 4 + 2]
|
|
550
|
+
const c = srgbToLab(r, g, b)
|
|
551
|
+
lab[i * 3] = c[0]
|
|
552
|
+
lab[i * 3 + 1] = c[1]
|
|
553
|
+
lab[i * 3 + 2] = c[2]
|
|
554
|
+
chroma[i] = Math.hypot(c[1], c[2])
|
|
555
|
+
const o = oklabOf(r, g, b)
|
|
556
|
+
ok[i * 3] = o[0]
|
|
557
|
+
ok[i * 3 + 1] = o[1]
|
|
558
|
+
ok[i * 3 + 2] = o[2]
|
|
559
|
+
}
|
|
560
|
+
return { n, lab, ok, chroma, width: pic.width, height: pic.height }
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/** k-means++ in OKLab over the pixels `index` names (all when null), from a
|
|
564
|
+
* seeded start and a fixed number of rounds. Clusters as member lists.
|
|
565
|
+
* `farthest` seeds each next centre at the pixel farthest from the ones
|
|
566
|
+
* chosen instead of by chance: a few pixels of a lamp among thousands of
|
|
567
|
+
* sky pixels are then always a centre of their own. */
|
|
568
|
+
function kmeans(px, index, k, rounds, seed, farthest = false) {
|
|
569
|
+
const ids = index ?? Array.from({ length: px.n }, (_, i) => i)
|
|
570
|
+
const m = ids.length
|
|
571
|
+
if (m === 0) return []
|
|
572
|
+
const rand = rng(seed)
|
|
573
|
+
const ok = px.ok
|
|
574
|
+
const dist = (i, c) => {
|
|
575
|
+
const dl = ok[i * 3] - c[0]
|
|
576
|
+
const da = ok[i * 3 + 1] - c[1]
|
|
577
|
+
const db = ok[i * 3 + 2] - c[2]
|
|
578
|
+
return dl * dl + da * da + db * db
|
|
579
|
+
}
|
|
580
|
+
const at = (i) => [ok[i * 3], ok[i * 3 + 1], ok[i * 3 + 2]]
|
|
581
|
+
const centres = [at(ids[Math.floor(rand() * m)])]
|
|
582
|
+
const near = new Float64Array(m)
|
|
583
|
+
for (let j = 0; j < m; j++) near[j] = dist(ids[j], centres[0])
|
|
584
|
+
while (centres.length < k) {
|
|
585
|
+
let total = 0
|
|
586
|
+
for (let j = 0; j < m; j++) total += near[j]
|
|
587
|
+
// Every pixel sits on a centre already: the picture has fewer colours than k.
|
|
588
|
+
if (total <= 1e-12) break
|
|
589
|
+
let pick = m - 1
|
|
590
|
+
if (farthest) {
|
|
591
|
+
for (let j = 0; j < m; j++) if (near[j] > near[pick]) pick = j
|
|
592
|
+
} else {
|
|
593
|
+
let r = rand() * total
|
|
594
|
+
for (let j = 0; j < m; j++) {
|
|
595
|
+
r -= near[j]
|
|
596
|
+
if (r <= 0) {
|
|
597
|
+
pick = j
|
|
598
|
+
break
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
const c = at(ids[pick])
|
|
603
|
+
centres.push(c)
|
|
604
|
+
for (let j = 0; j < m; j++) near[j] = Math.min(near[j], dist(ids[j], c))
|
|
605
|
+
}
|
|
606
|
+
let cs = centres
|
|
607
|
+
const assign = new Int32Array(m)
|
|
608
|
+
for (let round = 0; round <= rounds; round++) {
|
|
609
|
+
for (let j = 0; j < m; j++) {
|
|
610
|
+
let best = 0
|
|
611
|
+
let bestD = Infinity
|
|
612
|
+
for (let c = 0; c < cs.length; c++) {
|
|
613
|
+
const d = dist(ids[j], cs[c])
|
|
614
|
+
if (d < bestD) {
|
|
615
|
+
bestD = d
|
|
616
|
+
best = c
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
assign[j] = best
|
|
620
|
+
}
|
|
621
|
+
if (round === rounds) break
|
|
622
|
+
const sums = cs.map(() => [0, 0, 0, 0])
|
|
623
|
+
for (let j = 0; j < m; j++) {
|
|
624
|
+
const s = sums[assign[j]]
|
|
625
|
+
const i = ids[j]
|
|
626
|
+
s[0] += ok[i * 3]
|
|
627
|
+
s[1] += ok[i * 3 + 1]
|
|
628
|
+
s[2] += ok[i * 3 + 2]
|
|
629
|
+
s[3]++
|
|
630
|
+
}
|
|
631
|
+
// An empty cluster is dropped, not reseeded: reseeding would need chance.
|
|
632
|
+
cs = sums.filter((s) => s[3] > 0).map((s) => [s[0] / s[3], s[1] / s[3], s[2] / s[3]])
|
|
633
|
+
}
|
|
634
|
+
const members = cs.map(() => [])
|
|
635
|
+
for (let j = 0; j < m; j++) members[assign[j]].push(ids[j])
|
|
636
|
+
return members.filter((list) => list.length > 0)
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/** A cluster's colour: the mean of its pixels' Lab and OKLab, and its size. */
|
|
640
|
+
function summarise(px, list) {
|
|
641
|
+
const lab = [0, 0, 0]
|
|
642
|
+
const ok = [0, 0, 0]
|
|
643
|
+
let chroma = 0
|
|
644
|
+
for (const i of list) {
|
|
645
|
+
for (let c = 0; c < 3; c++) {
|
|
646
|
+
lab[c] += px.lab[i * 3 + c]
|
|
647
|
+
ok[c] += px.ok[i * 3 + c]
|
|
648
|
+
}
|
|
649
|
+
chroma += px.chroma[i]
|
|
650
|
+
}
|
|
651
|
+
const n = list.length
|
|
652
|
+
return { lab: lab.map((v) => v / n), ok: ok.map((v) => v / n), chroma: chroma / n, count: n }
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/** Clusters merged while two are closer than `limit` (deltaE2000), the closest pair first. */
|
|
656
|
+
function mergeNear(groups, limit) {
|
|
657
|
+
const out = groups.map((g) => ({ ...g }))
|
|
658
|
+
for (;;) {
|
|
659
|
+
let best = null
|
|
660
|
+
for (let a = 0; a < out.length; a++) {
|
|
661
|
+
for (let b = a + 1; b < out.length; b++) {
|
|
662
|
+
const d = deltaE2000(out[a].lab, out[b].lab)
|
|
663
|
+
if (d < limit && (!best || d < best.d)) best = { a, b, d }
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
if (!best) return out
|
|
667
|
+
const A = out[best.a]
|
|
668
|
+
const B = out.splice(best.b, 1)[0]
|
|
669
|
+
const n = A.count + B.count
|
|
670
|
+
for (let c = 0; c < 3; c++) {
|
|
671
|
+
A.lab[c] = (A.lab[c] * A.count + B.lab[c] * B.count) / n
|
|
672
|
+
A.ok[c] = (A.ok[c] * A.count + B.ok[c] * B.count) / n
|
|
673
|
+
}
|
|
674
|
+
A.chroma = (A.chroma * A.count + B.chroma * B.count) / n
|
|
675
|
+
A.count = n
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
/** The palette of the pixels `index` names (all when null): its colours by share. */
|
|
680
|
+
function paletteOf(px, index = null) {
|
|
681
|
+
const m = index?.length ?? px.n
|
|
682
|
+
let groups = mergeNear(kmeans(px, index, PALETTE_K, PALETTE_ROUNDS, SEED).map((list) => summarise(px, list)), PALETTE_MERGE)
|
|
683
|
+
// A sliver (an edge's blend of two colours) is part of its nearest colour.
|
|
684
|
+
for (;;) {
|
|
685
|
+
groups.sort((a, b) => a.count - b.count)
|
|
686
|
+
const small = groups[0]
|
|
687
|
+
if (groups.length < 2 || small.count / m >= PALETTE_MIN_SHARE) break
|
|
688
|
+
groups = groups.slice(1)
|
|
689
|
+
let nearest = groups[0]
|
|
690
|
+
for (const g of groups) if (deltaE2000(g.lab, small.lab) < deltaE2000(nearest.lab, small.lab)) nearest = g
|
|
691
|
+
const n = nearest.count + small.count
|
|
692
|
+
for (let c = 0; c < 3; c++) {
|
|
693
|
+
nearest.lab[c] = (nearest.lab[c] * nearest.count + small.lab[c] * small.count) / n
|
|
694
|
+
nearest.ok[c] = (nearest.ok[c] * nearest.count + small.ok[c] * small.count) / n
|
|
695
|
+
}
|
|
696
|
+
nearest.chroma = (nearest.chroma * nearest.count + small.chroma * small.count) / n
|
|
697
|
+
nearest.count = n
|
|
698
|
+
}
|
|
699
|
+
groups.sort((a, b) => b.count - a.count || b.lab[0] - a.lab[0])
|
|
700
|
+
const palette = groups.map((g) => ({
|
|
701
|
+
hex: labToHex(g.lab),
|
|
702
|
+
share: round(g.count / m),
|
|
703
|
+
lab: g.lab.map((v) => round(v, 2)),
|
|
704
|
+
oklab: g.ok.map((v) => round(v, 4)),
|
|
705
|
+
}))
|
|
706
|
+
// The shares are rounded; the largest takes the rest, so they sum to 1.
|
|
707
|
+
if (palette.length) palette[0].share = round(1 - palette.slice(1).reduce((s, p) => s + p.share, 0))
|
|
708
|
+
return palette
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/** Where each palette colour sits: every measured pixel goes to its nearest
|
|
712
|
+
* colour (OKLab), and each colour gets the centre of its pixels and the box
|
|
713
|
+
* from their 10th to their 90th percentile on each axis (fractions), so a
|
|
714
|
+
* colour's role in the brief can be read from where it is, not guessed. */
|
|
715
|
+
function paletteWhere(px, palette, index = null) {
|
|
716
|
+
const xs = palette.map(() => [])
|
|
717
|
+
const ys = palette.map(() => [])
|
|
718
|
+
const ids = index ?? Array.from({ length: px.n }, (_, i) => i)
|
|
719
|
+
for (const i of ids) {
|
|
720
|
+
let best = 0
|
|
721
|
+
let bestD = Infinity
|
|
722
|
+
for (let k = 0; k < palette.length; k++) {
|
|
723
|
+
const o = palette[k].oklab
|
|
724
|
+
const d = (px.ok[i * 3] - o[0]) ** 2 + (px.ok[i * 3 + 1] - o[1]) ** 2 + (px.ok[i * 3 + 2] - o[2]) ** 2
|
|
725
|
+
if (d < bestD) {
|
|
726
|
+
bestD = d
|
|
727
|
+
best = k
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
const x = i % px.width
|
|
731
|
+
xs[best].push((x + 0.5) / px.width)
|
|
732
|
+
ys[best].push(((i - x) / px.width + 0.5) / px.height)
|
|
733
|
+
}
|
|
734
|
+
const pct = (list, q) => {
|
|
735
|
+
const s = Float64Array.from(list).sort()
|
|
736
|
+
return s[clamp(Math.round(q * (s.length - 1)), 0, s.length - 1)]
|
|
737
|
+
}
|
|
738
|
+
return palette.map((p, k) =>
|
|
739
|
+
xs[k].length
|
|
740
|
+
? { ...p, where: { at: [round(xs[k].reduce((s, v) => s + v, 0) / xs[k].length, 3), round(ys[k].reduce((s, v) => s + v, 0) / ys[k].length, 3)], box: [pct(xs[k], 0.1), pct(ys[k], 0.1), pct(xs[k], 0.9), pct(ys[k], 0.9)].map((v) => round(v, 3)) } }
|
|
741
|
+
: p,
|
|
742
|
+
)
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/** Small, saturated colours: what carries the eye (a lamp, a flag, a flower). */
|
|
746
|
+
function accentsOf(px, meanChroma, index = null) {
|
|
747
|
+
const floor = Math.max(1.5 * meanChroma, 20)
|
|
748
|
+
const m = index?.length ?? px.n
|
|
749
|
+
const ids = []
|
|
750
|
+
for (const i of index ?? Array.from({ length: px.n }, (_, k) => k)) if (px.chroma[i] > floor) ids.push(i)
|
|
751
|
+
if (ids.length === 0) return []
|
|
752
|
+
const groups = mergeNear(kmeans(px, ids, 6, 8, SEED + 1, true).map((list) => summarise(px, list)), 8)
|
|
753
|
+
return groups
|
|
754
|
+
.filter((g) => g.count / m >= 0.0005 && g.count / m < 0.05 && g.chroma > 1.5 * meanChroma)
|
|
755
|
+
.sort((a, b) => b.count - a.count || b.chroma - a.chroma)
|
|
756
|
+
.slice(0, 4)
|
|
757
|
+
.map((g) => ({ hex: labToHex(g.lab), share: round(g.count / m) }))
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// ---- structure --------------------------------------------------------------
|
|
761
|
+
// The masked boxes are painted over with a smooth fill from their surround:
|
|
762
|
+
// right for the palette, but not the art's edges, texture or steps, so the
|
|
763
|
+
// structure measures leave them out (a mask, 1 where a box is) rather than
|
|
764
|
+
// read the fill's smoothness, or its rim against the art round it, as the
|
|
765
|
+
// picture's.
|
|
766
|
+
|
|
767
|
+
/** The pixels of a w x h picture in a box (fractions) or next to one (the
|
|
768
|
+
* Sobel kernel reaches a pixel round), 1 there; null without boxes. */
|
|
769
|
+
function maskOf(w, h, boxes) {
|
|
770
|
+
if (!boxes.length) return null
|
|
771
|
+
const m = new Uint8Array(w * h)
|
|
772
|
+
for (const b of boxes) {
|
|
773
|
+
const x0 = clamp(Math.floor(b[0] * w) - 1, 0, w)
|
|
774
|
+
const y0 = clamp(Math.floor(b[1] * h) - 1, 0, h)
|
|
775
|
+
const x1 = clamp(Math.ceil(b[2] * w) + 1, 0, w)
|
|
776
|
+
const y1 = clamp(Math.ceil(b[3] * h) + 1, 0, h)
|
|
777
|
+
for (let y = y0; y < y1; y++) m.fill(1, y * w + x0, y * w + x1)
|
|
778
|
+
}
|
|
779
|
+
return m
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/** Each block's standard deviation of L*, on a grid of `size` pixel blocks;
|
|
783
|
+
* a block mostly masked is left out. */
|
|
784
|
+
function blockSpread(L, w, h, size, mask = null) {
|
|
785
|
+
const bw = Math.max(1, Math.floor(w / size))
|
|
786
|
+
const bh = Math.max(1, Math.floor(h / size))
|
|
787
|
+
const sx = w / bw
|
|
788
|
+
const sy = h / bh
|
|
789
|
+
const out = []
|
|
790
|
+
for (let by = 0; by < bh; by++) {
|
|
791
|
+
for (let bx = 0; bx < bw; bx++) {
|
|
792
|
+
const x0 = Math.floor(bx * sx)
|
|
793
|
+
const x1 = Math.floor((bx + 1) * sx)
|
|
794
|
+
const y0 = Math.floor(by * sy)
|
|
795
|
+
const y1 = Math.floor((by + 1) * sy)
|
|
796
|
+
let s = 0
|
|
797
|
+
let q = 0
|
|
798
|
+
let n = 0
|
|
799
|
+
for (let y = y0; y < y1; y++) {
|
|
800
|
+
for (let x = x0; x < x1; x++) {
|
|
801
|
+
if (mask?.[y * w + x]) continue
|
|
802
|
+
const v = L[y * w + x]
|
|
803
|
+
s += v
|
|
804
|
+
q += v * v
|
|
805
|
+
n++
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
if (n < 0.5 * (y1 - y0) * (x1 - x0)) continue
|
|
809
|
+
const mean = s / n
|
|
810
|
+
out.push({ cx: (x0 + x1) / 2 / w, cy: (y0 + y1) / 2 / h, std: Math.sqrt(Math.max(0, q / n - mean * mean)) })
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
return out
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/** Sobel gradient magnitude of L*, per pixel (L* per pixel; the kernel's sum is divided out). */
|
|
817
|
+
function sobel(L, w, h) {
|
|
818
|
+
const g = new Float64Array(w * h)
|
|
819
|
+
const at = (x, y) => L[Math.min(h - 1, Math.max(0, y)) * w + Math.min(w - 1, Math.max(0, x))]
|
|
820
|
+
for (let y = 0; y < h; y++) {
|
|
821
|
+
for (let x = 0; x < w; x++) {
|
|
822
|
+
const gx = at(x + 1, y - 1) + 2 * at(x + 1, y) + at(x + 1, y + 1) - at(x - 1, y - 1) - 2 * at(x - 1, y) - at(x - 1, y + 1)
|
|
823
|
+
const gy = at(x - 1, y + 1) + 2 * at(x, y + 1) + at(x + 1, y + 1) - at(x - 1, y - 1) - 2 * at(x, y - 1) - at(x + 1, y - 1)
|
|
824
|
+
g[y * w + x] = Math.hypot(gx, gy) / 8
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
return g
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
/** Means of `values` over a rows x cols grid, each over the cell's pixels
|
|
831
|
+
* outside the mask; a cell with none of them takes the mean of the cells
|
|
832
|
+
* that have some (it says nothing of its own). */
|
|
833
|
+
function gridMeans(values, w, h, rows, cols, mask = null) {
|
|
834
|
+
const cells = gridCells(values, w, h, rows, cols, mask)
|
|
835
|
+
const known = cells.flat().filter((c) => c.kept > 0)
|
|
836
|
+
const rest = known.length ? known.reduce((s, c) => s + c.mean, 0) / known.length : 0
|
|
837
|
+
return cells.map((row) => row.map((c) => (c.kept > 0 ? c.mean : rest)))
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
/** Each cell of a rows x cols grid as { mean, kept }: the mean of `values`
|
|
841
|
+
* over its pixels outside the mask, and the share of them it has. */
|
|
842
|
+
function gridCells(values, w, h, rows, cols, mask = null) {
|
|
843
|
+
const out = []
|
|
844
|
+
for (let r = 0; r < rows; r++) {
|
|
845
|
+
const row = []
|
|
846
|
+
const y0 = Math.floor((r * h) / rows)
|
|
847
|
+
const y1 = Math.max(y0 + 1, Math.floor(((r + 1) * h) / rows))
|
|
848
|
+
for (let c = 0; c < cols; c++) {
|
|
849
|
+
const x0 = Math.floor((c * w) / cols)
|
|
850
|
+
const x1 = Math.max(x0 + 1, Math.floor(((c + 1) * w) / cols))
|
|
851
|
+
let s = 0
|
|
852
|
+
let n = 0
|
|
853
|
+
for (let y = y0; y < y1; y++) {
|
|
854
|
+
for (let x = x0; x < x1; x++) {
|
|
855
|
+
if (mask?.[y * w + x]) continue
|
|
856
|
+
s += values[y * w + x]
|
|
857
|
+
n++
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
row.push({ mean: n ? s / n : 0, kept: n / ((y1 - y0) * (x1 - x0)) })
|
|
861
|
+
}
|
|
862
|
+
out.push(row)
|
|
863
|
+
}
|
|
864
|
+
return out
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
const median = (list) => {
|
|
868
|
+
if (list.length === 0) return 0
|
|
869
|
+
const s = Float64Array.from(list).sort()
|
|
870
|
+
return s[Math.floor(s.length / 2)]
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
/**
|
|
874
|
+
* The horizon: the row that parts the picture into two tall regions of
|
|
875
|
+
* different lightness, colour and texture, as a fraction of the height from
|
|
876
|
+
* the top, with its tilt and how sure the reading is. Each region is the
|
|
877
|
+
* band of 12% of the height on its side, and a band is read by the medians
|
|
878
|
+
* of its rows, so a thin band of something else (a row of lettering, a
|
|
879
|
+
* bright cloud edge, a rail) moves neither region: a horizon is only where
|
|
880
|
+
* one tall region ends and another begins, never at a thin band, and only
|
|
881
|
+
* a step between them (a sky that lightens evenly has none). A step nearer
|
|
882
|
+
* an edge than 12% parts off a region thinner than that (a strip of ground,
|
|
883
|
+
* a line of credits) and is no horizon, and one the strips of the picture
|
|
884
|
+
* make on a jagged line (the tops of masses) reads the less sure the more
|
|
885
|
+
* jagged it is. Only rows and columns inside `art`
|
|
886
|
+
* ([x0, y0, x1, y1] in pixels) are searched, so the edge of a letterbox is
|
|
887
|
+
* never taken for one either, and only pixels outside `mask` are read: a
|
|
888
|
+
* row mostly masked says nothing, so neither a masked box's fill nor its
|
|
889
|
+
* rim is a step.
|
|
890
|
+
*/
|
|
891
|
+
function horizonOf(fullL, fullC, fullG, fullW, fullH, art = [0, 0, fullW, fullH], mask = null) {
|
|
892
|
+
const [ax0, ay0, ax1, ay1] = art
|
|
893
|
+
const w = Math.max(1, ax1 - ax0)
|
|
894
|
+
const h = Math.max(1, ay1 - ay0)
|
|
895
|
+
const none = (confidence = 0) => ({ y: null, tiltDeg: 0, confidence: round(confidence, 3) })
|
|
896
|
+
const tall = Math.max(3, Math.round(0.12 * h))
|
|
897
|
+
if (h < 2 * tall + 2) return none()
|
|
898
|
+
const strips = 8
|
|
899
|
+
/** Each row's mean lightness, chroma and edge strength over the columns
|
|
900
|
+
* [x0, x1) outside the mask; NaN for a row the mask covers half of or more. */
|
|
901
|
+
const profile = (x0, x1) => {
|
|
902
|
+
const rows = { L: new Float64Array(h), C: new Float64Array(h), G: new Float64Array(h) }
|
|
903
|
+
for (let y = 0; y < h; y++) {
|
|
904
|
+
let sl = 0
|
|
905
|
+
let sc = 0
|
|
906
|
+
let sg = 0
|
|
907
|
+
let k = 0
|
|
908
|
+
const o = (y + ay0) * fullW + ax0
|
|
909
|
+
for (let x = x0; x < x1; x++) {
|
|
910
|
+
if (mask?.[o + x]) continue
|
|
911
|
+
sl += fullL[o + x]
|
|
912
|
+
sc += fullC[o + x]
|
|
913
|
+
sg += fullG[o + x]
|
|
914
|
+
k++
|
|
915
|
+
}
|
|
916
|
+
const seen = k > 0.5 * (x1 - x0)
|
|
917
|
+
rows.L[y] = seen ? sl / k : NaN
|
|
918
|
+
rows.C[y] = seen ? sc / k : NaN
|
|
919
|
+
rows.G[y] = seen ? sg / k : NaN
|
|
920
|
+
}
|
|
921
|
+
return rows
|
|
922
|
+
}
|
|
923
|
+
/** The known rows of `list` in [y0, y1), or null when fewer than half are. */
|
|
924
|
+
const known = (list, y0, y1) => {
|
|
925
|
+
const out = Array.from(list.subarray(Math.max(0, y0), Math.min(h, y1))).filter(Number.isFinite)
|
|
926
|
+
return out.length >= 0.5 * (y1 - y0) ? out : null
|
|
927
|
+
}
|
|
928
|
+
/** How the rows in the `band` above y differ from the ones below it, by
|
|
929
|
+
* `of` (their median, or their mean): lightness, half the chroma, half the
|
|
930
|
+
* edges; NaN when either side is mostly unknown. */
|
|
931
|
+
const differ = (rows, y, band, of) => {
|
|
932
|
+
let sum = 0
|
|
933
|
+
for (const [key, weight] of [['L', 1], ['C', 0.5], ['G', 0.5]]) {
|
|
934
|
+
const a = known(rows[key], y - band, y)
|
|
935
|
+
const b = known(rows[key], y, y + band)
|
|
936
|
+
if (!a || !b) return NaN
|
|
937
|
+
sum += weight * Math.abs(of(b) - of(a))
|
|
938
|
+
}
|
|
939
|
+
return sum
|
|
940
|
+
}
|
|
941
|
+
const parts = (rows, y, band = tall) => differ(rows, y, band, median)
|
|
942
|
+
const whole = profile(0, w)
|
|
943
|
+
// A parting is a step, not a ramp: a sky that lightens evenly parts every
|
|
944
|
+
// pair of bands a little, and a horizon parts its pair far more than the
|
|
945
|
+
// pairs a band above and below it do (of those that are known).
|
|
946
|
+
const raw = new Float64Array(h).fill(NaN)
|
|
947
|
+
for (let y = tall; y <= h - tall; y++) raw[y] = parts(whole, y)
|
|
948
|
+
const d = new Float64Array(h)
|
|
949
|
+
let best = -1
|
|
950
|
+
for (let y = tall; y <= h - tall; y++) {
|
|
951
|
+
if (!Number.isFinite(raw[y])) continue
|
|
952
|
+
const beside = [y - tall, y + tall].filter((k) => k >= tall && k <= h - tall && Number.isFinite(raw[k])).map((k) => raw[k])
|
|
953
|
+
d[y] = Math.max(0, raw[y] - (beside.length ? beside.reduce((s, v) => s + v, 0) / beside.length : 0))
|
|
954
|
+
if (best < 0 || d[y] > d[best]) best = y
|
|
955
|
+
}
|
|
956
|
+
if (best < 0) return none()
|
|
957
|
+
const dmax = d[best]
|
|
958
|
+
if (!(dmax >= 4)) return none()
|
|
959
|
+
// The middle of the run of rows that part as much (a sharp step parts a run).
|
|
960
|
+
let last = best
|
|
961
|
+
while (last + 1 <= h - tall && d[last + 1] >= 0.98 * dmax) last++
|
|
962
|
+
best = Math.round((best + last) / 2)
|
|
963
|
+
// The row itself, within `reach` of the tall regions' parting: the split
|
|
964
|
+
// of the rows round it that two flat levels fit best (each side's rows
|
|
965
|
+
// against their median, the least absolute misfit), a band past the
|
|
966
|
+
// reach either side. A thin band on one side misfits the same wherever
|
|
967
|
+
// the split is, so it cannot pull the row to its edge, and a ramp under
|
|
968
|
+
// the step leans the row neither way.
|
|
969
|
+
const half = Math.max(2, Math.round(tall / 2))
|
|
970
|
+
const sharpest = (rows, around, reach = half) => {
|
|
971
|
+
const lo = Math.max(0, around - reach - tall)
|
|
972
|
+
const hi = Math.min(h, around + reach + tall)
|
|
973
|
+
let row = around
|
|
974
|
+
let least = Infinity
|
|
975
|
+
for (let y = Math.max(lo + 2, around - reach); y <= Math.min(hi - 2, around + reach); y++) {
|
|
976
|
+
let misfit = 0
|
|
977
|
+
for (const [key, weight] of [['L', 1], ['C', 0.5], ['G', 0.5]]) {
|
|
978
|
+
const a = known(rows[key], lo, y)
|
|
979
|
+
const b = known(rows[key], y, hi)
|
|
980
|
+
if (!a || !b) {
|
|
981
|
+
misfit = Infinity
|
|
982
|
+
break
|
|
983
|
+
}
|
|
984
|
+
const ma = median(a)
|
|
985
|
+
const mb = median(b)
|
|
986
|
+
misfit += weight * (a.reduce((s, v) => s + Math.abs(v - ma), 0) + b.reduce((s, v) => s + Math.abs(v - mb), 0))
|
|
987
|
+
}
|
|
988
|
+
if (misfit < least) {
|
|
989
|
+
least = misfit
|
|
990
|
+
row = y
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return row
|
|
994
|
+
}
|
|
995
|
+
const row = sharpest(whole, best)
|
|
996
|
+
if (row < tall || row > h - tall) return none()
|
|
997
|
+
// Sustained: the same parting shows in most vertical strips of the picture.
|
|
998
|
+
const stripW = Math.max(1, Math.floor(w / strips))
|
|
999
|
+
let sustained = 0
|
|
1000
|
+
const points = []
|
|
1001
|
+
for (let s = 0; s < strips; s++) {
|
|
1002
|
+
const x0 = s * stripW
|
|
1003
|
+
const x1 = s === strips - 1 ? w : x0 + stripW
|
|
1004
|
+
const p = profile(x0, x1)
|
|
1005
|
+
if (parts(p, best) >= 0.4 * raw[best]) sustained++
|
|
1006
|
+
// Each strip's row may lie a whole band off the picture's.
|
|
1007
|
+
points.push([(x0 + x1) / 2, sharpest(p, row, tall)])
|
|
1008
|
+
}
|
|
1009
|
+
let second = 0
|
|
1010
|
+
for (let y = tall; y <= h - tall; y++) if (Math.abs(y - best) > tall) second = Math.max(second, d[y])
|
|
1011
|
+
const prominence = clamp((dmax - Math.max(second, median(Array.from(d.subarray(tall, h - tall + 1))))) / dmax, 0, 1)
|
|
1012
|
+
const share = sustained / strips
|
|
1013
|
+
// A horizon is a line: the strips part on one straight row, level or
|
|
1014
|
+
// tilted, give or take a quarter band (hills, trees, a coast). The tops of
|
|
1015
|
+
// masses or the edge of a cloud bank part them on a jagged one, a boundary
|
|
1016
|
+
// of shapes rather than where the ground meets the sky, and read the less
|
|
1017
|
+
// sure the further the strips stray past that, to nothing at half a band.
|
|
1018
|
+
const mx = points.reduce((s, p) => s + p[0], 0) / points.length
|
|
1019
|
+
const my = points.reduce((s, p) => s + p[1], 0) / points.length
|
|
1020
|
+
let num = 0
|
|
1021
|
+
let den = 0
|
|
1022
|
+
for (const [x, y] of points) {
|
|
1023
|
+
num += (x - mx) * (y - my)
|
|
1024
|
+
den += (x - mx) * (x - mx)
|
|
1025
|
+
}
|
|
1026
|
+
const slope = den > 0 ? num / den : 0
|
|
1027
|
+
const stray = Math.sqrt(points.reduce((s, [x, y]) => s + (y - my - slope * (x - mx)) ** 2, 0) / points.length)
|
|
1028
|
+
const straight = clamp(1 - Math.max(0, stray - tall / 4) / (tall / 4), 0, 1)
|
|
1029
|
+
const tiltDeg = share >= 0.5 ? Math.atan(slope) * DEG : 0
|
|
1030
|
+
const confidence = clamp(dmax / 15, 0, 1) * share * (0.5 + 0.5 * prominence) * straight
|
|
1031
|
+
if (confidence < 0.1) return none(confidence)
|
|
1032
|
+
return { y: round((row + ay0) / fullH, 4), tiltDeg: round(tiltDeg, 2), confidence: round(confidence, 3) }
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/** The haze: from the near side of the picture to the far one, light rising
|
|
1036
|
+
* while contrast and chroma fall, as in aerial perspective (over the
|
|
1037
|
+
* pixels outside the mask). */
|
|
1038
|
+
function hazeOf(L, C, blocks, w, h, mask = null) {
|
|
1039
|
+
const names = ['top', 'upper', 'middle', 'lower', 'bottom']
|
|
1040
|
+
const bands = names.map((band, i) => {
|
|
1041
|
+
const y0 = Math.floor((i * h) / 5)
|
|
1042
|
+
const y1 = Math.floor(((i + 1) * h) / 5)
|
|
1043
|
+
let sl = 0
|
|
1044
|
+
let sc = 0
|
|
1045
|
+
let k = 0
|
|
1046
|
+
for (let y = y0; y < y1; y++) {
|
|
1047
|
+
for (let x = 0; x < w; x++) {
|
|
1048
|
+
if (mask?.[y * w + x]) continue
|
|
1049
|
+
sl += L[y * w + x]
|
|
1050
|
+
sc += C[y * w + x]
|
|
1051
|
+
k++
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
const n = Math.max(1, k)
|
|
1055
|
+
const inBand = blocks.filter((b) => b.cy >= i / 5 && b.cy < (i + 1) / 5)
|
|
1056
|
+
const contrast = inBand.length ? inBand.reduce((s, b) => s + b.std, 0) / inBand.length : 0
|
|
1057
|
+
return { band, meanL: sl / n, contrast, chroma: sc / n }
|
|
1058
|
+
})
|
|
1059
|
+
const mean = (list, key) => list.reduce((s, b) => s + b[key], 0) / list.length
|
|
1060
|
+
const scoreFor = (far, near, mid) => {
|
|
1061
|
+
const fL = mean(far, 'meanL')
|
|
1062
|
+
const nL = mean(near, 'meanL')
|
|
1063
|
+
const sL = clamp((fL - nL) / 25, 0, 1)
|
|
1064
|
+
if (sL < 0.1) return 0
|
|
1065
|
+
const nC = mean(near, 'chroma')
|
|
1066
|
+
const nK = mean(near, 'contrast')
|
|
1067
|
+
const sChroma = nC > 3 ? clamp((nC - mean(far, 'chroma')) / nC, 0, 1) : 0
|
|
1068
|
+
const sContrast = nK > 1.5 ? clamp((nK - mean(far, 'contrast')) / nK, 0, 1) : 0
|
|
1069
|
+
// A haze grows steadily with distance: the middle band lies between.
|
|
1070
|
+
const steady = mid.meanL >= nL - 5 && mid.meanL <= fL + 5 ? 1 : 0.5
|
|
1071
|
+
return steady * (0.45 * sL + 0.35 * sChroma + 0.2 * sContrast)
|
|
1072
|
+
}
|
|
1073
|
+
const up = scoreFor(bands.slice(0, 2), bands.slice(3), bands[2])
|
|
1074
|
+
const down = scoreFor(bands.slice(3), bands.slice(0, 2), bands[2])
|
|
1075
|
+
const score = Math.max(up, down)
|
|
1076
|
+
return {
|
|
1077
|
+
bands: bands.map((b) => ({ band: b.band, meanL: round(b.meanL, 2), contrast: round(b.contrast, 2), chroma: round(b.chroma, 2) })),
|
|
1078
|
+
score: round(score, 3),
|
|
1079
|
+
direction: score < 0.2 ? 'none' : up >= down ? 'toward-top' : 'toward-bottom',
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
/** Where the eye goes: distance from the median L* and chroma on a small grid,
|
|
1084
|
+
* its centroid and its three largest blobs (the focal points f1 to f3, each
|
|
1085
|
+
* with its box); mirror symmetry; empty space. A cell mostly masked is
|
|
1086
|
+
* left out of each (it holds no saliency, edges or emptiness of the art's). */
|
|
1087
|
+
function compositionOf(L, C, grad, w, h, mask = null) {
|
|
1088
|
+
const small = fitEdgeValues([L, C, Float64Array.from(mask ?? new Uint8Array(w * h))], w, h, 64)
|
|
1089
|
+
const { width: sw, height: sh } = small
|
|
1090
|
+
const [sL, sC, sM] = small.values
|
|
1091
|
+
const kept = (i) => sM[i] < 0.5
|
|
1092
|
+
const ids = Array.from({ length: sw * sh }, (_, i) => i).filter(kept)
|
|
1093
|
+
const mL = median(ids.map((i) => sL[i]))
|
|
1094
|
+
const mC = median(ids.map((i) => sC[i]))
|
|
1095
|
+
const sal = new Float64Array(sw * sh)
|
|
1096
|
+
let total = 0
|
|
1097
|
+
let cx = 0
|
|
1098
|
+
let cy = 0
|
|
1099
|
+
for (const i of ids) {
|
|
1100
|
+
const x = i % sw
|
|
1101
|
+
const y = (i - x) / sw
|
|
1102
|
+
const s = Math.abs(sL[i] - mL) + Math.abs(sC[i] - mC)
|
|
1103
|
+
sal[i] = s
|
|
1104
|
+
total += s
|
|
1105
|
+
cx += s * ((x + 0.5) / sw)
|
|
1106
|
+
cy += s * ((y + 0.5) / sh)
|
|
1107
|
+
}
|
|
1108
|
+
const centroid = total > 1e-9 ? [round(cx / total, 3), round(cy / total, 3)] : [0.5, 0.5]
|
|
1109
|
+
let focal = []
|
|
1110
|
+
if (total > 1e-9) {
|
|
1111
|
+
const mean = total / ids.length
|
|
1112
|
+
let q = 0
|
|
1113
|
+
for (const i of ids) q += (sal[i] - mean) * (sal[i] - mean)
|
|
1114
|
+
const t = mean + Math.sqrt(q / ids.length)
|
|
1115
|
+
const seen = new Uint8Array(sal.length)
|
|
1116
|
+
const blobs = []
|
|
1117
|
+
for (let i = 0; i < sal.length; i++) {
|
|
1118
|
+
if (seen[i] || sal[i] <= t) continue
|
|
1119
|
+
let sum = 0
|
|
1120
|
+
let bx = 0
|
|
1121
|
+
let by = 0
|
|
1122
|
+
const box = [sw, sh, 0, 0]
|
|
1123
|
+
const stack = [i]
|
|
1124
|
+
seen[i] = 1
|
|
1125
|
+
while (stack.length) {
|
|
1126
|
+
const j = stack.pop()
|
|
1127
|
+
const x = j % sw
|
|
1128
|
+
const y = (j - x) / sw
|
|
1129
|
+
sum += sal[j]
|
|
1130
|
+
bx += sal[j] * ((x + 0.5) / sw)
|
|
1131
|
+
by += sal[j] * ((y + 0.5) / sh)
|
|
1132
|
+
box[0] = Math.min(box[0], x)
|
|
1133
|
+
box[1] = Math.min(box[1], y)
|
|
1134
|
+
box[2] = Math.max(box[2], x + 1)
|
|
1135
|
+
box[3] = Math.max(box[3], y + 1)
|
|
1136
|
+
for (const [nx, ny] of [[x + 1, y], [x - 1, y], [x, y + 1], [x, y - 1]]) {
|
|
1137
|
+
if (nx < 0 || ny < 0 || nx >= sw || ny >= sh) continue
|
|
1138
|
+
const k = ny * sw + nx
|
|
1139
|
+
if (!seen[k] && sal[k] > t) {
|
|
1140
|
+
seen[k] = 1
|
|
1141
|
+
stack.push(k)
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
blobs.push({ x: bx / sum, y: by / sum, share: sum / total, box: [box[0] / sw, box[1] / sh, box[2] / sw, box[3] / sh] })
|
|
1146
|
+
}
|
|
1147
|
+
focal = blobs
|
|
1148
|
+
.sort((a, b) => b.share - a.share || a.y - b.y || a.x - b.x)
|
|
1149
|
+
.slice(0, 3)
|
|
1150
|
+
.map((b, i) => ({ id: `f${i + 1}`, x: round(b.x, 3), y: round(b.y, 3), share: round(b.share, 3), box: b.box.map((v) => round(v, 3)) }))
|
|
1151
|
+
}
|
|
1152
|
+
// Symmetry over the pairs of cells (one and its mirror) neither of which is masked.
|
|
1153
|
+
let diff = 0
|
|
1154
|
+
let s1 = 0
|
|
1155
|
+
let s2 = 0
|
|
1156
|
+
let n = 0
|
|
1157
|
+
for (const i of ids) {
|
|
1158
|
+
const x = i % sw
|
|
1159
|
+
const j = i - x + (sw - 1 - x)
|
|
1160
|
+
if (!kept(j)) continue
|
|
1161
|
+
diff += Math.abs(sL[i] - sL[j])
|
|
1162
|
+
s1 += sL[i]
|
|
1163
|
+
s2 += sL[i] * sL[i]
|
|
1164
|
+
n++
|
|
1165
|
+
}
|
|
1166
|
+
const std = n ? Math.sqrt(Math.max(0, s2 / n - (s1 / n) ** 2)) : 0
|
|
1167
|
+
const symmetry = n ? 1 - clamp(diff / n / Math.max(10, 2 * std), 0, 1) : 0
|
|
1168
|
+
// Empty space: blocks of the structure picture with almost no edges in them.
|
|
1169
|
+
const size = Math.max(2, Math.round(Math.max(w, h) / 32))
|
|
1170
|
+
const cells = gridCells(grad, w, h, Math.max(1, Math.floor(h / size)), Math.max(1, Math.floor(w / size)), mask)
|
|
1171
|
+
.flat()
|
|
1172
|
+
.filter((c) => c.kept >= 0.5)
|
|
1173
|
+
const negativeSpace = cells.length ? cells.filter((c) => c.mean < 1).length / cells.length : 0
|
|
1174
|
+
return {
|
|
1175
|
+
thirds: gridMeans(L, w, h, 3, 3, mask).map((row) => row.map((v) => round(v, 2))),
|
|
1176
|
+
centroid,
|
|
1177
|
+
focal,
|
|
1178
|
+
symmetry: round(symmetry, 3),
|
|
1179
|
+
negativeSpace: round(negativeSpace, 3),
|
|
1180
|
+
edgeGrid: gridMeans(grad, w, h, 4, 4, mask).map((row) => row.map((v) => round(v, 3))),
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
const BANDS = ['top', 'upper', 'middle', 'lower', 'bottom']
|
|
1185
|
+
const bandOf = (y) => BANDS[clamp(Math.floor(y * 5), 0, 4)]
|
|
1186
|
+
/** Where a point is in the frame, in words: "upper left", "centre". */
|
|
1187
|
+
export function whereOf(x, y) {
|
|
1188
|
+
const col = x < 1 / 3 ? 'left' : x > 2 / 3 ? 'right' : ''
|
|
1189
|
+
const row = y < 1 / 3 ? 'upper' : y > 2 / 3 ? 'lower' : ''
|
|
1190
|
+
return [row, col].filter(Boolean).join(' ') || 'centre'
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
const MASS_CELLS = 64
|
|
1194
|
+
const MASS_MIN_SHARE = 0.02
|
|
1195
|
+
const MASS_SPECK = 0.01
|
|
1196
|
+
const LEVELS = ['dark', 'mid', 'light']
|
|
1197
|
+
|
|
1198
|
+
/** The two lightness thresholds that part `values` (L*, 0 to 100) into
|
|
1199
|
+
* three classes with the most variance between them (Otsu's rule). */
|
|
1200
|
+
function otsu3(values) {
|
|
1201
|
+
const hist = new Float64Array(101)
|
|
1202
|
+
for (const v of values) hist[clamp(Math.round(v), 0, 100)]++
|
|
1203
|
+
const n = values.length
|
|
1204
|
+
const cum = new Float64Array(102)
|
|
1205
|
+
const cumL = new Float64Array(102)
|
|
1206
|
+
for (let i = 0; i <= 100; i++) {
|
|
1207
|
+
cum[i + 1] = cum[i] + hist[i]
|
|
1208
|
+
cumL[i + 1] = cumL[i] + i * hist[i]
|
|
1209
|
+
}
|
|
1210
|
+
const part = (a, b) => {
|
|
1211
|
+
const w = cum[b] - cum[a]
|
|
1212
|
+
return w > 0 ? (cumL[b] - cumL[a]) ** 2 / w : 0
|
|
1213
|
+
}
|
|
1214
|
+
let best = [33, 66]
|
|
1215
|
+
let most = -1
|
|
1216
|
+
for (let t1 = 1; t1 < 100; t1++) {
|
|
1217
|
+
for (let t2 = t1 + 1; t2 <= 100; t2++) {
|
|
1218
|
+
const v = part(0, t1) + part(t1, t2) + part(t2, 101)
|
|
1219
|
+
if (v > most) {
|
|
1220
|
+
most = v
|
|
1221
|
+
best = [t1, t2]
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
return n ? best : [33, 66]
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
/**
|
|
1229
|
+
* The big masses of the picture, measured as a painter's value study (a
|
|
1230
|
+
* notan) reads them: its lightness on a grid of 64 cells on the long edge,
|
|
1231
|
+
* smoothed over each cell's neighbours, parted into dark, mid and light by
|
|
1232
|
+
* the two thresholds that separate them most (otsu3); the shapes of one
|
|
1233
|
+
* value, opened (a run of it two cells thin or less no longer joins two
|
|
1234
|
+
* shapes) and with specks of under 1% folded into the shape around them,
|
|
1235
|
+
* are the masses. Each of 2% of the picture or more is a mass m1, m2, ... by
|
|
1236
|
+
* size, with its value, its box (fractions), its share, its mean colour,
|
|
1237
|
+
* how tall it is against how wide (in pixels), how much of its box it
|
|
1238
|
+
* fills, its band, and which masses lie directly above it and below it (how
|
|
1239
|
+
* they stack). A measurement of shape, not a reading of what it is: the
|
|
1240
|
+
* brief says that.
|
|
1241
|
+
*/
|
|
1242
|
+
function massesOf(lab, w, h) {
|
|
1243
|
+
const grid = fitEdgeValues([0, 1, 2].map((c) => Float64Array.from({ length: w * h }, (_, i) => lab[i * 3 + c])), w, h, MASS_CELLS)
|
|
1244
|
+
const { width: gw, height: gh } = grid
|
|
1245
|
+
const [gl, ga, gb] = grid.values
|
|
1246
|
+
const cells = gw * gh
|
|
1247
|
+
const smooth = new Float64Array(cells)
|
|
1248
|
+
for (let y = 0; y < gh; y++) {
|
|
1249
|
+
for (let x = 0; x < gw; x++) {
|
|
1250
|
+
let s = 0
|
|
1251
|
+
let k = 0
|
|
1252
|
+
for (let dy = -1; dy <= 1; dy++) {
|
|
1253
|
+
for (let dx = -1; dx <= 1; dx++) {
|
|
1254
|
+
const nx = x + dx
|
|
1255
|
+
const ny = y + dy
|
|
1256
|
+
if (nx < 0 || ny < 0 || nx >= gw || ny >= gh) continue
|
|
1257
|
+
s += gl[ny * gw + nx]
|
|
1258
|
+
k++
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
smooth[y * gw + x] = s / k
|
|
1262
|
+
}
|
|
1263
|
+
}
|
|
1264
|
+
const [t1, t2] = otsu3(smooth)
|
|
1265
|
+
const level = Uint8Array.from(smooth, (v) => (v < t1 ? 0 : v < t2 ? 1 : 2))
|
|
1266
|
+
// The cores of each value: its cells whose whole neighbourhood (two cells
|
|
1267
|
+
// round) is that value, so a thin run of it no longer joins two shapes.
|
|
1268
|
+
const R = 2
|
|
1269
|
+
const core = new Uint8Array(cells)
|
|
1270
|
+
for (let y = 0; y < gh; y++) {
|
|
1271
|
+
for (let x = 0; x < gw; x++) {
|
|
1272
|
+
const v = level[y * gw + x]
|
|
1273
|
+
let all = true
|
|
1274
|
+
for (let dy = -R; dy <= R && all; dy++) for (let dx = -R; dx <= R && all; dx++) if (level[clamp(y + dy, 0, gh - 1) * gw + clamp(x + dx, 0, gw - 1)] !== v) all = false
|
|
1275
|
+
core[y * gw + x] = all ? 1 : 0
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
// Shapes: each core's 4-connected cells, then grown back over its value
|
|
1279
|
+
// (at most R cells out, nearest core first); a cell no core reaches is a
|
|
1280
|
+
// shape of its own, as are runs of a value with no core at all.
|
|
1281
|
+
const label = new Int32Array(cells).fill(-1)
|
|
1282
|
+
let shapes = []
|
|
1283
|
+
const flood = (i, inside) => {
|
|
1284
|
+
const id = shapes.length
|
|
1285
|
+
const members = []
|
|
1286
|
+
const stack = [i]
|
|
1287
|
+
label[i] = id
|
|
1288
|
+
while (stack.length) {
|
|
1289
|
+
const j = stack.pop()
|
|
1290
|
+
members.push(j)
|
|
1291
|
+
const x = j % gw
|
|
1292
|
+
const y = (j - x) / gw
|
|
1293
|
+
for (const [dx, dy] of NEIGHBOURS) {
|
|
1294
|
+
const nx = x + dx
|
|
1295
|
+
const ny = y + dy
|
|
1296
|
+
if (nx < 0 || ny < 0 || nx >= gw || ny >= gh) continue
|
|
1297
|
+
const k = ny * gw + nx
|
|
1298
|
+
if (label[k] < 0 && level[k] === level[i] && inside(k)) {
|
|
1299
|
+
label[k] = id
|
|
1300
|
+
stack.push(k)
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
shapes.push({ id, level: level[i], members })
|
|
1305
|
+
}
|
|
1306
|
+
for (let i = 0; i < cells; i++) if (core[i] && label[i] < 0) flood(i, (k) => core[k] === 1)
|
|
1307
|
+
let front = shapes.flatMap((x) => x.members)
|
|
1308
|
+
for (let step = 0; step < R && front.length; step++) {
|
|
1309
|
+
const next = []
|
|
1310
|
+
for (const j of front) {
|
|
1311
|
+
const x = j % gw
|
|
1312
|
+
const y = (j - x) / gw
|
|
1313
|
+
for (const [dx, dy] of NEIGHBOURS) {
|
|
1314
|
+
const nx = x + dx
|
|
1315
|
+
const ny = y + dy
|
|
1316
|
+
if (nx < 0 || ny < 0 || nx >= gw || ny >= gh) continue
|
|
1317
|
+
const k = ny * gw + nx
|
|
1318
|
+
if (label[k] < 0 && level[k] === level[j]) {
|
|
1319
|
+
label[k] = label[j]
|
|
1320
|
+
shapes[label[j]].members.push(k)
|
|
1321
|
+
next.push(k)
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
front = next
|
|
1326
|
+
}
|
|
1327
|
+
for (let i = 0; i < cells; i++) if (label[i] < 0) flood(i, () => true)
|
|
1328
|
+
// A speck is part of the shape it borders most, the smallest first.
|
|
1329
|
+
for (;;) {
|
|
1330
|
+
const speck = shapes.filter((x) => x.members.length > 0 && x.members.length < MASS_SPECK * cells).sort((a, b) => a.members.length - b.members.length || a.id - b.id)[0]
|
|
1331
|
+
if (!speck) break
|
|
1332
|
+
const border = new Map()
|
|
1333
|
+
for (const j of speck.members) {
|
|
1334
|
+
const x = j % gw
|
|
1335
|
+
const y = (j - x) / gw
|
|
1336
|
+
for (const [dx, dy] of NEIGHBOURS) {
|
|
1337
|
+
const nx = x + dx
|
|
1338
|
+
const ny = y + dy
|
|
1339
|
+
if (nx < 0 || ny < 0 || nx >= gw || ny >= gh) continue
|
|
1340
|
+
const o = label[ny * gw + nx]
|
|
1341
|
+
if (o !== speck.id) border.set(o, (border.get(o) ?? 0) + 1)
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
const into = [...border.entries()].sort((a, b) => b[1] - a[1] || a[0] - b[0])[0]?.[0]
|
|
1345
|
+
if (into === undefined) break
|
|
1346
|
+
for (const j of speck.members) label[j] = into
|
|
1347
|
+
shapes[into].members.push(...speck.members)
|
|
1348
|
+
speck.members = []
|
|
1349
|
+
}
|
|
1350
|
+
shapes = shapes.filter((x) => x.members.length > 0)
|
|
1351
|
+
const masses = shapes
|
|
1352
|
+
.filter((x) => x.members.length >= MASS_MIN_SHARE * cells)
|
|
1353
|
+
.sort((a, b) => b.members.length - a.members.length || a.id - b.id)
|
|
1354
|
+
.slice(0, 8)
|
|
1355
|
+
const idOf = new Map(masses.map((x, i) => [x.id, `m${i + 1}`]))
|
|
1356
|
+
// How they stack: a mass lies above another where its cells sit directly
|
|
1357
|
+
// over the other's along a few columns or more.
|
|
1358
|
+
const touches = new Map()
|
|
1359
|
+
for (let y = 0; y + 1 < gh; y++) {
|
|
1360
|
+
for (let x = 0; x < gw; x++) {
|
|
1361
|
+
const a = idOf.get(label[y * gw + x])
|
|
1362
|
+
const b = idOf.get(label[(y + 1) * gw + x])
|
|
1363
|
+
if (a && b && a !== b) touches.set(`${a}>${b}`, (touches.get(`${a}>${b}`) ?? 0) + 1)
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
const over = (a, b) => (touches.get(`${a}>${b}`) ?? 0) >= 2 && (touches.get(`${a}>${b}`) ?? 0) > (touches.get(`${b}>${a}`) ?? 0)
|
|
1367
|
+
const list = masses.map((x) => {
|
|
1368
|
+
let [x0, y0, x1, y1] = [gw, gh, 0, 0]
|
|
1369
|
+
let sx = 0
|
|
1370
|
+
let sy = 0
|
|
1371
|
+
const mean = [0, 0, 0]
|
|
1372
|
+
for (const j of x.members) {
|
|
1373
|
+
const cx = j % gw
|
|
1374
|
+
const cy = (j - cx) / gw
|
|
1375
|
+
x0 = Math.min(x0, cx)
|
|
1376
|
+
y0 = Math.min(y0, cy)
|
|
1377
|
+
x1 = Math.max(x1, cx + 1)
|
|
1378
|
+
y1 = Math.max(y1, cy + 1)
|
|
1379
|
+
sx += cx + 0.5
|
|
1380
|
+
sy += cy + 0.5
|
|
1381
|
+
mean[0] += gl[j]
|
|
1382
|
+
mean[1] += ga[j]
|
|
1383
|
+
mean[2] += gb[j]
|
|
1384
|
+
}
|
|
1385
|
+
const k = x.members.length
|
|
1386
|
+
const at = [sx / k / gw, sy / k / gh]
|
|
1387
|
+
return {
|
|
1388
|
+
id: idOf.get(x.id),
|
|
1389
|
+
value: LEVELS[x.level],
|
|
1390
|
+
box: [x0 / gw, y0 / gh, x1 / gw, y1 / gh].map((v) => round(v, 3)),
|
|
1391
|
+
share: round(k / cells, 3),
|
|
1392
|
+
hex: labToHex(mean.map((v) => v / k)),
|
|
1393
|
+
tall: round(((y1 - y0) * h * gw) / Math.max(1e-9, (x1 - x0) * w * gh), 2),
|
|
1394
|
+
fill: round(k / Math.max(1, (x1 - x0) * (y1 - y0)), 2),
|
|
1395
|
+
at: at.map((v) => round(v, 3)),
|
|
1396
|
+
band: bandOf(at[1]),
|
|
1397
|
+
above: [],
|
|
1398
|
+
below: [],
|
|
1399
|
+
}
|
|
1400
|
+
})
|
|
1401
|
+
for (const m of list) {
|
|
1402
|
+
for (const o of list) {
|
|
1403
|
+
if (m === o) continue
|
|
1404
|
+
if (over(o.id, m.id)) m.above.push(o.id)
|
|
1405
|
+
if (over(m.id, o.id)) m.below.push(o.id)
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
const massAt = (fx, fy) => idOf.get(label[clamp(Math.floor(fy * gh), 0, gh - 1) * gw + clamp(Math.floor(fx * gw), 0, gw - 1)]) ?? null
|
|
1409
|
+
return { masses: list, massAt, levels: [t1, t2] }
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
/** Several per-pixel value arrays box-averaged to a long edge of `edge`. */
|
|
1413
|
+
function fitEdgeValues(arrays, w, h, edge) {
|
|
1414
|
+
const k = Math.min(1, edge / Math.max(w, h))
|
|
1415
|
+
const W = Math.max(1, Math.round(w * k))
|
|
1416
|
+
const H = Math.max(1, Math.round(h * k))
|
|
1417
|
+
const values = arrays.map((src) => {
|
|
1418
|
+
const out = new Float64Array(W * H)
|
|
1419
|
+
for (let y = 0; y < H; y++) {
|
|
1420
|
+
const y0 = Math.floor((y * h) / H)
|
|
1421
|
+
const y1 = Math.max(y0 + 1, Math.floor(((y + 1) * h) / H))
|
|
1422
|
+
for (let x = 0; x < W; x++) {
|
|
1423
|
+
const x0 = Math.floor((x * w) / W)
|
|
1424
|
+
const x1 = Math.max(x0 + 1, Math.floor(((x + 1) * w) / W))
|
|
1425
|
+
let s = 0
|
|
1426
|
+
for (let sy = y0; sy < y1; sy++) for (let sx = x0; sx < x1; sx++) s += src[sy * w + sx]
|
|
1427
|
+
out[y * W + x] = s / ((y1 - y0) * (x1 - x0))
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
return out
|
|
1431
|
+
})
|
|
1432
|
+
return { values, width: W, height: H }
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
/** A uniform border (letterbox, pillarbox, a mat) and the crop that leaves
|
|
1436
|
+
* the art: what is not the picture at all. Lettering over the art is not
|
|
1437
|
+
* cropped (that would cut the art with it): letteringOf finds it and it is
|
|
1438
|
+
* masked instead. */
|
|
1439
|
+
function frameOf(L, w, h) {
|
|
1440
|
+
const uniformRow = (y) => {
|
|
1441
|
+
let s = 0
|
|
1442
|
+
let q = 0
|
|
1443
|
+
for (let x = 0; x < w; x++) {
|
|
1444
|
+
const v = L[y * w + x]
|
|
1445
|
+
s += v
|
|
1446
|
+
q += v * v
|
|
1447
|
+
}
|
|
1448
|
+
const m = s / w
|
|
1449
|
+
return { m, std: Math.sqrt(Math.max(0, q / w - m * m)) }
|
|
1450
|
+
}
|
|
1451
|
+
const uniformCol = (x) => {
|
|
1452
|
+
let s = 0
|
|
1453
|
+
let q = 0
|
|
1454
|
+
for (let y = 0; y < h; y++) {
|
|
1455
|
+
const v = L[y * w + x]
|
|
1456
|
+
s += v
|
|
1457
|
+
q += v * v
|
|
1458
|
+
}
|
|
1459
|
+
const m = s / h
|
|
1460
|
+
return { m, std: Math.sqrt(Math.max(0, q / h - m * m)) }
|
|
1461
|
+
}
|
|
1462
|
+
// A border is uniform, at most 15% of its side (a flat sky or wall beyond
|
|
1463
|
+
// that is content) and ends in a step: the art after it differs by 8 L*
|
|
1464
|
+
// or more, where a smooth sky would only drift.
|
|
1465
|
+
const run = (count, line) => {
|
|
1466
|
+
const first = line(0)
|
|
1467
|
+
if (first.std >= 2) return 0
|
|
1468
|
+
let k = 0
|
|
1469
|
+
while (k < count) {
|
|
1470
|
+
const l = line(k)
|
|
1471
|
+
if (l.std >= 2 || Math.abs(l.m - first.m) >= 3) break
|
|
1472
|
+
k++
|
|
1473
|
+
}
|
|
1474
|
+
if (k < 2 || k > 0.15 * count || k + 3 > count) return 0
|
|
1475
|
+
const after = (line(k).m + line(k + 1).m + line(k + 2).m) / 3
|
|
1476
|
+
return Math.abs(after - first.m) >= 8 ? k : 0
|
|
1477
|
+
}
|
|
1478
|
+
const top = run(h, (k) => uniformRow(k))
|
|
1479
|
+
const bottom = run(h, (k) => uniformRow(h - 1 - k))
|
|
1480
|
+
const left = run(w, (k) => uniformCol(k))
|
|
1481
|
+
const right = run(w, (k) => uniformCol(w - 1 - k))
|
|
1482
|
+
const art = [left, top, w - right, h - bottom]
|
|
1483
|
+
const framed = top + bottom + left + right > 0 && art[3] - art[1] >= 0.4 * h && art[2] - art[0] >= 0.4 * w
|
|
1484
|
+
return {
|
|
1485
|
+
border: { top: round(top / h, 4), bottom: round(bottom / h, 4), left: round(left / w, 4), right: round(right / w, 4) },
|
|
1486
|
+
suggestedCrop: framed ? [round(art[0] / w, 4), round(art[1] / h, 4), round(art[2] / w, 4), round(art[3] / h, 4)] : null,
|
|
1487
|
+
art,
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
/** A horizon read at least this sure gives the camera guess its pitch; a
|
|
1492
|
+
* guess without one is a default, and its pitch says nothing. */
|
|
1493
|
+
const PITCH_FROM = 0.25
|
|
1494
|
+
|
|
1495
|
+
/** The camera, guessed from where the horizon sits and how detail spreads
|
|
1496
|
+
* from top to bottom. Honest about how little a single picture says. */
|
|
1497
|
+
function cameraOf(horizon, edgeGrid) {
|
|
1498
|
+
const fov = 50
|
|
1499
|
+
if (horizon.y !== null && horizon.confidence >= PITCH_FROM) {
|
|
1500
|
+
const y = horizon.y
|
|
1501
|
+
const pitchDeg = -Math.atan((0.5 - y) * 2 * Math.tan(fov / 2 / DEG)) * DEG
|
|
1502
|
+
const guess = y > 0.62 ? 'low' : y >= 0.38 ? 'eye-level' : y >= 0.15 ? 'high' : 'aerial'
|
|
1503
|
+
const where = y > 0.62 ? 'below the middle, so the camera looks up' : y >= 0.38 ? 'near the middle, so the camera looks about level' : 'above the middle, so the camera looks down'
|
|
1504
|
+
return {
|
|
1505
|
+
guess,
|
|
1506
|
+
pitchDeg: round(pitchDeg, 1),
|
|
1507
|
+
confidence: round(0.7 * horizon.confidence, 3),
|
|
1508
|
+
why: `The horizon is at ${Math.round(y * 100)}% of the height (confidence ${horizon.confidence.toFixed(2)}), ${where}; a ${fov} degree lens is assumed.`,
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
1511
|
+
const rowEnergy = edgeGrid.map((row) => row.reduce((s, v) => s + v, 0) / row.length)
|
|
1512
|
+
const topE = rowEnergy[0]
|
|
1513
|
+
const bottomE = rowEnergy[rowEnergy.length - 1]
|
|
1514
|
+
const ratio = (topE + 0.05) / (bottomE + 0.05)
|
|
1515
|
+
if (ratio >= 0.6 && ratio <= 1.6 && bottomE > 1) {
|
|
1516
|
+
return { guess: 'top-down', pitchDeg: -90, confidence: 0.2, why: 'No horizon, and detail is spread evenly from top to bottom, as when looking down on the ground. A weak guess.' }
|
|
1517
|
+
}
|
|
1518
|
+
return {
|
|
1519
|
+
guess: 'eye-level',
|
|
1520
|
+
pitchDeg: 0,
|
|
1521
|
+
confidence: ratio < 0.5 ? 0.2 : 0.1,
|
|
1522
|
+
why: ratio < 0.5 ? 'No sharp horizon, but the top holds much less detail than the bottom (a sky above ground). A weak guess.' : 'No horizon to read the camera from. A guess, not a reading.',
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
/** The pixels of a picture outside every box (fractions), as their indices. */
|
|
1527
|
+
function outside(pic, boxes) {
|
|
1528
|
+
if (!boxes.length) return null
|
|
1529
|
+
const { width: w, height: h } = pic
|
|
1530
|
+
const out = []
|
|
1531
|
+
for (let y = 0; y < h; y++) {
|
|
1532
|
+
const v = (y + 0.5) / h
|
|
1533
|
+
for (let x = 0; x < w; x++) {
|
|
1534
|
+
const u = (x + 0.5) / w
|
|
1535
|
+
if (!boxes.some((b) => u >= b[0] && u < b[2] && v >= b[1] && v < b[3])) out.push(y * w + x)
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
return out
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
/**
|
|
1542
|
+
* The facts of one picture (orchestrator spec 3.1). `pic` is its RGBA at
|
|
1543
|
+
* any size; `crop` (fractions) is applied first, and `source` is its
|
|
1544
|
+
* project-relative path, recorded as given. Its lettering is found and
|
|
1545
|
+
* masked out before anything else is measured (lettering false skips that,
|
|
1546
|
+
* for a shot of the game, which has none of the reference's); `mask` are
|
|
1547
|
+
* boxes to leave out as well (a shot measured against a reference leaves
|
|
1548
|
+
* out the reference's lettering, so both are measured over the same
|
|
1549
|
+
* pixels). Every fraction is of the picture after the crop.
|
|
1550
|
+
*/
|
|
1551
|
+
export function referenceFacts(pic, { source = '', crop = null, lettering = true, mask = [] } = {}) {
|
|
1552
|
+
const art = cropPicture(pic, crop)
|
|
1553
|
+
const text = lettering ? letteringOf(art) : []
|
|
1554
|
+
const boxes = [...text.map((t) => t.box), ...mask]
|
|
1555
|
+
const clean = maskBoxes(art, boxes)
|
|
1556
|
+
const colourPic = fitEdge(clean, COLOUR_EDGE)
|
|
1557
|
+
const colour = analyse(colourPic)
|
|
1558
|
+
const structurePic = fitEdge(clean, STRUCTURE_EDGE)
|
|
1559
|
+
const structure = analyse(structurePic)
|
|
1560
|
+
// The colour of what is measured: the masked boxes are not the art's.
|
|
1561
|
+
const kept = outside(colourPic, boxes)
|
|
1562
|
+
const ids = kept ?? Array.from({ length: colour.n }, (_, i) => i)
|
|
1563
|
+
const n = ids.length
|
|
1564
|
+
|
|
1565
|
+
// Value and saturation, on the colour picture.
|
|
1566
|
+
const Ls = new Float64Array(n)
|
|
1567
|
+
let sumL = 0
|
|
1568
|
+
let sumA = 0
|
|
1569
|
+
let sumB = 0
|
|
1570
|
+
let sumC = 0
|
|
1571
|
+
let high = 0
|
|
1572
|
+
let low = 0
|
|
1573
|
+
let warm = 0
|
|
1574
|
+
let cool = 0
|
|
1575
|
+
const histogram = new Array(16).fill(0)
|
|
1576
|
+
const bytes = colourPic.data
|
|
1577
|
+
let mrg = 0
|
|
1578
|
+
let myb = 0
|
|
1579
|
+
let qrg = 0
|
|
1580
|
+
let qyb = 0
|
|
1581
|
+
ids.forEach((i, j) => {
|
|
1582
|
+
const L = colour.lab[i * 3]
|
|
1583
|
+
const a = colour.lab[i * 3 + 1]
|
|
1584
|
+
const b = colour.lab[i * 3 + 2]
|
|
1585
|
+
const c = colour.chroma[i]
|
|
1586
|
+
Ls[j] = L
|
|
1587
|
+
sumL += L
|
|
1588
|
+
sumA += a
|
|
1589
|
+
sumB += b
|
|
1590
|
+
sumC += c
|
|
1591
|
+
histogram[clamp(Math.floor((L / 100) * 16), 0, 15)]++
|
|
1592
|
+
const r = bytes[i * 4]
|
|
1593
|
+
const g = bytes[i * 4 + 1]
|
|
1594
|
+
const bl = bytes[i * 4 + 2]
|
|
1595
|
+
if (r >= 250 && g >= 250 && bl >= 250) high++
|
|
1596
|
+
if (r <= 5 && g <= 5 && bl <= 5) low++
|
|
1597
|
+
if (c > 10) {
|
|
1598
|
+
const hue = ((Math.atan2(b, a) * DEG) + 360) % 360
|
|
1599
|
+
if (hue >= 330 || hue < 110) warm++
|
|
1600
|
+
else if (hue >= 150 && hue < 300) cool++
|
|
1601
|
+
}
|
|
1602
|
+
const rg = r - g
|
|
1603
|
+
const yb = 0.5 * (r + g) - bl
|
|
1604
|
+
mrg += rg
|
|
1605
|
+
myb += yb
|
|
1606
|
+
qrg += rg * rg
|
|
1607
|
+
qyb += yb * yb
|
|
1608
|
+
})
|
|
1609
|
+
const meanL = sumL / n
|
|
1610
|
+
let spread = 0
|
|
1611
|
+
for (let i = 0; i < n; i++) spread += (Ls[i] - meanL) ** 2
|
|
1612
|
+
const sorted = Float64Array.from(Ls).sort()
|
|
1613
|
+
const pct = (q) => sorted[clamp(Math.round(q * (n - 1)), 0, n - 1)]
|
|
1614
|
+
const chromaSorted = Float64Array.from(ids.map((i) => colour.chroma[i])).sort()
|
|
1615
|
+
const meanChroma = sumC / n
|
|
1616
|
+
mrg /= n
|
|
1617
|
+
myb /= n
|
|
1618
|
+
const colourfulness = Math.sqrt(Math.max(0, qrg / n - mrg * mrg) + Math.max(0, qyb / n - myb * myb)) + 0.3 * Math.hypot(mrg, myb)
|
|
1619
|
+
const p50 = pct(0.5)
|
|
1620
|
+
|
|
1621
|
+
// Structure, on the larger picture: L* and chroma per pixel.
|
|
1622
|
+
const w = structure.width
|
|
1623
|
+
const h = structure.height
|
|
1624
|
+
const SL = new Float64Array(structure.n)
|
|
1625
|
+
const SC = new Float64Array(structure.n)
|
|
1626
|
+
for (let i = 0; i < structure.n; i++) {
|
|
1627
|
+
SL[i] = structure.lab[i * 3]
|
|
1628
|
+
SC[i] = structure.chroma[i]
|
|
1629
|
+
}
|
|
1630
|
+
const masked = maskOf(w, h, boxes)
|
|
1631
|
+
const blocks = blockSpread(SL, w, h, Math.max(2, Math.round(Math.max(w, h) / 32)), masked)
|
|
1632
|
+
const grad = sobel(SL, w, h)
|
|
1633
|
+
const composition = compositionOf(SL, SC, grad, w, h, masked)
|
|
1634
|
+
const { art: inner, ...frame } = frameOf(SL, w, h)
|
|
1635
|
+
const horizon = horizonOf(SL, SC, grad, w, h, inner, masked)
|
|
1636
|
+
const { masses, massAt } = massesOf(structure.lab, w, h)
|
|
1637
|
+
composition.focal = composition.focal.map((f) => ({ ...f, mass: massAt(f.x, f.y), where: whereOf(f.x, f.y) }))
|
|
1638
|
+
const aspect = art.width / art.height
|
|
1639
|
+
|
|
1640
|
+
return {
|
|
1641
|
+
source,
|
|
1642
|
+
crop: crop ? crop.map((v) => round(clamp(Number(v) || 0, 0, 1), 4)) : null,
|
|
1643
|
+
size: { width: art.width, height: art.height, aspect: round(aspect, 4), shape: shapeOf(aspect) },
|
|
1644
|
+
text: { boxes: text, share: round(n === colour.n ? 0 : 1 - n / colour.n, 4) },
|
|
1645
|
+
palette: paletteWhere(colour, paletteOf(colour, kept), kept),
|
|
1646
|
+
accents: accentsOf(colour, meanChroma, kept),
|
|
1647
|
+
value: {
|
|
1648
|
+
key: p50 < 35 ? 'low' : p50 > 65 ? 'high' : 'mid',
|
|
1649
|
+
meanL: round(meanL, 2),
|
|
1650
|
+
p5: round(pct(0.05), 2),
|
|
1651
|
+
p50: round(p50, 2),
|
|
1652
|
+
p95: round(pct(0.95), 2),
|
|
1653
|
+
histogram: histogram.map((c) => round(c / n)),
|
|
1654
|
+
clipHigh: round(high / n),
|
|
1655
|
+
clipLow: round(low / n),
|
|
1656
|
+
},
|
|
1657
|
+
contrast: { stdL: round(Math.sqrt(spread / n), 2), local: round(blocks.reduce((s, b) => s + b.std, 0) / blocks.length, 2) },
|
|
1658
|
+
saturation: { meanChroma: round(meanChroma, 2), p90Chroma: round(chromaSorted[clamp(Math.round(0.9 * (n - 1)), 0, n - 1)], 2), colourfulness: round(colourfulness, 2) },
|
|
1659
|
+
temperature: { warmth: round(sumB / n, 2), tint: round(sumA / n, 2), warmShare: round(warm / n), coolShare: round(cool / n) },
|
|
1660
|
+
haze: hazeOf(SL, SC, blocks, w, h, masked),
|
|
1661
|
+
composition,
|
|
1662
|
+
masses,
|
|
1663
|
+
horizon,
|
|
1664
|
+
camera: cameraOf(horizon, composition.edgeGrid),
|
|
1665
|
+
frame: { ...frame, textLikely: text.length > 0 },
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
// ---- how sure ------------------------------------------------------------------
|
|
1670
|
+
|
|
1671
|
+
/**
|
|
1672
|
+
* The readings of a picture too unsure to stand on (confidence under
|
|
1673
|
+
* CONFIDENT): the horizon (or none found), the camera guess. Each may inform
|
|
1674
|
+
* an assumption of the brief; none may become a request element, a
|
|
1675
|
+
* camera's horizonY or a critic criterion. [{ fact, confidence, value, why }]:
|
|
1676
|
+
* value is the number read (the horizon's row, the guess's pitch), or null
|
|
1677
|
+
* when nothing was (no horizon; a pitch guessed without one).
|
|
1678
|
+
*/
|
|
1679
|
+
export function advisoryOf(facts) {
|
|
1680
|
+
if (!facts) return []
|
|
1681
|
+
const out = []
|
|
1682
|
+
const hz = facts.horizon
|
|
1683
|
+
if (hz && !(hz.y !== null && hz.confidence >= CONFIDENT)) out.push({ fact: 'horizon', confidence: hz.confidence ?? 0, value: hz.y ?? null, why: hz.y === null ? 'no horizon was found' : `the horizon at ${hz.y} reads at confidence ${hz.confidence}` })
|
|
1684
|
+
const cam = facts.camera
|
|
1685
|
+
const read = hz?.y !== null && hz?.y !== undefined && hz.confidence >= PITCH_FROM
|
|
1686
|
+
if (cam && cam.confidence < CONFIDENT) out.push({ fact: 'camera', confidence: cam.confidence, value: read ? cam.pitchDeg : null, why: `the camera guess (${cam.guess}, pitch ${cam.pitchDeg}) reads at confidence ${cam.confidence}` })
|
|
1687
|
+
return out
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
const HORIZON_WORDS = /\bhorizon/i
|
|
1691
|
+
/** The places in the frame a text gives, as fractions of its height: a
|
|
1692
|
+
* percentage, or a bare number from 0 to 1 (never a size or an angle). */
|
|
1693
|
+
const placesIn = (text) =>
|
|
1694
|
+
[...String(text).matchAll(/(\d+(?:\.\d+)?)\s*(%|percent\b|°|deg(?:rees?)?\b|m\b|metres?\b|meters?\b)?/gi)]
|
|
1695
|
+
.map((m) => (m[2] && /^(%|percent)$/i.test(m[2]) ? Number(m[1]) / 100 : !m[2] && Number(m[1]) <= 1 ? Number(m[1]) : null))
|
|
1696
|
+
.filter((v) => v !== null)
|
|
1697
|
+
/** The pitches a text gives, in degrees: a number with a degree mark, or one just after the word pitch. */
|
|
1698
|
+
const pitchesIn = (text) => [...String(text).matchAll(/(-?\d+(?:\.\d+)?)\s*(?:°|deg(?:rees?)?\b)|\bpitch(?:ed|es)?\b[^\d.-]{0,16}(-?\d+(?:\.\d+)?)/gi)].map((m) => Number(m[1] ?? m[2]))
|
|
1699
|
+
|
|
1700
|
+
/**
|
|
1701
|
+
* Which advisory readings a text stands on (advisoryOf): by the numbers it
|
|
1702
|
+
* gives that only the unsure reading could, never by the words for what the
|
|
1703
|
+
* picture shows (a low camera, looking up, eye level: a model sees those as
|
|
1704
|
+
* well as the facts do, and an advisory pitch may stay in the brief as an
|
|
1705
|
+
* assumption). A text about the
|
|
1706
|
+
* reference picture (`reference`: from the reference, or judged at its
|
|
1707
|
+
* camera) stands on the horizon when it gives the horizon a place in the
|
|
1708
|
+
* frame (the facts are the only measure of it, and they are unsure), and
|
|
1709
|
+
* on the camera guess when it gives the guess's pitch; any other text only
|
|
1710
|
+
* when it quotes the horizon's own row. The fact names, or [].
|
|
1711
|
+
*/
|
|
1712
|
+
export function citesAdvisory(text, facts, { reference = false } = {}) {
|
|
1713
|
+
const words = String(text ?? '')
|
|
1714
|
+
const out = []
|
|
1715
|
+
for (const a of advisoryOf(facts)) {
|
|
1716
|
+
if (a.fact === 'horizon' && HORIZON_WORDS.test(words)) {
|
|
1717
|
+
const places = placesIn(words)
|
|
1718
|
+
if (reference ? places.length > 0 : a.value !== null && places.some((v) => Math.abs(v - a.value) <= 0.02)) out.push('horizon')
|
|
1719
|
+
}
|
|
1720
|
+
if (a.fact === 'camera' && reference && a.value !== null && pitchesIn(words).some((n) => Math.abs(Math.abs(n) - Math.abs(a.value)) <= 1)) out.push('camera')
|
|
1721
|
+
}
|
|
1722
|
+
return out
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1725
|
+
/** A box of fractions, in a few characters. */
|
|
1726
|
+
const boxText = (b) => `[${b.map((v) => Number(v).toFixed(2)).join(', ')}]`
|
|
1727
|
+
|
|
1728
|
+
/** What was taken out of a picture before it was measured, in one sentence:
|
|
1729
|
+
* the lettering masked and the frame cropped, or that nothing was. */
|
|
1730
|
+
export function removedOf(facts) {
|
|
1731
|
+
const boxes = facts?.text?.boxes ?? []
|
|
1732
|
+
const parts = []
|
|
1733
|
+
if (boxes.length) parts.push(`its lettering was masked out (${boxes.length} box${boxes.length === 1 ? '' : 'es'}: ${boxes.map((b) => `${b.kind === 'title' ? 'a title' : 'small type'} at ${boxText(b.box)}`).join(', ')}, ${Math.round((facts.text.share ?? 0) * 100)}% of the picture, painted over with the colours around each box)`)
|
|
1734
|
+
if (facts?.crop) parts.push(`its frame was cropped away (the crop ${boxText(facts.crop)} of the whole picture)`)
|
|
1735
|
+
return parts.length ? `Before it was measured, ${parts.join(', and ')}.` : 'Nothing was masked or cropped: no lettering and no frame were found.'
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
/** The facts in a few lines a model reads before the JSON. */
|
|
1739
|
+
export function factsSummary(facts) {
|
|
1740
|
+
const pct = (v) => `${Math.round(v * 100)}%`
|
|
1741
|
+
const sure = (c) => (c >= CONFIDENT ? `confidence ${c}` : `confidence ${c}, under ${CONFIDENT}: advisory`)
|
|
1742
|
+
const lines = []
|
|
1743
|
+
lines.push(`${facts.source || 'The picture'}: ${facts.size.width} x ${facts.size.height}, aspect ${facts.size.aspect} (${facts.size.shape})${facts.crop ? `, cropped to ${boxText(facts.crop)}` : ''}. Every box is [x0, y0, x1, y1] in fractions of it, from its top left.`)
|
|
1744
|
+
lines.push(removedOf(facts))
|
|
1745
|
+
lines.push(`Palette (where each colour sits: its centre and the box most of it lies in): ${facts.palette.map((p) => `${p.hex} ${pct(p.share)}${p.where ? ` at (${p.where.at.join(', ')}) in ${boxText(p.where.box)}` : ''}`).join('; ')}.`)
|
|
1746
|
+
if (facts.accents.length) lines.push(`Accents (small and saturated): ${facts.accents.map((a) => `${a.hex} ${(a.share * 100).toFixed(1)}%`).join(', ')}.`)
|
|
1747
|
+
const v = facts.value
|
|
1748
|
+
lines.push(`Value: ${v.key} key, median L* ${v.p50} (5th ${v.p5}, 95th ${v.p95}); clipped highlights ${pct(v.clipHigh)}, crushed shadows ${pct(v.clipLow)}.`)
|
|
1749
|
+
lines.push(`Contrast: L* spread ${facts.contrast.stdL}, local ${facts.contrast.local}. Saturation: mean chroma ${facts.saturation.meanChroma}, 90th ${facts.saturation.p90Chroma}, colourfulness ${facts.saturation.colourfulness}.`)
|
|
1750
|
+
const t = facts.temperature
|
|
1751
|
+
lines.push(`Temperature: b* ${t.warmth} (${t.warmth > 2 ? 'warm' : t.warmth < -2 ? 'cool' : 'neutral'}), a* ${t.tint}; warm colours ${pct(t.warmShare)}, cool ${pct(t.coolShare)}.`)
|
|
1752
|
+
const hz = facts.haze
|
|
1753
|
+
lines.push(`Haze: score ${hz.score}, ${hz.direction === 'none' ? 'none to speak of' : hz.direction === 'toward-top' ? 'far layers toward the top are lighter and flatter' : 'far layers toward the bottom are lighter and flatter'} (band L* top to bottom ${hz.bands.map((b) => Math.round(b.meanL)).join('/')}).`)
|
|
1754
|
+
if (facts.masses?.length) {
|
|
1755
|
+
lines.push(
|
|
1756
|
+
`Masses (a value study of the big dark, mid and light shapes, largest first; tall is height over width): ${facts.masses
|
|
1757
|
+
.map((m) => `${m.id} ${m.value} ${boxText(m.box)} ${pct(m.share)}, tall ${m.tall}${m.above.length ? `, under ${m.above.join(' and ')}` : ''}${m.below.length ? `, over ${m.below.join(' and ')}` : ''}`)
|
|
1758
|
+
.join('; ')}.`,
|
|
1759
|
+
)
|
|
1760
|
+
}
|
|
1761
|
+
const c = facts.composition
|
|
1762
|
+
lines.push(`Composition: the eye goes to (${c.centroid.join(', ')})${c.focal.length ? `; focal points ${c.focal.map((f) => `${f.id ?? ''} (${f.x}, ${f.y}) ${f.where ?? ''} ${pct(f.share)}${f.box ? ` in ${boxText(f.box)}` : ''}${f.mass ? `, on ${f.mass}` : ''}`.replace(/\s+/g, ' ').trim()).join('; ')}` : ''}; symmetry ${c.symmetry}; empty space ${pct(c.negativeSpace)}.`)
|
|
1763
|
+
lines.push(`Horizon: ${facts.horizon.y === null ? 'none found' : `at ${pct(facts.horizon.y)} of the height, tilt ${facts.horizon.tiltDeg} degrees`} (${sure(facts.horizon.confidence)}).`)
|
|
1764
|
+
lines.push(`Camera guess: ${facts.camera.guess}, pitch ${facts.camera.pitchDeg} degrees (${sure(facts.camera.confidence)}). ${facts.camera.why}`)
|
|
1765
|
+
const advisory = advisoryOf(facts)
|
|
1766
|
+
if (advisory.length) lines.push(`Advisory (${advisory.map((a) => a.why).join('; ')}): an advisory reading's numbers (a horizon's place in the frame, a pitch) may inform an assumption, never a request element, a camera's horizonY or a critic criterion; what the picture clearly shows (a low camera, looking up) is said in words.`)
|
|
1767
|
+
const f = facts.frame
|
|
1768
|
+
const border = Object.entries(f.border).filter(([, v]) => v > 0)
|
|
1769
|
+
if (border.length) lines.push(`Frame: a uniform border (${border.map(([k, v]) => `${k} ${pct(v)}`).join(', ')}).${f.suggestedCrop ? ` The art alone is the crop ${boxText(f.suggestedCrop)}: read that, not the frame.` : ''}`)
|
|
1770
|
+
lines.push('These describe the picture; the space draws its mood, light, palette and forms from them and never reproduces a specific character, logo or text from it.')
|
|
1771
|
+
return lines.join('\n')
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
// ---- a shot against the reference ------------------------------------------
|
|
1775
|
+
|
|
1776
|
+
/** The L* means of a picture on a grid of about 32 cells shaped like `aspect`. */
|
|
1777
|
+
function lGrid(pic, aspect) {
|
|
1778
|
+
const rows = Math.max(1, Math.round(Math.sqrt(32 / aspect)))
|
|
1779
|
+
const cols = Math.max(1, Math.round(32 / rows))
|
|
1780
|
+
const small = fitEdge(pic, 128)
|
|
1781
|
+
const L = new Float64Array(small.width * small.height)
|
|
1782
|
+
for (let i = 0; i < L.length; i++) L[i] = srgbToLab(small.data[i * 4], small.data[i * 4 + 1], small.data[i * 4 + 2])[0]
|
|
1783
|
+
return gridMeans(L, small.width, small.height, rows, cols).flat()
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
function correlation(a, b) {
|
|
1787
|
+
const n = a.length
|
|
1788
|
+
const ma = a.reduce((s, v) => s + v, 0) / n
|
|
1789
|
+
const mb = b.reduce((s, v) => s + v, 0) / n
|
|
1790
|
+
let sab = 0
|
|
1791
|
+
let saa = 0
|
|
1792
|
+
let sbb = 0
|
|
1793
|
+
for (let i = 0; i < n; i++) {
|
|
1794
|
+
sab += (a[i] - ma) * (b[i] - mb)
|
|
1795
|
+
saa += (a[i] - ma) ** 2
|
|
1796
|
+
sbb += (b[i] - mb) ** 2
|
|
1797
|
+
}
|
|
1798
|
+
if (saa < 1e-9 && sbb < 1e-9) return a.every((v, i) => Math.abs(v - b[i]) < 0.5) ? 1 : 0
|
|
1799
|
+
if (saa < 1e-9 || sbb < 1e-9) return 0
|
|
1800
|
+
return sab / Math.sqrt(saa * sbb)
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
const meanOf = (grid) => {
|
|
1804
|
+
const flat = grid.flat()
|
|
1805
|
+
return flat.reduce((s, v) => s + v, 0) / flat.length
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
/**
|
|
1809
|
+
* How far a shot of the game is from the reference (orchestrator spec 6.2):
|
|
1810
|
+
* the colour comparison look_compare makes, the structure (the L* layout,
|
|
1811
|
+
* the horizon, where the eye goes, edges and detail), one distance from 0 to
|
|
1812
|
+
* 1 for telling progress from stagnation, and the gross flags. The shot is
|
|
1813
|
+
* cut to the reference's aspect first, and the reference's lettering boxes
|
|
1814
|
+
* are masked in both and left out of both pictures' structure measures.
|
|
1815
|
+
* Structure and detail flags are raised only at a camera whose purpose is
|
|
1816
|
+
* `reference`: another camera is not meant to frame the same picture.
|
|
1817
|
+
*/
|
|
1818
|
+
export function compareMetrics(shotPic, refPic, { camera = 'shot', purpose = 'reference', refFacts = null, shotFacts = null } = {}) {
|
|
1819
|
+
const aspect = refPic.width / refPic.height
|
|
1820
|
+
const ref = refFacts ?? referenceFacts(refPic)
|
|
1821
|
+
// The reference's lettering is masked in both: what is under it is not
|
|
1822
|
+
// known, so neither picture is measured there.
|
|
1823
|
+
const boxes = (ref.text?.boxes ?? []).map((b) => b.box)
|
|
1824
|
+
const framed = fitAspect(shotPic, aspect)
|
|
1825
|
+
const shot = maskBoxes(framed, boxes)
|
|
1826
|
+
const art = maskBoxes(refPic, boxes)
|
|
1827
|
+
const own = shotFacts ?? referenceFacts(framed, { lettering: false, mask: boxes })
|
|
1828
|
+
const shotSmall = fitEdge(shot, COLOUR_EDGE)
|
|
1829
|
+
const refSmall = fitEdge(art, COLOUR_EDGE)
|
|
1830
|
+
const colour = compareLook(lookStats(shotSmall.data, shotSmall.width, shotSmall.height), lookStats(refSmall.data, refSmall.width, refSmall.height))
|
|
1831
|
+
const corr = correlation(lGrid(shot, aspect), lGrid(art, aspect))
|
|
1832
|
+
const edgeRatio = (meanOf(own.composition.edgeGrid) + 0.05) / (meanOf(ref.composition.edgeGrid) + 0.05)
|
|
1833
|
+
const detailRatio = (own.contrast.local + 0.1) / (ref.contrast.local + 0.1)
|
|
1834
|
+
const horizonDelta = own.horizon.y !== null && ref.horizon.y !== null ? own.horizon.y - ref.horizon.y : null
|
|
1835
|
+
const centroidDelta = Math.hypot(own.composition.centroid[0] - ref.composition.centroid[0], own.composition.centroid[1] - ref.composition.centroid[1])
|
|
1836
|
+
const distance =
|
|
1837
|
+
0.3 * Math.min(1, colour.paletteDeltaE / 30) +
|
|
1838
|
+
0.2 * Math.min(1, Math.abs(colour.deltaMeanL) / 40) +
|
|
1839
|
+
0.3 * (1 - Math.max(0, corr)) +
|
|
1840
|
+
0.1 * Math.min(1, Math.abs(Math.log2(edgeRatio)) / 2) +
|
|
1841
|
+
0.1 * Math.min(1, centroidDelta / 0.5)
|
|
1842
|
+
const KEYS = ['low', 'mid', 'high']
|
|
1843
|
+
const flags = []
|
|
1844
|
+
if (colour.paletteDeltaE > 20) flags.push('palette-far')
|
|
1845
|
+
if (Math.abs(colour.deltaMeanL) > 20 || Math.abs(KEYS.indexOf(own.value.key) - KEYS.indexOf(ref.value.key)) >= 2) flags.push('value-far')
|
|
1846
|
+
if (purpose === 'reference') {
|
|
1847
|
+
if (corr < 0.15) flags.push('structure-far')
|
|
1848
|
+
if (edgeRatio < 0.35 || edgeRatio > 3) flags.push('detail-far')
|
|
1849
|
+
}
|
|
1850
|
+
const r4 = (v) => round(v, 4)
|
|
1851
|
+
return {
|
|
1852
|
+
camera,
|
|
1853
|
+
colour: {
|
|
1854
|
+
...colour,
|
|
1855
|
+
paletteDeltaE: r4(colour.paletteDeltaE),
|
|
1856
|
+
deltaMeanL: r4(colour.deltaMeanL),
|
|
1857
|
+
contrastRatio: r4(colour.contrastRatio),
|
|
1858
|
+
chromaRatio: r4(colour.chromaRatio),
|
|
1859
|
+
warmthDelta: r4(colour.warmthDelta),
|
|
1860
|
+
tintDelta: r4(colour.tintDelta),
|
|
1861
|
+
clip: { high: colour.clip.high.map(r4), low: colour.clip.low.map(r4) },
|
|
1862
|
+
thirds: { shot: colour.thirds.shot.map((v) => round(v, 2)), ref: colour.thirds.ref.map((v) => round(v, 2)) },
|
|
1863
|
+
},
|
|
1864
|
+
structure: {
|
|
1865
|
+
correlation: r4(corr),
|
|
1866
|
+
horizonDelta: horizonDelta === null ? null : r4(horizonDelta),
|
|
1867
|
+
centroidDelta: r4(centroidDelta),
|
|
1868
|
+
edgeRatio: r4(edgeRatio),
|
|
1869
|
+
detailRatio: r4(detailRatio),
|
|
1870
|
+
},
|
|
1871
|
+
distance: r4(distance),
|
|
1872
|
+
flags,
|
|
1873
|
+
}
|
|
1874
|
+
}
|