imgtune 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +69 -6
- package/dist/audit.js +1 -0
- package/dist/backgrounds.js +31 -7
- package/dist/broken.js +5 -1
- package/dist/browser.js +4 -4
- package/dist/candidates.js +25 -0
- package/dist/check-report.js +37 -4
- package/dist/check.js +18 -1
- package/dist/cli.js +20 -0
- package/dist/egress-proxy.js +2 -1
- package/dist/ffmpeg.js +35 -0
- package/dist/hints.js +294 -18
- package/dist/image-open.js +9 -0
- package/dist/images.js +116 -7
- package/dist/layout.js +7 -7
- package/dist/lcp.js +2 -1
- package/dist/measure.js +39 -1
- package/dist/media-encode.js +5 -1
- package/dist/media-model.js +8 -4
- package/dist/media-phase.js +25 -2
- package/dist/px.js +11 -0
- package/dist/recheck.js +9 -1
- package/dist/remote.js +616 -63
- package/dist/replay.js +20 -0
- package/dist/resources/audio.js +248 -0
- package/dist/resources/brotli.js +22 -0
- package/dist/resources/cache.js +59 -0
- package/dist/resources/capture.js +225 -0
- package/dist/resources/compression.js +122 -0
- package/dist/resources/favicon.js +130 -0
- package/dist/resources/font.js +440 -0
- package/dist/resources/index.js +8 -0
- package/dist/resources/lottie.js +491 -0
- package/dist/resources/model.js +478 -0
- package/dist/resources/phase.js +171 -0
- package/dist/resources/pool.js +47 -0
- package/dist/resources/report.js +57 -0
- package/dist/resources/sfnt.js +305 -0
- package/dist/resources/svg.js +178 -0
- package/dist/resources/types.js +1 -0
- package/dist/resources/zip.js +67 -0
- package/dist/run.js +24 -4
- package/dist/sizes.js +7 -5
- package/dist/summary.js +47 -1
- package/dist/vector-art.js +294 -0
- package/dist/vector-phase.js +222 -0
- package/dist/video-analysis.js +119 -21
- package/package.json +25 -4
- package/skills/imgtune/SKILL.md +8 -4
- package/skills/imgtune/reference.md +1 -1
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
// 3D model analyzer: the model files a page loads (glTF, GLB, USDZ, KTX2, Draco, OBJ, FBX, splats, PLY). A glTF or GLB is
|
|
2
|
+
// parsed for its geometry, its textures and its extensions. The geometry saving is measured: the file is repacked and
|
|
3
|
+
// run through meshopt with gltf-transform (the same transform as `--compress meshopt`), so the bytes are real, not
|
|
4
|
+
// estimated. Each PNG or JPEG texture gets a WebP (q80) re-encode with sharp, an estimate for textures over 4 MP. Other
|
|
5
|
+
// formats are informational. It never throws.
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import sharp from "sharp";
|
|
8
|
+
import { digestOf, heavyValue } from "./cache.js";
|
|
9
|
+
const GLTF_EXTS = new Set(["glb", "gltf"]);
|
|
10
|
+
const OTHER_EXTS = new Set(["usdz", "ktx2", "drc", "obj", "fbx", "splat", "ply"]);
|
|
11
|
+
const MODEL_TYPES = /^model\//;
|
|
12
|
+
/** Geometry bigger than this is not run through meshopt within the budget; it is reported without a measured saving. */
|
|
13
|
+
export const MAX_MEASURED_GEOMETRY_BYTES = 12 * 1024 * 1024;
|
|
14
|
+
/** Textures over this many pixels are re-encoded at 2048 px on the long side and scaled up (an estimate). */
|
|
15
|
+
export const TEXTURE_FULL_PIXELS = 4_000_000;
|
|
16
|
+
/** The WebP quality the texture estimate is made at. */
|
|
17
|
+
export const TEXTURE_WEBP_QUALITY = 80;
|
|
18
|
+
/** Stop re-encoding textures once this much time has passed in one page view, in ms. */
|
|
19
|
+
const TEXTURE_BUDGET_MS = 4_000;
|
|
20
|
+
/** A saving is reported only when it is at least this much, absolutely and as a share of the bytes per view. */
|
|
21
|
+
const MIN_SAVING_BYTES = 5 * 1024;
|
|
22
|
+
const MIN_SAVING_SHARE = 0.05;
|
|
23
|
+
function hashUrl(url) {
|
|
24
|
+
return createHash("sha1").update(url).digest("hex").slice(0, 12);
|
|
25
|
+
}
|
|
26
|
+
function extOf(url) {
|
|
27
|
+
try {
|
|
28
|
+
const path = new URL(url).pathname;
|
|
29
|
+
const dot = path.lastIndexOf(".");
|
|
30
|
+
return dot < 0 ? "" : path.slice(dot + 1).toLowerCase();
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return "";
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function fileName(url) {
|
|
37
|
+
try {
|
|
38
|
+
return decodeURIComponent(new URL(url).pathname.split("/").pop() ?? "") || "model";
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return "model";
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function formatBytes(n) {
|
|
45
|
+
if (n < 1024)
|
|
46
|
+
return `${Math.round(n)} B`;
|
|
47
|
+
if (n < 1024 * 1024)
|
|
48
|
+
return `${Math.round(n / 1024)} KB`;
|
|
49
|
+
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
|
50
|
+
}
|
|
51
|
+
/** Reads a GLB (container of a JSON chunk and a BIN chunk) or a glTF JSON document. Undefined when it is neither. */
|
|
52
|
+
export function parseGltf(body) {
|
|
53
|
+
try {
|
|
54
|
+
if (body.length >= 20 && body.subarray(0, 4).toString("ascii") === "glTF") {
|
|
55
|
+
const jsonLen = body.readUInt32LE(12);
|
|
56
|
+
if (body.subarray(16, 20).toString("ascii") !== "JSON")
|
|
57
|
+
return undefined;
|
|
58
|
+
const json = JSON.parse(body.subarray(20, 20 + jsonLen).toString("utf8"));
|
|
59
|
+
let bin;
|
|
60
|
+
const binHead = 20 + jsonLen;
|
|
61
|
+
if (binHead + 8 <= body.length && body.subarray(binHead + 4, binHead + 8).toString("ascii") === "BIN\0") {
|
|
62
|
+
const binLen = body.readUInt32LE(binHead);
|
|
63
|
+
bin = body.subarray(binHead + 8, binHead + 8 + binLen);
|
|
64
|
+
}
|
|
65
|
+
return { json, bin, jsonBytes: jsonLen };
|
|
66
|
+
}
|
|
67
|
+
const json = JSON.parse(body.toString("utf8"));
|
|
68
|
+
if (typeof json !== "object" || json === null || !("asset" in json))
|
|
69
|
+
return undefined;
|
|
70
|
+
return { json, jsonBytes: body.length };
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/** A data: URI's decoded bytes and the base64 overhead (encoded length minus decoded length), or undefined. */
|
|
77
|
+
function decodeDataUri(uri) {
|
|
78
|
+
if (!uri.startsWith("data:"))
|
|
79
|
+
return undefined;
|
|
80
|
+
const comma = uri.indexOf(",");
|
|
81
|
+
if (comma < 0 || !/;base64$/i.test(uri.slice(0, comma)))
|
|
82
|
+
return undefined;
|
|
83
|
+
const data = Buffer.from(uri.slice(comma + 1), "base64");
|
|
84
|
+
const mime = uri.slice(5, uri.indexOf(";") > 0 ? uri.indexOf(";") : comma).toLowerCase();
|
|
85
|
+
return { data, mime, overhead: uri.length - comma - 1 - data.length };
|
|
86
|
+
}
|
|
87
|
+
/** The bytes of one glTF buffer, from the GLB BIN chunk or a data URI. Undefined for an external file. */
|
|
88
|
+
function bufferBytes(parsed, index) {
|
|
89
|
+
const b = parsed.json.buffers?.[index];
|
|
90
|
+
if (!b)
|
|
91
|
+
return undefined;
|
|
92
|
+
if (b.uri === undefined)
|
|
93
|
+
return index === 0 ? parsed.bin : undefined;
|
|
94
|
+
return decodeDataUri(b.uri)?.data;
|
|
95
|
+
}
|
|
96
|
+
/** The bytes a bufferView covers. Undefined when its buffer is external or not in the file. */
|
|
97
|
+
function viewBytes(parsed, viewIndex) {
|
|
98
|
+
const view = parsed.json.bufferViews?.[viewIndex];
|
|
99
|
+
if (!view)
|
|
100
|
+
return undefined;
|
|
101
|
+
const buf = bufferBytes(parsed, view.buffer);
|
|
102
|
+
if (!buf)
|
|
103
|
+
return undefined;
|
|
104
|
+
const start = view.byteOffset ?? 0;
|
|
105
|
+
return buf.subarray(start, start + view.byteLength);
|
|
106
|
+
}
|
|
107
|
+
/** The bufferViews the meshes read (vertex attributes, morph targets, indices). */
|
|
108
|
+
function geometryViews(json) {
|
|
109
|
+
const views = new Set();
|
|
110
|
+
const accessorViews = (accessor) => {
|
|
111
|
+
if (accessor === undefined)
|
|
112
|
+
return;
|
|
113
|
+
const view = json.accessors?.[accessor]?.bufferView;
|
|
114
|
+
if (view !== undefined)
|
|
115
|
+
views.add(view);
|
|
116
|
+
};
|
|
117
|
+
for (const mesh of json.meshes ?? []) {
|
|
118
|
+
for (const prim of mesh.primitives ?? []) {
|
|
119
|
+
for (const acc of Object.values(prim.attributes ?? {}))
|
|
120
|
+
accessorViews(acc);
|
|
121
|
+
accessorViews(prim.indices);
|
|
122
|
+
for (const target of prim.targets ?? [])
|
|
123
|
+
for (const acc of Object.values(target))
|
|
124
|
+
accessorViews(acc);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return views;
|
|
128
|
+
}
|
|
129
|
+
function sumViews(json, views) {
|
|
130
|
+
let total = 0;
|
|
131
|
+
for (const v of views)
|
|
132
|
+
total += json.bufferViews?.[v]?.byteLength ?? 0;
|
|
133
|
+
return total;
|
|
134
|
+
}
|
|
135
|
+
/** The images the textures use, with their bytes when the file holds them. */
|
|
136
|
+
function textureImages(parsed) {
|
|
137
|
+
const out = [];
|
|
138
|
+
const images = parsed.json.images ?? [];
|
|
139
|
+
images.forEach((img, index) => {
|
|
140
|
+
if (img.bufferView !== undefined) {
|
|
141
|
+
const data = viewBytes(parsed, img.bufferView);
|
|
142
|
+
out.push({ index, mime: img.mimeType ?? "", bytes: data?.length, data });
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (img.uri !== undefined) {
|
|
146
|
+
const decoded = decodeDataUri(img.uri);
|
|
147
|
+
if (decoded)
|
|
148
|
+
out.push({ index, mime: img.mimeType ?? decoded.mime, bytes: decoded.data.length, data: decoded.data });
|
|
149
|
+
else
|
|
150
|
+
out.push({ index, mime: img.mimeType ?? "" });
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
return out;
|
|
154
|
+
}
|
|
155
|
+
/** The WebP (q80) size of a PNG or JPEG texture: measured at full size, or at 2048 px and scaled (an estimate). */
|
|
156
|
+
async function webpBytes(data) {
|
|
157
|
+
const meta = await sharp(data, { limitInputPixels: 100_000_000 }).metadata();
|
|
158
|
+
const width = meta.width ?? 0;
|
|
159
|
+
const height = meta.height ?? 0;
|
|
160
|
+
if (width <= 0 || height <= 0)
|
|
161
|
+
return undefined;
|
|
162
|
+
const pixels = width * height;
|
|
163
|
+
if (pixels <= TEXTURE_FULL_PIXELS) {
|
|
164
|
+
const out = await sharp(data, { limitInputPixels: 100_000_000 }).webp({ quality: TEXTURE_WEBP_QUALITY }).toBuffer();
|
|
165
|
+
return { bytes: out.length, estimate: false, width, height };
|
|
166
|
+
}
|
|
167
|
+
const long = Math.max(width, height);
|
|
168
|
+
const scale = 2048 / long;
|
|
169
|
+
const small = await sharp(data, { limitInputPixels: 100_000_000 })
|
|
170
|
+
.resize({ width: Math.round(width * scale), height: Math.round(height * scale) })
|
|
171
|
+
.webp({ quality: TEXTURE_WEBP_QUALITY })
|
|
172
|
+
.toBuffer();
|
|
173
|
+
const ratio = (Math.round(width * scale) * Math.round(height * scale)) / pixels;
|
|
174
|
+
return { bytes: Math.round(small.length / ratio), estimate: true, width, height };
|
|
175
|
+
}
|
|
176
|
+
/** The note for a missing measurement package: what installing it adds (printed with the model findings). */
|
|
177
|
+
export const GLTF_NOTE = "install @gltf-transform/core, @gltf-transform/extensions, @gltf-transform/functions and meshoptimizer to measure 3D compression";
|
|
178
|
+
async function loadGltfPackages() {
|
|
179
|
+
const [core, extensions, functions, meshoptimizer] = await Promise.all([
|
|
180
|
+
import("@gltf-transform/core"),
|
|
181
|
+
import("@gltf-transform/extensions"),
|
|
182
|
+
import("@gltf-transform/functions"),
|
|
183
|
+
import("meshoptimizer"),
|
|
184
|
+
]);
|
|
185
|
+
return { Logger: core.Logger, NodeIO: core.NodeIO, EXTMeshoptCompression: extensions.EXTMeshoptCompression, meshopt: functions.meshopt, MeshoptEncoder: meshoptimizer.MeshoptEncoder };
|
|
186
|
+
}
|
|
187
|
+
let gltfTools;
|
|
188
|
+
/**
|
|
189
|
+
* The measurement packages, loaded once (dynamic import, so a missing optional dependency costs nothing until a model is
|
|
190
|
+
* seen). Undefined when one will not load. Never throws. `load` is the loader (tests pass a failing one).
|
|
191
|
+
*/
|
|
192
|
+
export function importGltfTools(load = loadGltfPackages) {
|
|
193
|
+
if (load !== loadGltfPackages)
|
|
194
|
+
return load().then((tools) => tools, () => undefined);
|
|
195
|
+
gltfTools ??= loadGltfPackages().then((tools) => tools, () => undefined);
|
|
196
|
+
return gltfTools;
|
|
197
|
+
}
|
|
198
|
+
/** The repacked (GLB) and meshopt-compressed sizes of a glTF, from gltf-transform (see MeshoptOutcome). Never throws. */
|
|
199
|
+
async function measureMeshopt(body, parsed) {
|
|
200
|
+
const tools = await importGltfTools();
|
|
201
|
+
if (tools === undefined)
|
|
202
|
+
return { kind: "missing" };
|
|
203
|
+
try {
|
|
204
|
+
await tools.MeshoptEncoder.ready;
|
|
205
|
+
const io = new tools.NodeIO()
|
|
206
|
+
.setLogger(new tools.Logger(tools.Logger.Verbosity.ERROR))
|
|
207
|
+
.registerExtensions([tools.EXTMeshoptCompression])
|
|
208
|
+
.registerDependencies({ "meshopt.encoder": tools.MeshoptEncoder });
|
|
209
|
+
let doc;
|
|
210
|
+
if (body.subarray(0, 4).toString("ascii") === "glTF") {
|
|
211
|
+
doc = await io.readBinary(new Uint8Array(body));
|
|
212
|
+
}
|
|
213
|
+
else {
|
|
214
|
+
doc = await io.readJSON({ json: parsed.json, resources: {} });
|
|
215
|
+
}
|
|
216
|
+
const repackBytes = (await io.writeBinary(doc)).byteLength;
|
|
217
|
+
await doc.transform(tools.meshopt({ encoder: tools.MeshoptEncoder, level: "medium" }));
|
|
218
|
+
const meshBytes = (await io.writeBinary(doc)).byteLength;
|
|
219
|
+
return { kind: "ok", repackBytes, meshBytes };
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
return { kind: "unreadable" };
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
function displayBox(ctx, url) {
|
|
226
|
+
const el = ctx.models.find((m) => m.src === url);
|
|
227
|
+
return el === undefined ? undefined : { selector: el.selector, width: el.width, height: el.height };
|
|
228
|
+
}
|
|
229
|
+
async function analyzeGltf(r, body, ctx) {
|
|
230
|
+
const parsed = parseGltf(body);
|
|
231
|
+
if (!parsed)
|
|
232
|
+
return [];
|
|
233
|
+
const name = fileName(r.url);
|
|
234
|
+
const fmt = extOf(r.url) === "gltf" || !parsed.bin ? "glTF" : "GLB";
|
|
235
|
+
const id = `model:${hashUrl(r.url)}`;
|
|
236
|
+
const box = displayBox(ctx, r.url);
|
|
237
|
+
const locator = box === undefined ? undefined : { selector: box.selector };
|
|
238
|
+
const extensions = parsed.json.extensionsUsed ?? [];
|
|
239
|
+
const compressed = extensions.includes("KHR_draco_mesh_compression") || extensions.includes("EXT_meshopt_compression");
|
|
240
|
+
const geomViews = geometryViews(parsed.json);
|
|
241
|
+
const geomBytes = sumViews(parsed.json, geomViews);
|
|
242
|
+
const externalBuffers = (parsed.json.buffers ?? []).some((b, i) => b.uri !== undefined && bufferBytes(parsed, i) === undefined);
|
|
243
|
+
const details = {
|
|
244
|
+
format: fmt,
|
|
245
|
+
extensionsUsed: extensions,
|
|
246
|
+
geometryBytes: geomBytes,
|
|
247
|
+
meshes: parsed.json.meshes?.length ?? 0,
|
|
248
|
+
animations: parsed.json.animations?.length ?? 0,
|
|
249
|
+
externalBuffers,
|
|
250
|
+
};
|
|
251
|
+
const findings = [];
|
|
252
|
+
// Textures.
|
|
253
|
+
const images = textureImages(parsed);
|
|
254
|
+
const textures = parsed.json.textures?.length ?? 0;
|
|
255
|
+
const texDetails = [];
|
|
256
|
+
let texSaved = 0;
|
|
257
|
+
let texBytes = 0;
|
|
258
|
+
let texWebp = 0;
|
|
259
|
+
const startedAt = Date.now();
|
|
260
|
+
let unmeasured = 0;
|
|
261
|
+
for (const img of images) {
|
|
262
|
+
if (img.data === undefined || img.bytes === undefined)
|
|
263
|
+
continue;
|
|
264
|
+
texBytes += img.bytes;
|
|
265
|
+
const isRaster = /^image\/(png|jpeg)$/.test(img.mime) || (img.mime === "" && /^\x89PNG|^\xff\xd8/.test(img.data.subarray(0, 8).toString("latin1")));
|
|
266
|
+
if (!isRaster)
|
|
267
|
+
continue;
|
|
268
|
+
if (Date.now() - startedAt > TEXTURE_BUDGET_MS) {
|
|
269
|
+
unmeasured++;
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
try {
|
|
273
|
+
const w = await webpBytes(img.data);
|
|
274
|
+
if (w === undefined)
|
|
275
|
+
continue;
|
|
276
|
+
texWebp += w.bytes;
|
|
277
|
+
texSaved += Math.max(0, img.bytes - w.bytes);
|
|
278
|
+
texDetails.push({ image: img.index, mime: img.mime, bytes: img.bytes, width: w.width, height: w.height, webpBytes: w.bytes, estimate: w.estimate });
|
|
279
|
+
}
|
|
280
|
+
catch {
|
|
281
|
+
unmeasured++;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
const longestTexture = Math.max(0, ...texDetails.map((t) => Math.max(Number(t.width), Number(t.height))));
|
|
285
|
+
const resizeTo = box === undefined ? undefined : Math.max(box.width, box.height) * ctx.viewport.dpr;
|
|
286
|
+
const texResize = resizeTo !== undefined && longestTexture > 2 * resizeTo;
|
|
287
|
+
details.textures = textures;
|
|
288
|
+
details.textureBytes = texBytes;
|
|
289
|
+
details.textureDetails = texDetails;
|
|
290
|
+
if (unmeasured > 0)
|
|
291
|
+
details.texturesNotMeasured = unmeasured;
|
|
292
|
+
if (texSaved >= MIN_SAVING_BYTES || (texResize && texBytes > 0)) {
|
|
293
|
+
const fix = [`textures: npx @gltf-transform/cli optimize in.glb out.glb --texture-compress webp (WebP q${TEXTURE_WEBP_QUALITY})`];
|
|
294
|
+
if (texSaved >= MIN_SAVING_BYTES) {
|
|
295
|
+
fix.push(`expected ~${formatBytes(texWebp)} for the textures, from ${formatBytes(texBytes)} (estimate)`);
|
|
296
|
+
}
|
|
297
|
+
if (texResize && resizeTo !== undefined) {
|
|
298
|
+
fix.push(`resize textures to at most ${Math.round(resizeTo)} px on the long side, keeping the aspect ratio ` +
|
|
299
|
+
`(the model shows at ${Math.round(box.width)}x${Math.round(box.height)} CSS px at ${ctx.viewport.dpr}x)`);
|
|
300
|
+
}
|
|
301
|
+
findings.push({
|
|
302
|
+
id: `${id}:textures`,
|
|
303
|
+
kind: "model",
|
|
304
|
+
url: r.url,
|
|
305
|
+
bytes: r.bytes,
|
|
306
|
+
savedBytes: texSaved,
|
|
307
|
+
estimate: true,
|
|
308
|
+
title: `3D model ${name}: ${textures} texture${textures === 1 ? "" : "s"}, ${formatBytes(texBytes)} as PNG/JPEG` +
|
|
309
|
+
(texSaved >= MIN_SAVING_BYTES ? `; WebP ~${formatBytes(texWebp)}` : ""),
|
|
310
|
+
fix,
|
|
311
|
+
details: { ...details, savedBytes: texSaved },
|
|
312
|
+
...(locator ? { locator } : {}),
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
// Geometry.
|
|
316
|
+
if (compressed) {
|
|
317
|
+
findings.push({
|
|
318
|
+
id,
|
|
319
|
+
kind: "model",
|
|
320
|
+
url: r.url,
|
|
321
|
+
bytes: r.bytes,
|
|
322
|
+
savedBytes: 0,
|
|
323
|
+
estimate: false,
|
|
324
|
+
title: `3D model ${name}: already compressed (${extensions.filter((e) => /draco|meshopt/i.test(e)).join(", ")}), geometry not re-measured`,
|
|
325
|
+
fix: [],
|
|
326
|
+
details,
|
|
327
|
+
...(locator ? { locator } : {}),
|
|
328
|
+
});
|
|
329
|
+
return findings;
|
|
330
|
+
}
|
|
331
|
+
if (externalBuffers) {
|
|
332
|
+
findings.push({
|
|
333
|
+
id,
|
|
334
|
+
kind: "model",
|
|
335
|
+
url: r.url,
|
|
336
|
+
bytes: r.bytes,
|
|
337
|
+
savedBytes: 0,
|
|
338
|
+
estimate: true,
|
|
339
|
+
title: `3D model ${name} (${fmt}): ${formatBytes(r.bytes)}; its .bin buffers are external and not in the scan, so geometry is not measured`,
|
|
340
|
+
fix: ["measure: npx @gltf-transform/cli optimize in.gltf out.glb --compress meshopt (run it on the source file)"],
|
|
341
|
+
details,
|
|
342
|
+
...(locator ? { locator } : {}),
|
|
343
|
+
});
|
|
344
|
+
return findings;
|
|
345
|
+
}
|
|
346
|
+
if (geomBytes > MAX_MEASURED_GEOMETRY_BYTES) {
|
|
347
|
+
findings.push({
|
|
348
|
+
id,
|
|
349
|
+
kind: "model",
|
|
350
|
+
url: r.url,
|
|
351
|
+
bytes: r.bytes,
|
|
352
|
+
savedBytes: 0,
|
|
353
|
+
estimate: true,
|
|
354
|
+
title: `3D model ${name} (${fmt}): ${formatBytes(geomBytes)} of geometry, too large to measure within the budget`,
|
|
355
|
+
fix: ["npx @gltf-transform/cli optimize in.glb out.glb --compress meshopt (meshopt quantizes positions and normals: check the model visually)"],
|
|
356
|
+
details,
|
|
357
|
+
...(locator ? { locator } : {}),
|
|
358
|
+
});
|
|
359
|
+
return findings;
|
|
360
|
+
}
|
|
361
|
+
// The meshopt measurement is heavy: once per run by content, and not at all once the run's budget is spent.
|
|
362
|
+
const outcome = await heavyValue(`model:meshopt:${digestOf(body)}`, () => measureMeshopt(body, parsed));
|
|
363
|
+
if (outcome === undefined || outcome.kind !== "ok") {
|
|
364
|
+
// No measurement: the model is reported with no number, and the reason (a missing package prints its one note).
|
|
365
|
+
if (outcome?.kind === "unreadable")
|
|
366
|
+
return findings;
|
|
367
|
+
const reason = outcome === undefined ? "over the run's --resource-time budget" : GLTF_NOTE;
|
|
368
|
+
findings.push({
|
|
369
|
+
id: `${id}:geometry`,
|
|
370
|
+
kind: "model",
|
|
371
|
+
url: r.url,
|
|
372
|
+
bytes: r.bytes,
|
|
373
|
+
savedBytes: 0,
|
|
374
|
+
estimate: true,
|
|
375
|
+
title: `3D model ${name} (${fmt}): geometry not measured (${reason})`,
|
|
376
|
+
fix: [outcome === undefined ? "npx @gltf-transform/cli optimize in.glb out.glb --compress meshopt (run it on the source file)" : GLTF_NOTE],
|
|
377
|
+
details: { ...details, ...(outcome?.kind === "missing" ? { note: GLTF_NOTE } : {}) },
|
|
378
|
+
...(locator ? { locator } : {}),
|
|
379
|
+
});
|
|
380
|
+
return findings;
|
|
381
|
+
}
|
|
382
|
+
const measured = outcome;
|
|
383
|
+
const savedBytes = body.length - measured.meshBytes;
|
|
384
|
+
details.repackBytes = measured.repackBytes;
|
|
385
|
+
details.meshoptBytes = measured.meshBytes;
|
|
386
|
+
details.repackSavedBytes = body.length - measured.repackBytes;
|
|
387
|
+
if (savedBytes < MIN_SAVING_BYTES || savedBytes < MIN_SAVING_SHARE * body.length)
|
|
388
|
+
return findings;
|
|
389
|
+
const fix = [];
|
|
390
|
+
if (details.repackSavedBytes >= MIN_SAVING_BYTES) {
|
|
391
|
+
fix.push(`the file embeds its buffers as base64 (about a third larger): writing it as a GLB removes ` +
|
|
392
|
+
`${formatBytes(details.repackSavedBytes)} on its own`);
|
|
393
|
+
}
|
|
394
|
+
fix.push(`npx @gltf-transform/cli optimize in.glb out.glb --compress meshopt`);
|
|
395
|
+
const combined = measured.meshBytes - (texSaved >= MIN_SAVING_BYTES ? texSaved : 0);
|
|
396
|
+
fix.push(`expected ~${formatBytes(measured.meshBytes)} for the file as it is (measured; meshopt quantizes positions and normals, ` +
|
|
397
|
+
`check the model visually)` +
|
|
398
|
+
(texSaved >= MIN_SAVING_BYTES ? `; with the texture fix above ~${formatBytes(combined)}` : ""));
|
|
399
|
+
findings.push({
|
|
400
|
+
id,
|
|
401
|
+
kind: "model",
|
|
402
|
+
url: r.url,
|
|
403
|
+
bytes: r.bytes,
|
|
404
|
+
savedBytes,
|
|
405
|
+
estimate: false,
|
|
406
|
+
title: `3D model ${name} (${fmt}): ${formatBytes(body.length)}; with meshopt ~${formatBytes(measured.meshBytes)} (measured)`,
|
|
407
|
+
fix,
|
|
408
|
+
details,
|
|
409
|
+
...(locator ? { locator } : {}),
|
|
410
|
+
});
|
|
411
|
+
return findings;
|
|
412
|
+
}
|
|
413
|
+
function otherFormat(r, ext) {
|
|
414
|
+
const name = fileName(r.url);
|
|
415
|
+
const fix = {
|
|
416
|
+
usdz: "convert to .glb (Blender: export glTF 2.0 binary), then npx @gltf-transform/cli optimize in.glb out.glb --compress meshopt --texture-compress webp",
|
|
417
|
+
obj: "convert to .glb (Blender: export glTF 2.0 binary), then npx @gltf-transform/cli optimize in.glb out.glb --compress meshopt --texture-compress webp",
|
|
418
|
+
fbx: "convert to .glb (Blender: export glTF 2.0 binary), then npx @gltf-transform/cli optimize in.glb out.glb --compress meshopt --texture-compress webp",
|
|
419
|
+
ply: "a point cloud or splat: serve a compressed .glb or a splat-specific compressed format; the saving is not measured here",
|
|
420
|
+
splat: "a Gaussian splat is large by design; serve a splat-specific compressed format (not measured here)",
|
|
421
|
+
ktx2: "already a GPU texture container: nothing to re-encode here",
|
|
422
|
+
drc: "standalone Draco file: the decoder is part of the page's cost; a glTF with KHR_draco_mesh_compression keeps the same bytes",
|
|
423
|
+
};
|
|
424
|
+
return {
|
|
425
|
+
id: `model:${hashUrl(r.url)}`,
|
|
426
|
+
kind: "model",
|
|
427
|
+
url: r.url,
|
|
428
|
+
bytes: r.bytes,
|
|
429
|
+
savedBytes: 0,
|
|
430
|
+
estimate: true,
|
|
431
|
+
title: `3D model ${name} (${ext}): ${formatBytes(r.bytes)}; informational, not converted`,
|
|
432
|
+
fix: [fix[ext] ?? "informational"],
|
|
433
|
+
details: { format: ext },
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
export const analyzer = {
|
|
437
|
+
kind: "model",
|
|
438
|
+
wants(r) {
|
|
439
|
+
try {
|
|
440
|
+
const ext = extOf(r.url);
|
|
441
|
+
return GLTF_EXTS.has(ext) || OTHER_EXTS.has(ext) || MODEL_TYPES.test(r.contentType);
|
|
442
|
+
}
|
|
443
|
+
catch {
|
|
444
|
+
return false;
|
|
445
|
+
}
|
|
446
|
+
},
|
|
447
|
+
async analyze(records, ctx) {
|
|
448
|
+
const out = [];
|
|
449
|
+
for (const r of records) {
|
|
450
|
+
try {
|
|
451
|
+
const ext = extOf(r.url);
|
|
452
|
+
if (GLTF_EXTS.has(ext) || (MODEL_TYPES.test(r.contentType) && (r.contentType.includes("gltf") || r.contentType.includes("glb")))) {
|
|
453
|
+
if (!r.body) {
|
|
454
|
+
out.push({
|
|
455
|
+
id: `model:${hashUrl(r.url)}`,
|
|
456
|
+
kind: "model",
|
|
457
|
+
url: r.url,
|
|
458
|
+
bytes: r.bytes,
|
|
459
|
+
savedBytes: 0,
|
|
460
|
+
estimate: true,
|
|
461
|
+
title: `3D model ${fileName(r.url)}: ${formatBytes(r.bytes)}; over 20 MB, not parsed`,
|
|
462
|
+
fix: [],
|
|
463
|
+
});
|
|
464
|
+
continue;
|
|
465
|
+
}
|
|
466
|
+
out.push(...(await analyzeGltf(r, r.body, ctx)));
|
|
467
|
+
}
|
|
468
|
+
else if (OTHER_EXTS.has(ext) || MODEL_TYPES.test(r.contentType)) {
|
|
469
|
+
out.push(otherFormat(r, ext || r.contentType));
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
catch {
|
|
473
|
+
// One unreadable model does not drop the others.
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
return out;
|
|
477
|
+
},
|
|
478
|
+
};
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { count, measure } from "../timing.js";
|
|
2
|
+
import { isFirstParty, isThirdPartyBlocked } from "../third-party.js";
|
|
3
|
+
import { newResourceRun, withResourceScope } from "./cache.js";
|
|
4
|
+
import { ANALYZERS } from "./index.js";
|
|
5
|
+
import { ICON_READ_MAX } from "./favicon.js";
|
|
6
|
+
import { completeFontFaces, fontFaceRulesOf, RESOURCE_BODY_MAX } from "./capture.js";
|
|
7
|
+
import { findingTier, LEAD_BYTES } from "./report.js";
|
|
8
|
+
/** The kinds that are weak for their cost: a small saving is a lead, not a headline (the spec: svg minify, lottie, favicon). */
|
|
9
|
+
const WEAK_KINDS = new Set(["svg", "lottie", "favicon"]);
|
|
10
|
+
/**
|
|
11
|
+
* Labels a finding by where its bytes come from: a file on another registrable domain than the page is a third-party
|
|
12
|
+
* finding (the vendor fixes it), and a small saving of a weak-for-cost kind is a lead. The title gets the prefix, the
|
|
13
|
+
* details the flags. Applied once per finding, when it enters the page's findings.
|
|
14
|
+
*/
|
|
15
|
+
export function labelFinding(finding, pageUrl) {
|
|
16
|
+
const thirdParty = finding.url !== "" && !isFirstParty(pageUrl, finding.url);
|
|
17
|
+
const lead = WEAK_KINDS.has(finding.kind) && finding.savedBytes > 0 && finding.savedBytes < LEAD_BYTES;
|
|
18
|
+
if (!thirdParty && !lead)
|
|
19
|
+
return finding;
|
|
20
|
+
const prefix = `${thirdParty ? "third-party (ask the vendor): " : ""}${lead ? "lead: " : ""}`;
|
|
21
|
+
return {
|
|
22
|
+
...finding,
|
|
23
|
+
title: `${prefix}${finding.title}`,
|
|
24
|
+
details: { ...finding.details, ...(thirdParty ? { thirdParty: true } : {}), ...(lead ? { lead: true } : {}) },
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* A response the capture kept no body for (its read timed out) is fetched again only up to this size: a larger one is not worth
|
|
29
|
+
* a second download in the phase; its analysis is left out, and its bytes are still counted from the response.
|
|
30
|
+
*/
|
|
31
|
+
export const REFETCH_MAX_BYTES = 1024 * 1024;
|
|
32
|
+
const EMPTY_CONTEXT = { text: "", usedFonts: [], fontFaces: [], icons: [], audio: [], lottie: [], models: [] };
|
|
33
|
+
/** The image type of a body by its first bytes, or undefined when it is not a known image. */
|
|
34
|
+
export function sniffImageType(body) {
|
|
35
|
+
if (body.subarray(0, 4).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47])))
|
|
36
|
+
return "image/png";
|
|
37
|
+
if (body.subarray(0, 4).equals(Buffer.from([0, 0, 1, 0])))
|
|
38
|
+
return "image/x-icon";
|
|
39
|
+
if (body[0] === 0xff && body[1] === 0xd8)
|
|
40
|
+
return "image/jpeg";
|
|
41
|
+
if (body.subarray(0, 4).toString("latin1") === "RIFF" && body.subarray(8, 12).toString("latin1") === "WEBP")
|
|
42
|
+
return "image/webp";
|
|
43
|
+
if (/<svg[\s>]/i.test(body.subarray(0, 4096).toString("utf8")))
|
|
44
|
+
return "image/svg+xml";
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The icon URLs the phase fetches: the page's declared icon links (whatever their names; the capture reads only icons
|
|
49
|
+
* named like favicons), or the site's /favicon.ico when none is declared. An icon whose body the capture kept is not
|
|
50
|
+
* fetched again.
|
|
51
|
+
*/
|
|
52
|
+
function iconUrlsToFetch(view, pageUrl) {
|
|
53
|
+
const have = new Set((view.resources ?? []).filter((r) => r.body !== undefined).map((r) => r.url));
|
|
54
|
+
const declared = (view.pageContext?.icons ?? []).map((icon) => icon.href);
|
|
55
|
+
const urls = declared.length > 0 ? declared : [new URL("/favicon.ico", pageUrl).href];
|
|
56
|
+
return [...new Set(urls)].filter((url) => !have.has(url) && /^https?:/i.test(url));
|
|
57
|
+
}
|
|
58
|
+
/** The findings of a page's views, from the registered analyzers (see ANALYZERS). */
|
|
59
|
+
export function analyzeResources(views, pageUrl, deps) {
|
|
60
|
+
return runPhaseWith(ANALYZERS, views, pageUrl, deps);
|
|
61
|
+
}
|
|
62
|
+
/** The phase over the given analyzers: the same merge for any list (the tests run one analyzer through it). */
|
|
63
|
+
export async function runPhaseWith(analyzers, views, pageUrl, deps) {
|
|
64
|
+
const run = deps.run ?? newResourceRun();
|
|
65
|
+
return withResourceScope(deps.cache, async () => {
|
|
66
|
+
// Icons are fetched once per run (a favicon is shared by the pages of a site), and the fetches of a view run at once.
|
|
67
|
+
const fetchIcon = (url) => {
|
|
68
|
+
const key = `resources:icon:${url}`;
|
|
69
|
+
let pending = run.memo.get(key);
|
|
70
|
+
if (pending === undefined) {
|
|
71
|
+
pending = measure("resources icon fetch", () => deps.fetcher.get(url, { maxBytes: ICON_READ_MAX, allowPrivate: deps.allowPrivate })).catch(() => undefined);
|
|
72
|
+
run.memo.set(key, pending);
|
|
73
|
+
}
|
|
74
|
+
return pending;
|
|
75
|
+
};
|
|
76
|
+
const byId = new Map();
|
|
77
|
+
// The viewports each finding was found in, by id: a finding marked perViewport "all" survives only when every view has it.
|
|
78
|
+
const foundIn = new Map();
|
|
79
|
+
// A body the capture did not keep (Chromium gives none for a range response, or a read timed out) is fetched again when
|
|
80
|
+
// an analyzer wants the response: once per URL, up to RESOURCE_BODY_MAX, and not when the third-party policy blocks it.
|
|
81
|
+
const bodies = new Map();
|
|
82
|
+
const bodyOf = (url) => {
|
|
83
|
+
let pending = bodies.get(url);
|
|
84
|
+
if (pending === undefined) {
|
|
85
|
+
pending = measure("resources refetch body", () => deps.fetcher.get(url, { maxBytes: RESOURCE_BODY_MAX, allowPrivate: deps.allowPrivate })).catch(() => undefined);
|
|
86
|
+
count("resources refetch bodies");
|
|
87
|
+
bodies.set(url, pending);
|
|
88
|
+
}
|
|
89
|
+
return pending;
|
|
90
|
+
};
|
|
91
|
+
for (const view of views) {
|
|
92
|
+
const records = [];
|
|
93
|
+
for (const r of view.resources ?? []) {
|
|
94
|
+
const needsBody = r.body === undefined && r.status >= 200 && r.status < 300 && r.bytes <= REFETCH_MAX_BYTES && analyzers.some((a) => a.wants(r));
|
|
95
|
+
const blocked = deps.thirdParty !== undefined && isThirdPartyBlocked(deps.thirdParty, { pageUrl: r.pageUrl, requestUrl: r.url, resourceType: r.resourceType, subFrame: false });
|
|
96
|
+
if (needsBody && !blocked) {
|
|
97
|
+
const body = await bodyOf(r.url);
|
|
98
|
+
if (body !== undefined) {
|
|
99
|
+
records.push({ ...r, body, bytes: body.length });
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
records.push(r);
|
|
104
|
+
}
|
|
105
|
+
// The icons fetched here: the favicon analyzer takes them whatever their names (they are icons by the page's links).
|
|
106
|
+
const fetchedIcons = new Set();
|
|
107
|
+
const iconUrls = iconUrlsToFetch(view, pageUrl).filter((url) => deps.thirdParty === undefined || !isThirdPartyBlocked(deps.thirdParty, { pageUrl, requestUrl: url, resourceType: "image", subFrame: false }));
|
|
108
|
+
const iconBodies = await Promise.all(iconUrls.map((url) => fetchIcon(url)));
|
|
109
|
+
for (const [i, url] of iconUrls.entries()) {
|
|
110
|
+
const body = iconBodies[i];
|
|
111
|
+
const type = body === undefined ? undefined : sniffImageType(body);
|
|
112
|
+
if (body === undefined || type === undefined)
|
|
113
|
+
continue;
|
|
114
|
+
const record = {
|
|
115
|
+
url,
|
|
116
|
+
pageUrl,
|
|
117
|
+
viewport: view.viewport.name,
|
|
118
|
+
resourceType: "image",
|
|
119
|
+
contentType: type,
|
|
120
|
+
bytes: body.length,
|
|
121
|
+
status: 200,
|
|
122
|
+
thirdParty: !isFirstParty(pageUrl, url),
|
|
123
|
+
body,
|
|
124
|
+
};
|
|
125
|
+
records.push(record);
|
|
126
|
+
fetchedIcons.add(record);
|
|
127
|
+
}
|
|
128
|
+
// The @font-face rules of the stylesheets the page loaded, read from their captured text: the faces the CSSOM does not
|
|
129
|
+
// show (cross-origin sheets) get their URL and unicode-range from them.
|
|
130
|
+
const rules = records
|
|
131
|
+
.filter((r) => r.contentType === "text/css" && r.body !== undefined)
|
|
132
|
+
.flatMap((r) => fontFaceRulesOf(r.body.toString("utf8"), r.url));
|
|
133
|
+
const pageContext = view.pageContext ?? EMPTY_CONTEXT;
|
|
134
|
+
const ctx = {
|
|
135
|
+
url: pageUrl,
|
|
136
|
+
viewport: { name: view.viewport.name, width: view.viewport.width, height: view.viewport.height, dpr: view.viewport.dpr },
|
|
137
|
+
...EMPTY_CONTEXT,
|
|
138
|
+
...pageContext,
|
|
139
|
+
fontFaces: completeFontFaces(pageContext.fontFaces ?? [], rules),
|
|
140
|
+
};
|
|
141
|
+
for (const analyzer of analyzers) {
|
|
142
|
+
// A fetched icon was not a response of the page load: the phase's fetch decodes it and reports no transfer encoding,
|
|
143
|
+
// so only the favicon analyzer takes it (the others would judge its encoding wrongly).
|
|
144
|
+
const wanted = records.filter((r) => fetchedIcons.has(r) ? analyzer.kind === "favicon" : analyzer.wants(r));
|
|
145
|
+
if (wanted.length === 0)
|
|
146
|
+
continue;
|
|
147
|
+
count(`resources ${analyzer.kind} records`, wanted.length);
|
|
148
|
+
count(`resources ${analyzer.kind} bytes in`, wanted.reduce((sum, r) => sum + (r.body?.length ?? r.bytes), 0));
|
|
149
|
+
let found = [];
|
|
150
|
+
try {
|
|
151
|
+
found = await measure(`resources ${analyzer.kind}`, () => analyzer.analyze(wanted, ctx));
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
found = [];
|
|
155
|
+
}
|
|
156
|
+
for (const finding of found) {
|
|
157
|
+
const prev = byId.get(finding.id);
|
|
158
|
+
if (prev === undefined || finding.bytes > prev.bytes)
|
|
159
|
+
byId.set(finding.id, labelFinding(finding, pageUrl));
|
|
160
|
+
const seen = foundIn.get(finding.id) ?? new Set();
|
|
161
|
+
seen.add(view.viewport.name);
|
|
162
|
+
foundIn.set(finding.id, seen);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// A finding about what a viewport shows (details.perViewport "all", e.g. an unused font face) is kept only when every
|
|
167
|
+
// viewport reports it. Savings come first (first-party, then third-party, then leads), informational findings last.
|
|
168
|
+
const kept = [...byId.values()].filter((f) => f.details?.perViewport !== "all" || (foundIn.get(f.id)?.size ?? 0) === views.length);
|
|
169
|
+
return kept.sort((a, b) => findingTier(a) - findingTier(b) || b.savedBytes - a.savedBytes || a.id.localeCompare(b.id));
|
|
170
|
+
}, run);
|
|
171
|
+
}
|