imgtune 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +69 -6
- package/dist/audit.js +1 -0
- package/dist/backgrounds.js +31 -7
- package/dist/broken.js +5 -1
- package/dist/browser.js +4 -4
- package/dist/candidates.js +25 -0
- package/dist/check-report.js +37 -4
- package/dist/check.js +18 -1
- package/dist/cli.js +20 -0
- package/dist/egress-proxy.js +2 -1
- package/dist/ffmpeg.js +35 -0
- package/dist/hints.js +294 -18
- package/dist/image-open.js +9 -0
- package/dist/images.js +116 -7
- package/dist/layout.js +7 -7
- package/dist/lcp.js +2 -1
- package/dist/measure.js +39 -1
- package/dist/media-encode.js +5 -1
- package/dist/media-model.js +8 -4
- package/dist/media-phase.js +25 -2
- package/dist/px.js +11 -0
- package/dist/recheck.js +9 -1
- package/dist/remote.js +616 -63
- package/dist/replay.js +20 -0
- package/dist/resources/audio.js +248 -0
- package/dist/resources/brotli.js +22 -0
- package/dist/resources/cache.js +59 -0
- package/dist/resources/capture.js +225 -0
- package/dist/resources/compression.js +122 -0
- package/dist/resources/favicon.js +130 -0
- package/dist/resources/font.js +440 -0
- package/dist/resources/index.js +8 -0
- package/dist/resources/lottie.js +491 -0
- package/dist/resources/model.js +478 -0
- package/dist/resources/phase.js +171 -0
- package/dist/resources/pool.js +47 -0
- package/dist/resources/report.js +57 -0
- package/dist/resources/sfnt.js +305 -0
- package/dist/resources/svg.js +178 -0
- package/dist/resources/types.js +1 -0
- package/dist/resources/zip.js +67 -0
- package/dist/run.js +24 -4
- package/dist/sizes.js +7 -5
- package/dist/summary.js +47 -1
- package/dist/vector-art.js +294 -0
- package/dist/vector-phase.js +222 -0
- package/dist/video-analysis.js +119 -21
- package/package.json +25 -4
- package/skills/imgtune/SKILL.md +8 -4
- package/skills/imgtune/reference.md +1 -1
package/dist/remote.js
CHANGED
|
@@ -5,12 +5,16 @@ import { dirname, join, resolve } from "node:path";
|
|
|
5
5
|
import { launchChromium } from "./browser.js";
|
|
6
6
|
import sharp from "sharp";
|
|
7
7
|
import { writeFileAtomic, sha1 } from "./atomic.js";
|
|
8
|
+
import { MAX_INPUT_PIXELS, openImage } from "./image-open.js";
|
|
8
9
|
import { errorLine, mdCode, mdText, safeText } from "./text.js";
|
|
9
|
-
import { commentHints, videoRowOf, videoTotalsLine } from "./hints.js";
|
|
10
|
+
import { hiddenPagesLine, hiddenVideoPagesLine, commentHints, hiddenSummaryLine, videoRowOf, videoTotalsLine } from "./hints.js";
|
|
10
11
|
import { lcpLinesOf } from "./lcp.js";
|
|
11
12
|
import { MAX_MAX_QUALITIES, MIN_MAX_QUALITIES, planQualityCap, qualitiesLine } from "./qualities-cap.js";
|
|
12
13
|
export { mdCode, mdText, safeText };
|
|
14
|
+
export { MAX_INPUT_PIXELS, openImage };
|
|
13
15
|
import { isLocalTarget, systemResolver } from "./address.js";
|
|
16
|
+
import { loadVectorizer, stillFlatness } from "./vector-art.js";
|
|
17
|
+
import { applyVector, FlatBuffers, runVectorPhase } from "./vector-phase.js";
|
|
14
18
|
import { startEgressProxy } from "./egress-proxy.js";
|
|
15
19
|
import { makeFetcher, MAX_IMAGE_BYTES } from "./fetch.js";
|
|
16
20
|
import { sameSite } from "./site.js";
|
|
@@ -20,7 +24,7 @@ import { DEFAULT_VIEWPORTS } from "./config.js";
|
|
|
20
24
|
import { encode, encodedBytes, ENCODE_THREADS, ENCODER_ID, FAST_AVIF_EFFORT } from "./encode.js";
|
|
21
25
|
import { adapterFor, formatAutoOf, originalOf } from "./loaders.js";
|
|
22
26
|
import { COMPONENT_NAME, MAX_SOURCE_READ, imageAtLine, parseSourceMap, resolveFunctionLocations, resolveOwnerChain, resolveOwners } from "./locate.js";
|
|
23
|
-
import { buildImageRecords, hasFix, rungOfPage } from "./images.js";
|
|
27
|
+
import { buildImageRecords, hasFix, hiddenTotalsOf, isPlainNoSrcset, rungOfPage } from "./images.js";
|
|
24
28
|
import { animationOf } from "./animated.js";
|
|
25
29
|
import { fullBytesOf, SCAN_MAX_VIDEO_WIDTHS, videoRecordsOf, withPosterImages } from "./video-analysis.js";
|
|
26
30
|
import { backgroundPage, backgroundTotals, buildBackgroundRecords, displayedCssWidth, MAX_BACKGROUND_ELEMENTS, MAX_BACKGROUND_VALUE, backgroundTooLarge, neededDevicePixels, parseBackgroundImage, parseBackgroundSize, pickLoaded, } from "./backgrounds.js";
|
|
@@ -33,7 +37,16 @@ import { gpuNote, startGpuScorer, gpuSettingOf, gpuSettingOne } from "./gpu.js";
|
|
|
33
37
|
import { bisectQuality, confirmBisect, minPassingQuality } from "./search.js";
|
|
34
38
|
import { SURVEY_IDLE_TIMEOUT_MS, SURVEY_PAGE_CONCURRENCY, SURVEY_VIEWPORTS, surveyNote } from "./survey.js";
|
|
35
39
|
import { cacheScope, CACHE_DIR, CACHE_VERSION, contentHash, ResultCache, resizeScope } from "./result-cache.js";
|
|
36
|
-
import { attachMediaResults, mediaAssetsOf, partitionByPolicy, probeLengths, refineAssetLength, runMediaPhase, skippedMedia, } from "./media-phase.js";
|
|
40
|
+
import { attachMediaResults, mediaAssetsOf, NEVER_SHOWN, partitionByPolicy, probeLengths, refineAssetLength, runMediaPhase, skippedMedia, splitNeverShown, } from "./media-phase.js";
|
|
41
|
+
import { analyzeResources } from "./resources/phase.js";
|
|
42
|
+
import { replayHarPath } from "./replay.js";
|
|
43
|
+
import { ResourcePool } from "./resources/pool.js";
|
|
44
|
+
import { RESOURCE_BODY_SETTLE_MS } from "./resources/capture.js";
|
|
45
|
+
import { DEFAULT_RESOURCE_BUDGET_MS, newResourceRun } from "./resources/cache.js";
|
|
46
|
+
import { resourceLinesOf, resourceTotalsLine } from "./resources/report.js";
|
|
47
|
+
/** Resource findings listed in the PR-comment Markdown (the JSON and the text output have more). */
|
|
48
|
+
const RESOURCE_MARKDOWN_ROWS = 5;
|
|
49
|
+
import { readPageContext, readResourceBody, resourceRecordOf } from "./resources/capture.js";
|
|
37
50
|
import { BROKEN_EXIT, withBroken } from "./broken.js";
|
|
38
51
|
import { EstimateRegistry, hashOf, withEstimates } from "./estimate-jobs.js";
|
|
39
52
|
import { isFolder, sourceRootOf, unreadSourceNote } from "./source-root.js";
|
|
@@ -75,7 +88,15 @@ export const MAX_PAGE_CONCURRENCY = 6;
|
|
|
75
88
|
/** The longest --idle-timeout: ten minutes (a timer past 2^31 ms would fire at once). */
|
|
76
89
|
export const MAX_IDLE_TIMEOUT_MS = 600_000;
|
|
77
90
|
const CHALLENGE_STATUS = [403, 429, 503];
|
|
78
|
-
const CHALLENGE_MARKERS = ["cf-chl", "captcha"];
|
|
91
|
+
const CHALLENGE_MARKERS = ["cf-chl", "captcha", "just a moment", "captcha-delivery", "awswaf"];
|
|
92
|
+
/**
|
|
93
|
+
* Response headers a WAF or CDN sends with its challenge page. They decide whatever the page has in it: a challenge page can
|
|
94
|
+
* carry a few images (S071 had 3 of a page's 148), so the image count does not clear them.
|
|
95
|
+
*/
|
|
96
|
+
const CHALLENGE_HEADERS = [
|
|
97
|
+
{ name: "x-amzn-waf-action", values: ["challenge", "captcha"] },
|
|
98
|
+
{ name: "cf-mitigated", values: ["challenge"] },
|
|
99
|
+
];
|
|
79
100
|
/** Quality assumed for an image whose URL does not say (next/image's default). */
|
|
80
101
|
export const ASSUMED_QUALITY = 75;
|
|
81
102
|
/** Rules longer than this are ignored; a robots.txt is read up to MAX_ROBOTS_BYTES. */
|
|
@@ -165,7 +186,7 @@ function decidingRule(robotsTxt, target) {
|
|
|
165
186
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
166
187
|
// Resolution: srcset candidates and the right one
|
|
167
188
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
168
|
-
import { UNDERSIZED_RATIO, isUndersized, neededWidth, rightCandidate, undersizedProposal } from "./candidates.js";
|
|
189
|
+
import { UNDERSIZED_RATIO, isUndersized, neededWidth, rightCandidate, srcsetWidthLabel, srcsetWidthsOf, undersizedProposal } from "./candidates.js";
|
|
169
190
|
export { isUndersized, neededWidth, rightCandidate } from "./candidates.js";
|
|
170
191
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
171
192
|
// References and the current quality of a URL
|
|
@@ -308,7 +329,11 @@ export function groupKey(sizes, classes, refWidth) {
|
|
|
308
329
|
return [sizes ?? "-", cls || "-", bucket].join("|");
|
|
309
330
|
}
|
|
310
331
|
/** A page is a challenge when the document is refused or throttled, or it has no image and shows a challenge marker. */
|
|
311
|
-
export function isChallenge(status, imageCount, html) {
|
|
332
|
+
export function isChallenge(status, imageCount, html, headers = {}) {
|
|
333
|
+
for (const { name, values } of CHALLENGE_HEADERS) {
|
|
334
|
+
if (values.includes((headers[name] ?? "").toLowerCase()))
|
|
335
|
+
return true;
|
|
336
|
+
}
|
|
312
337
|
if (status !== undefined && CHALLENGE_STATUS.includes(status))
|
|
313
338
|
return true;
|
|
314
339
|
if (imageCount > 0)
|
|
@@ -374,6 +399,7 @@ const VALUE_OPTIONS = [
|
|
|
374
399
|
"--page-concurrency",
|
|
375
400
|
"--idle-timeout",
|
|
376
401
|
"--media-budget",
|
|
402
|
+
"--svg-budget",
|
|
377
403
|
"--media-time",
|
|
378
404
|
"--video-bar",
|
|
379
405
|
"--video-still-bar",
|
|
@@ -406,6 +432,15 @@ export const DEFAULT_MEDIA_TIME = 600;
|
|
|
406
432
|
const MAX_MEDIA_TIME = 3600;
|
|
407
433
|
export const DEFAULT_VIDEO_BAR = 93;
|
|
408
434
|
export const DEFAULT_VIDEO_STILL_BAR = 65;
|
|
435
|
+
/** The vector check's budget: the flat images traced per run, heaviest first (--svg-budget). */
|
|
436
|
+
export const DEFAULT_SVG_BUDGET = 50;
|
|
437
|
+
export const MAX_SVG_BUDGET = 1000;
|
|
438
|
+
/** The longest --resource-time, in seconds. */
|
|
439
|
+
export const MAX_RESOURCE_TIME = 3600;
|
|
440
|
+
/** The resource analyses of a scan that run at once, in the phase-1 pool (see resources/pool.ts). */
|
|
441
|
+
const RESOURCE_PARALLEL = 2;
|
|
442
|
+
/** The default --resource-time, in seconds: the heavy resource work of a run (see resources/cache.ts). */
|
|
443
|
+
export const DEFAULT_RESOURCE_TIME_S = DEFAULT_RESOURCE_BUDGET_MS / 1000;
|
|
409
444
|
/** The arguments after `scan`: one or more page URLs and the options. Throws with a message the CLI prints. */
|
|
410
445
|
export function parseScanArgs(argv) {
|
|
411
446
|
const args = { urls: [], out: DEFAULT_OUT, aggressive: false, quality: true, thirdParty: DEFAULT_THIRD_PARTY };
|
|
@@ -430,6 +465,8 @@ export function parseScanArgs(argv) {
|
|
|
430
465
|
args.gpu ??= gpuSettingOne(0);
|
|
431
466
|
else if (arg === "--no-cache")
|
|
432
467
|
args.noCache = true;
|
|
468
|
+
else if (arg === "--no-resources")
|
|
469
|
+
args.resources = false;
|
|
433
470
|
else if (arg === "--gpu-encode")
|
|
434
471
|
args.gpuEncode = true;
|
|
435
472
|
else if (arg === "--harness")
|
|
@@ -467,6 +504,10 @@ export function parseScanArgs(argv) {
|
|
|
467
504
|
args.idleTimeoutMs = parseRange(arg, value, 0, MAX_IDLE_TIMEOUT_MS);
|
|
468
505
|
else if (arg === "--media-budget")
|
|
469
506
|
args.mediaBudget = parseRange(arg, value, 0, MAX_MEDIA_BUDGET);
|
|
507
|
+
else if (arg === "--svg-budget")
|
|
508
|
+
args.svgBudget = parseRange(arg, value, 0, MAX_SVG_BUDGET);
|
|
509
|
+
else if (arg === "--resource-time")
|
|
510
|
+
args.resourceTime = parseRange(arg, value, 0, MAX_RESOURCE_TIME);
|
|
470
511
|
else if (arg === "--media-time")
|
|
471
512
|
args.mediaTime = parseRange(arg, value, 1, MAX_MEDIA_TIME);
|
|
472
513
|
else if (arg === "--video-bar")
|
|
@@ -504,16 +545,10 @@ export function parseScanArgs(argv) {
|
|
|
504
545
|
args.thirdParty = thirdPartyFromFlags(blockThirdParty, allowHosts);
|
|
505
546
|
return args;
|
|
506
547
|
}
|
|
507
|
-
/** The most input pixels sharp accepts for a downloaded file. */
|
|
508
|
-
export const MAX_INPUT_PIXELS = 100_000_000;
|
|
509
548
|
/** A width clamped to 1..MAX_WIDTH. */
|
|
510
549
|
export function clampWidth(width) {
|
|
511
550
|
return Math.max(1, Math.min(MAX_WIDTH, Math.floor(width)));
|
|
512
551
|
}
|
|
513
|
-
/** sharp over a downloaded file, refusing inputs above `limit` pixels. */
|
|
514
|
-
export function openImage(buffer, limit = MAX_INPUT_PIXELS) {
|
|
515
|
-
return sharp(buffer, { limitInputPixels: limit });
|
|
516
|
-
}
|
|
517
552
|
/**
|
|
518
553
|
* sharp names an AVIF file "heif" (its compression is "av1"). Every other use in this file says "avif", so that is the
|
|
519
554
|
* name a pixel size carries. A HEIF with another compression (HEVC) keeps sharp's name and gets no estimate.
|
|
@@ -921,9 +956,25 @@ export function addMediaResponse(media, response) {
|
|
|
921
956
|
...(total !== undefined ? { total } : {}),
|
|
922
957
|
});
|
|
923
958
|
}
|
|
924
|
-
/**
|
|
925
|
-
|
|
926
|
-
|
|
959
|
+
/**
|
|
960
|
+
* The most candidates per viewport that are re-read after a scroll (see readView): an opacity, clipped or zero-size copy.
|
|
961
|
+
* A candidate past the cap keeps its snapshot reason, and the page counts it (ViewportObservation.revealCapped).
|
|
962
|
+
*/
|
|
963
|
+
export const REVEAL_CAP = 60;
|
|
964
|
+
/** The wait between the snapshot and the second read of a hidden candidate (see readView): it must span a toggle period. */
|
|
965
|
+
export const REVEAL_STABLE_MS = 1000;
|
|
966
|
+
/** The wait after the scroll pass, before the last read: a fade the scroll started has finished by then. */
|
|
967
|
+
export const REVEAL_SETTLE_MS = 800;
|
|
968
|
+
/** The candidates of a page's viewports that were past REVEAL_CAP (not re-read after a scroll), summed. */
|
|
969
|
+
function revealCappedOf(observations) {
|
|
970
|
+
return observations.reduce((sum, obs) => sum + (obs.revealCapped ?? 0), 0);
|
|
971
|
+
}
|
|
972
|
+
/**
|
|
973
|
+
* One read of the page as it stands: its images and whether it is a Next.js app. `quick` takes no stability pass (see
|
|
974
|
+
* settledView): every hidden copy it finds is unverified.
|
|
975
|
+
*/
|
|
976
|
+
async function readView(page, light = false, withVideos = false, quick = false) {
|
|
977
|
+
const seen = await page.evaluate(async ({ depth, maxOwners, maxBackgrounds, maxValue, light, withVideos, revealCap, revealWait, revealSettle, reveal }) => {
|
|
927
978
|
const num = (value) => {
|
|
928
979
|
const n = parseFloat(value);
|
|
929
980
|
return Number.isFinite(n) ? n : 0;
|
|
@@ -1068,8 +1119,86 @@ async function readView(page, light = false, withVideos = false) {
|
|
|
1068
1119
|
return { nodes, skipped, truncated: el !== null };
|
|
1069
1120
|
};
|
|
1070
1121
|
const nextjs = document.querySelector('img[data-nimg], script[src*="/_next/"], link[href*="/_next/"]') !== null;
|
|
1122
|
+
// Why an image is not shown, if it is not (see HiddenReason). Read in full reads only: a light read has no use for it.
|
|
1123
|
+
// The selectors of the :hover rules that swap what is shown (a rule setting display, visibility or opacity), with the
|
|
1124
|
+
// :hover taken out: an element they match is shown on hover, not on load.
|
|
1125
|
+
const hoverSelectors = [];
|
|
1126
|
+
const collectHover = (rules) => {
|
|
1127
|
+
for (const rule of Array.from(rules)) {
|
|
1128
|
+
if (hoverSelectors.length >= 2000)
|
|
1129
|
+
return;
|
|
1130
|
+
if (rule instanceof CSSStyleRule) {
|
|
1131
|
+
if (rule.selectorText.includes(":hover") && (rule.style.display || rule.style.visibility || rule.style.opacity)) {
|
|
1132
|
+
hoverSelectors.push(rule.selectorText.replace(/:hover/g, ""));
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
else if ("cssRules" in rule) {
|
|
1136
|
+
collectHover(rule.cssRules);
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
};
|
|
1140
|
+
for (const sheet of Array.from(document.styleSheets)) {
|
|
1141
|
+
try {
|
|
1142
|
+
collectHover(sheet.cssRules);
|
|
1143
|
+
}
|
|
1144
|
+
catch {
|
|
1145
|
+
// A cross-origin sheet has no readable rules.
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
const hoverSwap = (el) => hoverSelectors.some((selector) => {
|
|
1149
|
+
try {
|
|
1150
|
+
return el.matches(selector);
|
|
1151
|
+
}
|
|
1152
|
+
catch {
|
|
1153
|
+
return false;
|
|
1154
|
+
}
|
|
1155
|
+
});
|
|
1156
|
+
const hiddenReasonOf = (img, rect) => {
|
|
1157
|
+
let displayNone = false;
|
|
1158
|
+
let opacityZero = false;
|
|
1159
|
+
let inPanel = false;
|
|
1160
|
+
let clipped = false;
|
|
1161
|
+
for (let el = img; el !== null; el = el.parentElement) {
|
|
1162
|
+
const s = getComputedStyle(el);
|
|
1163
|
+
// content-visibility: hidden keeps the box (a closed <details> does) but skips the rendering: not shown either.
|
|
1164
|
+
if (s.display === "none" || s.getPropertyValue("content-visibility") === "hidden")
|
|
1165
|
+
displayNone = true;
|
|
1166
|
+
if (s.opacity === "0")
|
|
1167
|
+
opacityZero = true;
|
|
1168
|
+
// A tab panel, a hidden subtree, an aria-hidden one, or the content of a closed <details> (rendered when it opens).
|
|
1169
|
+
if (el.getAttribute("role") === "tabpanel" || el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true")
|
|
1170
|
+
inPanel = true;
|
|
1171
|
+
// The content of a closed <details> is not rendered (Chrome skips it with a pseudo-element, so the box is laid out
|
|
1172
|
+
// and only the content is not shown); the summary stays visible.
|
|
1173
|
+
if (el.tagName === "DETAILS" && !el.hasAttribute("open") && img.closest("summary") === null) {
|
|
1174
|
+
inPanel = true;
|
|
1175
|
+
displayNone = true;
|
|
1176
|
+
}
|
|
1177
|
+
// The root's overflow is the viewport's (it propagates from html or body), so it clips nothing: only the ancestors between.
|
|
1178
|
+
const root = el === document.documentElement || el === document.body;
|
|
1179
|
+
if (!root && el !== img && (s.overflowX === "hidden" || s.overflowY === "hidden" || s.overflowX === "clip" || s.overflowY === "clip")) {
|
|
1180
|
+
const box = el.getBoundingClientRect();
|
|
1181
|
+
if (rect.width > 0 && rect.height > 0 && (rect.right <= box.left || rect.left >= box.right || rect.bottom <= box.top || rect.top >= box.bottom))
|
|
1182
|
+
clipped = true;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
const own = getComputedStyle(img);
|
|
1186
|
+
const reason = displayNone
|
|
1187
|
+
? "display"
|
|
1188
|
+
: own.visibility === "hidden" || own.visibility === "collapse"
|
|
1189
|
+
? "visibility"
|
|
1190
|
+
: opacityZero
|
|
1191
|
+
? "opacity"
|
|
1192
|
+
: rect.width === 0 || rect.height === 0
|
|
1193
|
+
? "zero-size"
|
|
1194
|
+
: clipped
|
|
1195
|
+
? "clipped"
|
|
1196
|
+
: undefined;
|
|
1197
|
+
return reason === undefined ? undefined : { reason, inPanel, hover: hoverSwap(img) };
|
|
1198
|
+
};
|
|
1071
1199
|
const images = Array.from(document.querySelectorAll("img"), (img, index) => {
|
|
1072
1200
|
const rect = img.getBoundingClientRect();
|
|
1201
|
+
const hidden = light ? undefined : hiddenReasonOf(img, rect);
|
|
1073
1202
|
return {
|
|
1074
1203
|
index,
|
|
1075
1204
|
currentSrc: img.currentSrc,
|
|
@@ -1081,6 +1210,9 @@ async function readView(page, light = false, withVideos = false) {
|
|
|
1081
1210
|
nextImage: img.hasAttribute("data-nimg"),
|
|
1082
1211
|
inFirstViewport: rect.bottom > 0 && rect.top < window.innerHeight,
|
|
1083
1212
|
lazy: img.loading === "lazy",
|
|
1213
|
+
...(hidden !== undefined
|
|
1214
|
+
? { hidden: hidden.reason, ...(hidden.inPanel ? { inPanel: true } : {}), ...(hidden.hover ? { hover: true } : {}) }
|
|
1215
|
+
: {}),
|
|
1084
1216
|
layout: light ? undefined : layoutOf(img),
|
|
1085
1217
|
locator: {
|
|
1086
1218
|
testid: img.closest("[data-testid]")?.getAttribute("data-testid") ?? undefined,
|
|
@@ -1105,6 +1237,7 @@ async function readView(page, light = false, withVideos = false) {
|
|
|
1105
1237
|
if (!value || value === "none" || !/url\(|image-set\(/.test(value))
|
|
1106
1238
|
continue;
|
|
1107
1239
|
const rect = el.getBoundingClientRect();
|
|
1240
|
+
const hiddenInfo = hiddenReasonOf(el, rect);
|
|
1108
1241
|
// The element's own fiber only: an ancestor's owner would name a component that does not draw this background.
|
|
1109
1242
|
const fiberKey = Object.keys(el).find((k) => k.startsWith("__reactFiber$"));
|
|
1110
1243
|
backgrounds.push({
|
|
@@ -1115,7 +1248,11 @@ async function readView(page, light = false, withVideos = false) {
|
|
|
1115
1248
|
owners: ownersOf(fiberKey ? el[fiberKey] : undefined),
|
|
1116
1249
|
width: rect.width,
|
|
1117
1250
|
height: rect.height,
|
|
1118
|
-
|
|
1251
|
+
// Hidden: the reason (see hiddenReasonOf). A hidden layer is measured only when its file was downloaded.
|
|
1252
|
+
hidden: hiddenInfo !== undefined,
|
|
1253
|
+
...(hiddenInfo !== undefined
|
|
1254
|
+
? { hiddenReason: hiddenInfo.reason, ...(hiddenInfo.inPanel ? { inPanel: true } : {}), ...(hiddenInfo.hover ? { hover: true } : {}) }
|
|
1255
|
+
: {}),
|
|
1119
1256
|
// Capped here as well: one character over the cap is enough for Node to skip the element.
|
|
1120
1257
|
backgroundImage: (value.length > maxValue ? value.slice(0, maxValue + 1) : value).replace(/url\((["']?)data:[^)]*\)/g, 'url("data:")'),
|
|
1121
1258
|
backgroundSize: cs.backgroundSize.slice(0, maxValue + 1),
|
|
@@ -1142,6 +1279,7 @@ async function readView(page, light = false, withVideos = false) {
|
|
|
1142
1279
|
return;
|
|
1143
1280
|
const cs = getComputedStyle(video);
|
|
1144
1281
|
const rect = video.getBoundingClientRect();
|
|
1282
|
+
const hiddenInfo = light ? undefined : hiddenReasonOf(video, rect);
|
|
1145
1283
|
videos.push({
|
|
1146
1284
|
index,
|
|
1147
1285
|
currentSrc: shortUrl(playing),
|
|
@@ -1166,7 +1304,11 @@ async function readView(page, light = false, withVideos = false) {
|
|
|
1166
1304
|
renderedHeight: rect.height,
|
|
1167
1305
|
...(video.videoWidth > 0 ? { videoWidth: video.videoWidth } : {}),
|
|
1168
1306
|
inFirstViewport: rect.bottom > 0 && rect.top < window.innerHeight,
|
|
1169
|
-
|
|
1307
|
+
// A full read uses the same check as an image (hiddenReasonOf); a light read keeps the display and box check only.
|
|
1308
|
+
hidden: hiddenInfo !== undefined || (light && (cs.display === "none" || cs.visibility === "hidden" || rect.width === 0 || rect.height === 0)),
|
|
1309
|
+
...(hiddenInfo !== undefined
|
|
1310
|
+
? { hiddenReason: hiddenInfo.reason, ...(hiddenInfo.inPanel ? { inPanel: true } : {}), ...(hiddenInfo.hover ? { hover: true } : {}) }
|
|
1311
|
+
: {}),
|
|
1170
1312
|
locator: {
|
|
1171
1313
|
testid: video.closest("[data-testid]")?.getAttribute("data-testid") ?? undefined,
|
|
1172
1314
|
selector: cssPath(video),
|
|
@@ -1175,8 +1317,149 @@ async function readView(page, light = false, withVideos = false) {
|
|
|
1175
1317
|
});
|
|
1176
1318
|
});
|
|
1177
1319
|
}
|
|
1178
|
-
|
|
1179
|
-
|
|
1320
|
+
// Stability: a copy is hidden only when it reads hidden on every read: the snapshot, a read REVEAL_STABLE_MS later with no
|
|
1321
|
+
// scroll, and a read after a scroll to it (the same reason each time). A copy that reads shown at any read is shown: a
|
|
1322
|
+
// carousel slide that auto-advances, a crossfade, or a fade-in a scroll brings in. The candidates are every hidden copy
|
|
1323
|
+
// (images, then videos, in document order); past revealCap per viewport a candidate keeps its snapshot reason and is
|
|
1324
|
+
// flagged unstable: "unknown". The scroll pass is one pass over the candidates: each is brought to the centre of the
|
|
1325
|
+
// window (the window only, so an overflow box such as a carousel track is not scrolled), read after two frames, and
|
|
1326
|
+
// the window is put back at the end. A light read does none of this.
|
|
1327
|
+
let revealCapped = 0;
|
|
1328
|
+
if (!light) {
|
|
1329
|
+
const candidates = [];
|
|
1330
|
+
// A copy the scan's scroll pass saw visible at some step is shown (see MARK_VISIBLE_SCRIPT in measure.ts).
|
|
1331
|
+
const seenVisible = window.__imgtuneSeen;
|
|
1332
|
+
document.querySelectorAll("img").forEach((img, index) => {
|
|
1333
|
+
const obj = images[index];
|
|
1334
|
+
if (obj === undefined || obj.hidden === undefined)
|
|
1335
|
+
return;
|
|
1336
|
+
if (seenVisible?.has(img) === true) {
|
|
1337
|
+
delete obj.hidden;
|
|
1338
|
+
delete obj.inPanel;
|
|
1339
|
+
delete obj.hover;
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
candidates.push({
|
|
1343
|
+
el: img,
|
|
1344
|
+
isVideo: false,
|
|
1345
|
+
snap: { reason: obj.hidden, inPanel: obj.inPanel === true, hover: obj.hover === true },
|
|
1346
|
+
apply: (info, unstable) => {
|
|
1347
|
+
delete obj.hidden;
|
|
1348
|
+
delete obj.inPanel;
|
|
1349
|
+
delete obj.hover;
|
|
1350
|
+
if (unstable)
|
|
1351
|
+
obj.unstable = "unknown";
|
|
1352
|
+
if (info !== undefined) {
|
|
1353
|
+
obj.hidden = info.reason;
|
|
1354
|
+
if (info.inPanel)
|
|
1355
|
+
obj.inPanel = true;
|
|
1356
|
+
if (info.hover)
|
|
1357
|
+
obj.hover = true;
|
|
1358
|
+
}
|
|
1359
|
+
},
|
|
1360
|
+
});
|
|
1361
|
+
});
|
|
1362
|
+
document.querySelectorAll("video").forEach((video, index) => {
|
|
1363
|
+
const obj = videos.find((v) => v.index === index);
|
|
1364
|
+
if (obj === undefined || !obj.hidden || obj.hiddenReason === undefined)
|
|
1365
|
+
return;
|
|
1366
|
+
if (seenVisible?.has(video) === true) {
|
|
1367
|
+
delete obj.hiddenReason;
|
|
1368
|
+
delete obj.inPanel;
|
|
1369
|
+
delete obj.hover;
|
|
1370
|
+
obj.hidden = false;
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
candidates.push({
|
|
1374
|
+
el: video,
|
|
1375
|
+
isVideo: true,
|
|
1376
|
+
snap: { reason: obj.hiddenReason, inPanel: obj.inPanel === true, hover: obj.hover === true },
|
|
1377
|
+
apply: (info, unstable) => {
|
|
1378
|
+
delete obj.hiddenReason;
|
|
1379
|
+
delete obj.inPanel;
|
|
1380
|
+
delete obj.hover;
|
|
1381
|
+
obj.hidden = info !== undefined;
|
|
1382
|
+
if (unstable)
|
|
1383
|
+
obj.unstable = "unknown";
|
|
1384
|
+
if (info !== undefined) {
|
|
1385
|
+
obj.hiddenReason = info.reason;
|
|
1386
|
+
if (info.inPanel)
|
|
1387
|
+
obj.inPanel = true;
|
|
1388
|
+
if (info.hover)
|
|
1389
|
+
obj.hover = true;
|
|
1390
|
+
}
|
|
1391
|
+
},
|
|
1392
|
+
});
|
|
1393
|
+
});
|
|
1394
|
+
// Videos first: they are the heavy copies, so the cap never leaves a video unverified while images are checked.
|
|
1395
|
+
candidates.sort((a, b) => Number(b.isVideo) - Number(a.isVideo));
|
|
1396
|
+
// No candidate, or a quick read (a read taken after the page navigated during a full read): no stability pass, so no
|
|
1397
|
+
// sleeps. A hidden copy the pass would have verified is unverified here (unstable: "unknown"), as past revealCap.
|
|
1398
|
+
if (candidates.length === 0 || !reveal) {
|
|
1399
|
+
for (const c of candidates)
|
|
1400
|
+
c.apply(c.snap, true);
|
|
1401
|
+
return { nextjs, images, backgrounds, videos, revealCapped: 0 };
|
|
1402
|
+
}
|
|
1403
|
+
const checked = candidates.slice(0, revealCap);
|
|
1404
|
+
revealCapped = candidates.length - checked.length;
|
|
1405
|
+
const x0 = window.scrollX;
|
|
1406
|
+
const y0 = window.scrollY;
|
|
1407
|
+
const frames = () => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(() => resolve())));
|
|
1408
|
+
// Read 2: a second read, revealWait ms after the snapshot, with no scroll. Attribute changes in that window are recorded
|
|
1409
|
+
// too: a toggle whose two reads happen to agree (a period that fits twice in the window) still shows as a change.
|
|
1410
|
+
const touched = [];
|
|
1411
|
+
const watch = new MutationObserver((records) => {
|
|
1412
|
+
for (const record of records)
|
|
1413
|
+
touched.push(record.target);
|
|
1414
|
+
});
|
|
1415
|
+
watch.observe(document.documentElement, { subtree: true, attributes: true, attributeFilter: ["style", "class", "hidden", "open", "aria-hidden"] });
|
|
1416
|
+
await new Promise((resolve) => setTimeout(resolve, revealWait));
|
|
1417
|
+
watch.disconnect();
|
|
1418
|
+
// A change on the copy, its descendants, or one of its three nearest ancestors (a carousel's track or slide wrapper).
|
|
1419
|
+
const changedNear = (el) => touched.some((node) => {
|
|
1420
|
+
if (node === el || el.contains(node))
|
|
1421
|
+
return true;
|
|
1422
|
+
let up = el;
|
|
1423
|
+
for (let level = 0; level < 3 && up !== null; level++, up = up.parentElement)
|
|
1424
|
+
if (up === node)
|
|
1425
|
+
return true;
|
|
1426
|
+
return false;
|
|
1427
|
+
});
|
|
1428
|
+
const second = checked.map((c) => hiddenReasonOf(c.el, c.el.getBoundingClientRect()));
|
|
1429
|
+
// Read 3: one pass, each candidate read after a scroll to it.
|
|
1430
|
+
// Read 3a: each candidate right after its own scroll (a display that a scroll shows is read at its own position).
|
|
1431
|
+
const third = [];
|
|
1432
|
+
for (const c of checked) {
|
|
1433
|
+
const box = c.el.getBoundingClientRect();
|
|
1434
|
+
window.scrollBy(box.left + box.width / 2 - window.innerWidth / 2, box.top + box.height / 2 - window.innerHeight / 2);
|
|
1435
|
+
await frames();
|
|
1436
|
+
third.push(hiddenReasonOf(c.el, c.el.getBoundingClientRect()));
|
|
1437
|
+
}
|
|
1438
|
+
// Read 3b: once, after the pass, so a CSS fade started by the scroll has finished (one wait for the whole pass).
|
|
1439
|
+
await new Promise((resolve) => setTimeout(resolve, revealSettle));
|
|
1440
|
+
const fourth = checked.map((c) => hiddenReasonOf(c.el, c.el.getBoundingClientRect()));
|
|
1441
|
+
window.scrollTo(x0, y0);
|
|
1442
|
+
checked.forEach((c, i) => {
|
|
1443
|
+
const a = c.snap;
|
|
1444
|
+
const b = second[i];
|
|
1445
|
+
const t = third[i];
|
|
1446
|
+
const f = fourth[i];
|
|
1447
|
+
// Hidden only when every read is hidden with the same reason; any read that shows it, or a change of reason, is
|
|
1448
|
+
// shown (a toggle, a fade, or a box a scroll brings in).
|
|
1449
|
+
const stable = b !== undefined &&
|
|
1450
|
+
t !== undefined &&
|
|
1451
|
+
f !== undefined &&
|
|
1452
|
+
b.reason === a.reason &&
|
|
1453
|
+
t.reason === a.reason &&
|
|
1454
|
+
f.reason === a.reason &&
|
|
1455
|
+
!changedNear(c.el);
|
|
1456
|
+
c.apply(stable ? f : undefined, false);
|
|
1457
|
+
});
|
|
1458
|
+
for (const c of candidates.slice(revealCap))
|
|
1459
|
+
c.apply(c.snap, true);
|
|
1460
|
+
}
|
|
1461
|
+
return { nextjs, images, backgrounds, videos, revealCapped };
|
|
1462
|
+
}, { depth: LAYOUT_DEPTH, maxOwners: MAX_OWNERS, maxBackgrounds: MAX_BACKGROUND_ELEMENTS, maxValue: MAX_BACKGROUND_VALUE, light, withVideos, revealCap: REVEAL_CAP, revealWait: REVEAL_STABLE_MS, revealSettle: REVEAL_SETTLE_MS, reveal: !quick });
|
|
1180
1463
|
const html = seen.images.length === 0 ? await page.content() : "";
|
|
1181
1464
|
return { seen, html };
|
|
1182
1465
|
}
|
|
@@ -1263,6 +1546,10 @@ export async function readLcp(page) {
|
|
|
1263
1546
|
*/
|
|
1264
1547
|
async function settledView(page, navigations, loadCapMs, light = false, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, afterSettle, readLcpFirst = false, withVideos = false) {
|
|
1265
1548
|
let last;
|
|
1549
|
+
// Once a read was cut off by a navigation, the later reads are quick ones. A full read holds the page for its stability
|
|
1550
|
+
// pass (about 1.8 s of waits and a scroll), and a page that reloads itself on a timer would cut every full read. The
|
|
1551
|
+
// quick read's hidden copies are unverified, not claimed.
|
|
1552
|
+
let quick = false;
|
|
1266
1553
|
for (let repeat = 0;; repeat++) {
|
|
1267
1554
|
const start = navigations();
|
|
1268
1555
|
let view;
|
|
@@ -1280,7 +1567,7 @@ async function settledView(page, navigations, loadCapMs, light = false, idleTime
|
|
|
1280
1567
|
});
|
|
1281
1568
|
if (afterSettle !== undefined)
|
|
1282
1569
|
await afterSettle();
|
|
1283
|
-
view = await measure("read view", () => readView(page, light, withVideos));
|
|
1570
|
+
view = await measure("read view", () => readView(page, light, withVideos, quick));
|
|
1284
1571
|
view.lcp = lcp;
|
|
1285
1572
|
}
|
|
1286
1573
|
catch (err) {
|
|
@@ -1296,6 +1583,7 @@ async function settledView(page, navigations, loadCapMs, light = false, idleTime
|
|
|
1296
1583
|
return last;
|
|
1297
1584
|
throw new Error(`the page kept navigating after load (${MAX_POST_LOAD_NAVIGATIONS + 1} attempts)`);
|
|
1298
1585
|
}
|
|
1586
|
+
quick = true;
|
|
1299
1587
|
// The new document settles first: its load event, or the cap if it never comes.
|
|
1300
1588
|
await waitForLoadWithin(page, loadCapMs).catch(() => false);
|
|
1301
1589
|
}
|
|
@@ -1442,8 +1730,6 @@ function responsesOfStep(fetched, answered, seen, dpr) {
|
|
|
1442
1730
|
if (img.currentSrc)
|
|
1443
1731
|
keep(img.currentSrc);
|
|
1444
1732
|
for (const bg of seen.backgrounds) {
|
|
1445
|
-
if (bg.hidden)
|
|
1446
|
-
continue;
|
|
1447
1733
|
for (const layer of parseBackgroundImage(bg.backgroundImage)) {
|
|
1448
1734
|
if (layer.dataUri)
|
|
1449
1735
|
continue;
|
|
@@ -1463,15 +1749,21 @@ function responsesOfStep(fetched, answered, seen, dpr) {
|
|
|
1463
1749
|
*
|
|
1464
1750
|
* `light` (the locate and sizes-only paths) reads no image bodies and no layout chain, and is always a single fresh load.
|
|
1465
1751
|
*/
|
|
1466
|
-
async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos = false) {
|
|
1752
|
+
async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos = false, captureResources = false) {
|
|
1467
1753
|
const first = viewports[0];
|
|
1468
1754
|
const outcomes = [];
|
|
1469
1755
|
let fresh = [];
|
|
1470
1756
|
// Every image response of the document, by URL (the latest one for a URL fetched twice).
|
|
1471
1757
|
const fetched = new Map();
|
|
1758
|
+
// Every other response of the page load (not media, not in a light read), by URL: the resource analyzers' records. The
|
|
1759
|
+
// records are kept across the viewport steps, since a resized viewport is the same page and its resources stay loaded.
|
|
1760
|
+
const others = new Map();
|
|
1472
1761
|
// Every media (video) response of the page load, by URL: its bytes from the headers, never from the body.
|
|
1473
1762
|
const media = new Map();
|
|
1474
1763
|
const bodyReads = [];
|
|
1764
|
+
// The resource bodies (see resources/): settled on their own short wait, so one stream that never ends costs seconds, not the
|
|
1765
|
+
// image bodies' wait. A body not read in time is fetched again by the resource phase, when an analyzer wants it.
|
|
1766
|
+
const resourceReads = [];
|
|
1475
1767
|
// The media bodies, awaited on their own: they share one deadline for the whole page load (see mediaDeadline below).
|
|
1476
1768
|
const mediaReads = [];
|
|
1477
1769
|
// Per media response: counts the bytes that arrived, for a response still streaming when the media reads settle.
|
|
@@ -1489,9 +1781,13 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1489
1781
|
// costs BODY_READ_TIMEOUT_MS once per page, not once per viewport.
|
|
1490
1782
|
let mediaDeadline;
|
|
1491
1783
|
let blockedRequests = 0;
|
|
1784
|
+
// The names the egress proxy could not resolve during this page load (see startEgressProxy).
|
|
1785
|
+
const unresolvedHosts = new Set();
|
|
1492
1786
|
let thirdPartyBlocked = 0;
|
|
1493
1787
|
// Stylesheet URLs the browser could not apply (refused, failed or blocked): one entry per URL.
|
|
1494
1788
|
const failedSheets = new Set();
|
|
1789
|
+
// Stylesheet URLs the browser applied (answered below 400): with failedSheets, it tells a partly styled page from an unstyled one.
|
|
1790
|
+
const loadedSheets = new Set();
|
|
1495
1791
|
// Requests the browser has not finished, by request id (a redirect keeps its id).
|
|
1496
1792
|
const busy = new Set();
|
|
1497
1793
|
// The image URLs answered during the current step (see responsesOfStep).
|
|
@@ -1499,8 +1795,11 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1499
1795
|
const decoded = new WeakSet();
|
|
1500
1796
|
// Every request the browser makes goes through this proxy, so each redirect hop and each tunnel is checked at connect
|
|
1501
1797
|
// time. Chromium gives the route handler no route for a redirect hop, so the route guard alone would miss them.
|
|
1502
|
-
const egress = await measure("egress proxy start", () => startEgressProxy(guard.policy, () => {
|
|
1798
|
+
const egress = await measure("egress proxy start", () => startEgressProxy(guard.policy, (host) => {
|
|
1503
1799
|
blockedRequests++;
|
|
1800
|
+
// A name the proxy could not resolve (not a refused address): the page's own scripts for it may not run here.
|
|
1801
|
+
if (host !== undefined)
|
|
1802
|
+
unresolvedHosts.add(host);
|
|
1504
1803
|
}));
|
|
1505
1804
|
try {
|
|
1506
1805
|
const context = await measure("context start", () => browser.newContext({
|
|
@@ -1537,6 +1836,14 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1537
1836
|
}
|
|
1538
1837
|
catch {
|
|
1539
1838
|
blocked = true;
|
|
1839
|
+
// The guard could not resolve the name (it throws on a failed lookup, not on a refused address): a name a browser
|
|
1840
|
+
// may still load (see unresolvedHosts).
|
|
1841
|
+
try {
|
|
1842
|
+
unresolvedHosts.add(new URL(url).hostname);
|
|
1843
|
+
}
|
|
1844
|
+
catch {
|
|
1845
|
+
// Not a URL the guard looked at.
|
|
1846
|
+
}
|
|
1540
1847
|
}
|
|
1541
1848
|
if (blocked) {
|
|
1542
1849
|
blockedRequests++;
|
|
@@ -1566,14 +1873,22 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1566
1873
|
socket.connectToServer();
|
|
1567
1874
|
}
|
|
1568
1875
|
});
|
|
1876
|
+
// Replay for development (IMGTUNE_REPLAY_HAR, see replay.ts): registered last, so the recorded page answers first.
|
|
1877
|
+
const replayHar = replayHarPath(pageUrl, first.name);
|
|
1878
|
+
if (replayHar !== undefined)
|
|
1879
|
+
await context.routeFromHAR(replayHar, { notFound: "abort" });
|
|
1569
1880
|
context.on("requestfailed", (request) => {
|
|
1570
1881
|
if (request.resourceType() === "stylesheet")
|
|
1571
1882
|
failedSheets.add(request.url());
|
|
1572
1883
|
});
|
|
1573
1884
|
context.on("response", (response) => {
|
|
1574
1885
|
// A stylesheet the server refused is not applied (Chromium still makes an empty sheet for it): counted for the scan's warning.
|
|
1575
|
-
if (response.request().resourceType() === "stylesheet"
|
|
1576
|
-
|
|
1886
|
+
if (response.request().resourceType() === "stylesheet") {
|
|
1887
|
+
if (response.status() >= 400)
|
|
1888
|
+
failedSheets.add(response.url());
|
|
1889
|
+
else
|
|
1890
|
+
loadedSheets.add(response.url());
|
|
1891
|
+
}
|
|
1577
1892
|
const contentType = response.headers()["content-type"];
|
|
1578
1893
|
// A media response is sized once it has finished, from the browser's count of its body (sizes): the body is never
|
|
1579
1894
|
// read, since a video can be far too big to buffer. Chunked answers have no Content-Length, so the headers are the
|
|
@@ -1587,12 +1902,19 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1587
1902
|
const headers = response.headers();
|
|
1588
1903
|
const headerBytes = mediaBytesOf(headers) ?? 0;
|
|
1589
1904
|
const { start, total } = contentRangeOf(headers);
|
|
1905
|
+
// A media file is a resource record too, without a body: the resource phase fetches it again when it is wanted
|
|
1906
|
+
// (Chromium gives no body for a range response, see resources/phase.ts).
|
|
1907
|
+
const resourceRecord = captureResources ? resourceRecordOf(response, pageUrl, "", contentType) : undefined;
|
|
1908
|
+
if (resourceRecord !== undefined)
|
|
1909
|
+
others.set(url, resourceRecord);
|
|
1590
1910
|
let counted = false;
|
|
1591
1911
|
const count = (bytes) => {
|
|
1592
1912
|
if (counted)
|
|
1593
1913
|
return;
|
|
1594
1914
|
counted = true;
|
|
1595
1915
|
addMediaResponse(media, { url, status, contentType, bytes, start, total });
|
|
1916
|
+
if (resourceRecord !== undefined)
|
|
1917
|
+
resourceRecord.bytes = bytes;
|
|
1596
1918
|
};
|
|
1597
1919
|
mediaReads.push((async () => {
|
|
1598
1920
|
try {
|
|
@@ -1609,6 +1931,12 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1609
1931
|
mediaFallbacks.push(() => count(Math.max(headerBytes, streamedBytesOf(url))));
|
|
1610
1932
|
return;
|
|
1611
1933
|
}
|
|
1934
|
+
// Every other response is a resource record (see resources/): its body is read only when an analyzer wants it.
|
|
1935
|
+
if (captureResources && !light) {
|
|
1936
|
+
const record = resourceRecordOf(response, pageUrl, "", contentType);
|
|
1937
|
+
others.set(response.url(), record);
|
|
1938
|
+
resourceReads.push(readResourceBody(record, response).catch(() => { }));
|
|
1939
|
+
}
|
|
1612
1940
|
const isImage = response.request().resourceType() === "image" || (contentType ?? "").toLowerCase().startsWith("image/");
|
|
1613
1941
|
if (!isImage)
|
|
1614
1942
|
return;
|
|
@@ -1692,6 +2020,8 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1692
2020
|
fresh = viewports.slice(step);
|
|
1693
2021
|
break;
|
|
1694
2022
|
}
|
|
2023
|
+
// What the page shows, once it has settled (the resource analyzers read it; a light read takes none).
|
|
2024
|
+
const pageContext = captureResources && !light ? await measure("page context", () => readPageContext(page)) : undefined;
|
|
1695
2025
|
const status = loaded?.response?.status();
|
|
1696
2026
|
if (debug !== undefined && functionsWanted !== undefined) {
|
|
1697
2027
|
for (const img of seen.images) {
|
|
@@ -1704,6 +2034,7 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1704
2034
|
await debug.session.detach().catch(() => undefined);
|
|
1705
2035
|
}
|
|
1706
2036
|
await measure("body reads", () => settleWithin(bodyReads, BODY_READ_TIMEOUT_MS));
|
|
2037
|
+
await measure("resource bodies", () => settleWithin(resourceReads, RESOURCE_BODY_SETTLE_MS));
|
|
1707
2038
|
mediaDeadline ??= Date.now() + BODY_READ_TIMEOUT_MS;
|
|
1708
2039
|
await measure("media reads", () => settleWithin(mediaReads, Math.max(0, mediaDeadline - Date.now())));
|
|
1709
2040
|
// A media response still streaming at the settle counts the bytes that arrived; it is not waited for.
|
|
@@ -1717,27 +2048,43 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1717
2048
|
decoded.add(record);
|
|
1718
2049
|
record.pixels = await measure("decode: pixel size", () => pixelSize(record.buffer));
|
|
1719
2050
|
}
|
|
2051
|
+
// A page with a name the proxy could not resolve may not run its own scripts as a visitor's browser runs them: its
|
|
2052
|
+
// hidden copies are not verified (unstable: "unknown"), and the page says which names (unresolvedHosts).
|
|
2053
|
+
if (unresolvedHosts.size > 0) {
|
|
2054
|
+
for (const image of seen.images)
|
|
2055
|
+
if (image.hidden !== undefined)
|
|
2056
|
+
image.unstable = "unknown";
|
|
2057
|
+
for (const video of seen.videos)
|
|
2058
|
+
if (video.hidden && video.hiddenReason !== undefined)
|
|
2059
|
+
video.unstable = "unknown";
|
|
2060
|
+
}
|
|
1720
2061
|
outcomes.push({
|
|
1721
2062
|
ok: true,
|
|
1722
2063
|
value: {
|
|
1723
2064
|
viewport,
|
|
1724
2065
|
load: step === 0 ? "fresh" : "resized",
|
|
2066
|
+
...(unresolvedHosts.size > 0 ? { unresolvedHosts: [...unresolvedHosts].slice(0, 20) } : {}),
|
|
1725
2067
|
status,
|
|
1726
2068
|
finalUrl: page.url(),
|
|
1727
2069
|
...(step === 0 && loaded !== undefined ? { loadTimedOut: loaded.loadTimedOut } : {}),
|
|
1728
2070
|
...(lcp !== undefined ? { lcp } : {}),
|
|
1729
|
-
blocked: isChallenge(status, seen.images.length, html),
|
|
2071
|
+
blocked: isChallenge(status, seen.images.length, html, loaded?.response?.headers()),
|
|
1730
2072
|
blockedRequests: blockedRequests - blockedBefore,
|
|
1731
2073
|
thirdPartyBlocked: thirdPartyBlocked - thirdBefore,
|
|
1732
2074
|
nextjs: seen.nextjs,
|
|
1733
2075
|
images: seen.images,
|
|
2076
|
+
revealCapped: seen.revealCapped,
|
|
1734
2077
|
backgrounds: seen.backgrounds,
|
|
1735
2078
|
videos: seen.videos,
|
|
1736
2079
|
// A copy: the page load's records keep growing for the viewports after this one.
|
|
1737
2080
|
media: new Map(media),
|
|
1738
2081
|
stylesheetsFailed: failedSheets.size,
|
|
1739
2082
|
stylesheetHosts: hostsOf(failedSheets),
|
|
2083
|
+
stylesheetsLoaded: loadedSheets.size,
|
|
1740
2084
|
responses,
|
|
2085
|
+
...(captureResources && !light
|
|
2086
|
+
? { resources: [...others.values()].map((record) => ({ ...record, viewport: viewport.name })), ...(pageContext !== undefined ? { pageContext } : {}) }
|
|
2087
|
+
: {}),
|
|
1741
2088
|
},
|
|
1742
2089
|
});
|
|
1743
2090
|
}
|
|
@@ -1762,10 +2109,10 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1762
2109
|
* The observations of one load group (see readGroup), one outcome per viewport in the order given. A group of one
|
|
1763
2110
|
* viewport is a fresh load: the default. `--shared-loads` groups the viewports of a class into one load.
|
|
1764
2111
|
*/
|
|
1765
|
-
export async function observeLoads(browser, pageUrl, viewports, guard, loadCapMs = LOAD_EVENT_CAP_MS, storageState, light = false, functionsWanted, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, withVideos = false) {
|
|
1766
|
-
const { outcomes, fresh } = await readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos);
|
|
2112
|
+
export async function observeLoads(browser, pageUrl, viewports, guard, loadCapMs = LOAD_EVENT_CAP_MS, storageState, light = false, functionsWanted, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, withVideos = false, captureResources = false) {
|
|
2113
|
+
const { outcomes, fresh } = await readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos, captureResources);
|
|
1767
2114
|
for (const viewport of fresh) {
|
|
1768
|
-
const [outcome] = await observeLoads(browser, pageUrl, [viewport], guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos);
|
|
2115
|
+
const [outcome] = await observeLoads(browser, pageUrl, [viewport], guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos, captureResources);
|
|
1769
2116
|
outcomes.push(outcome);
|
|
1770
2117
|
}
|
|
1771
2118
|
return outcomes;
|
|
@@ -1946,8 +2293,23 @@ async function measureBackground(ctx, obs, bg, layerIndex, layer, size) {
|
|
|
1946
2293
|
size,
|
|
1947
2294
|
};
|
|
1948
2295
|
const skip = (reason) => ({ ...instance, skipped: reason });
|
|
1949
|
-
if (bg.hidden)
|
|
1950
|
-
|
|
2296
|
+
if (bg.hidden) {
|
|
2297
|
+
// Not shown on load. Its file is measured only if the browser downloaded it (display:none usually does not download).
|
|
2298
|
+
const loaded = layer.dataUri ? undefined : pickLoaded(layer.candidates, (url) => obs.responses.get(url)?.buffer !== undefined, dpr);
|
|
2299
|
+
if (!loaded)
|
|
2300
|
+
return skip("hidden");
|
|
2301
|
+
const downloaded = obs.responses.get(loaded.url);
|
|
2302
|
+
return {
|
|
2303
|
+
...instance,
|
|
2304
|
+
url: loaded.url,
|
|
2305
|
+
bytes: downloaded.bytes,
|
|
2306
|
+
pixelWidth: downloaded.pixels?.width ?? 0,
|
|
2307
|
+
format: formatName(downloaded.pixels?.format),
|
|
2308
|
+
hidden: bg.hiddenReason,
|
|
2309
|
+
...(bg.inPanel === true ? { inPanel: true } : {}),
|
|
2310
|
+
...(bg.hover === true ? { hover: true } : {}),
|
|
2311
|
+
};
|
|
2312
|
+
}
|
|
1951
2313
|
if (bg.width < MIN_RENDERED_PX || bg.height < MIN_RENDERED_PX)
|
|
1952
2314
|
return skip("under 16 px");
|
|
1953
2315
|
if (layer.dataUri)
|
|
@@ -1978,6 +2340,11 @@ async function measureBackground(ctx, obs, bg, layerIndex, layer, size) {
|
|
|
1978
2340
|
const animated = animationOfBuffer(response.buffer);
|
|
1979
2341
|
if (animated !== undefined)
|
|
1980
2342
|
instance.animated = animated;
|
|
2343
|
+
const flatness = await stillFlatness(response, served, animated !== undefined);
|
|
2344
|
+
if (flatness !== undefined)
|
|
2345
|
+
instance.flatness = flatness;
|
|
2346
|
+
if (flatness?.flat === true)
|
|
2347
|
+
ctx.flatBuffers?.offer(chosen.url, response.buffer, response.bytes);
|
|
1981
2348
|
if (ctx.estimates !== false && animated === undefined && served.width > PLAIN_RESIZE_RATIO * needed && served.format) {
|
|
1982
2349
|
const resized = await measure("background resize estimate", () => resizeEstimateOf(ctx, response.buffer, served.format, needed));
|
|
1983
2350
|
if (resized)
|
|
@@ -2010,7 +2377,8 @@ async function baseline(ctx, obs, img) {
|
|
|
2010
2377
|
...loaderOf(img.currentSrc, ctx.pageUrl),
|
|
2011
2378
|
};
|
|
2012
2379
|
const skip = (reason) => ({ ...instance, skipped: reason });
|
|
2013
|
-
|
|
2380
|
+
// A hidden element is measured whatever its box: its file was downloaded, and that is what the page spends (see hidden).
|
|
2381
|
+
if (img.renderedWidth < MIN_RENDERED_PX && img.hidden === undefined)
|
|
2014
2382
|
return skip("under 16 px");
|
|
2015
2383
|
if (!img.currentSrc || img.currentSrc.startsWith("data:"))
|
|
2016
2384
|
return skip("data URI");
|
|
@@ -2018,6 +2386,15 @@ async function baseline(ctx, obs, img) {
|
|
|
2018
2386
|
if (!response?.buffer)
|
|
2019
2387
|
return skip("not loaded");
|
|
2020
2388
|
const served = response.pixels;
|
|
2389
|
+
if (img.hidden !== undefined) {
|
|
2390
|
+
// Loaded but not shown at this viewport: the bytes, and nothing to fix in the file itself (no sizes, no quality, no saving).
|
|
2391
|
+
instance.kind = "hidden";
|
|
2392
|
+
Object.assign(instance, { bytes: response.bytes, pixelWidth: served?.width ?? 0, format: served?.format ? formatName(served.format) : undefined });
|
|
2393
|
+
const hiddenAnimated = animationOfBuffer(response.buffer);
|
|
2394
|
+
if (hiddenAnimated !== undefined)
|
|
2395
|
+
instance.animated = hiddenAnimated;
|
|
2396
|
+
return instance;
|
|
2397
|
+
}
|
|
2021
2398
|
if (response.contentType?.toLowerCase().includes("svg") || served?.format === "svg")
|
|
2022
2399
|
return skip("vector");
|
|
2023
2400
|
if (!served)
|
|
@@ -2026,6 +2403,11 @@ async function baseline(ctx, obs, img) {
|
|
|
2026
2403
|
const animated = animationOfBuffer(response.buffer);
|
|
2027
2404
|
if (animated !== undefined)
|
|
2028
2405
|
instance.animated = animated;
|
|
2406
|
+
const flatness = await stillFlatness(response, served, animated !== undefined);
|
|
2407
|
+
if (flatness !== undefined)
|
|
2408
|
+
instance.flatness = flatness;
|
|
2409
|
+
if (flatness?.flat === true)
|
|
2410
|
+
ctx.flatBuffers?.offer(img.currentSrc, response.buffer, response.bytes);
|
|
2029
2411
|
const candidates = parseSrcset(img.srcset, ctx.pageUrl);
|
|
2030
2412
|
if (candidates.length > 0) {
|
|
2031
2413
|
const largest = Math.max(...candidates.map((c) => c.width));
|
|
@@ -2114,14 +2496,14 @@ export const CDN_AUTO_QUALITY_SHARE = 0.1;
|
|
|
2114
2496
|
export function applyCdnPolicies(instances, viewportCount) {
|
|
2115
2497
|
for (const inst of instances) {
|
|
2116
2498
|
const lossless = inst.format === "jpeg" || inst.format === "png" || (inst.format === "gif" && inst.animated === undefined);
|
|
2117
|
-
if (inst.formatAuto === true && inst.kind !== "skipped" && lossless) {
|
|
2499
|
+
if (inst.formatAuto === true && inst.kind !== "skipped" && inst.kind !== "hidden" && lossless) {
|
|
2118
2500
|
inst.est = 0;
|
|
2119
2501
|
inst.estAggressive = 0;
|
|
2120
2502
|
}
|
|
2121
2503
|
}
|
|
2122
2504
|
const groups = new Map();
|
|
2123
2505
|
for (const inst of instances) {
|
|
2124
|
-
if (inst.qualityParam !== "auto" || inst.kind === "skipped")
|
|
2506
|
+
if (inst.qualityParam !== "auto" || inst.kind === "skipped" || inst.kind === "hidden")
|
|
2125
2507
|
continue;
|
|
2126
2508
|
const list = groups.get(inst.group);
|
|
2127
2509
|
if (list)
|
|
@@ -2294,7 +2676,7 @@ export function applyQuality(inst) {
|
|
|
2294
2676
|
/** The `robots` value recorded for a page on a loopback or private host: its owner's own server is not checked. */
|
|
2295
2677
|
export const ROBOTS_SKIPPED_LOCAL = "skipped (local host)";
|
|
2296
2678
|
/** The options of one page's output, for the completeness check of --resume. */
|
|
2297
|
-
export function recordOptionsOf(opts, ffmpeg) {
|
|
2679
|
+
export function recordOptionsOf(opts, ffmpeg, vectorizer) {
|
|
2298
2680
|
const thirdParty = opts.thirdParty ?? DEFAULT_THIRD_PARTY;
|
|
2299
2681
|
// The media settings are recorded when encodes can run, so a page scanned under other settings (or without ffmpeg) is
|
|
2300
2682
|
// not reused by --resume. Without encodes they are left out, and the record keeps its shape.
|
|
@@ -2309,8 +2691,12 @@ export function recordOptionsOf(opts, ffmpeg) {
|
|
|
2309
2691
|
ffmpeg,
|
|
2310
2692
|
}
|
|
2311
2693
|
: {};
|
|
2694
|
+
const vector = opts.quality === true && opts.survey !== true
|
|
2695
|
+
? { svgBudget: opts.svgBudget ?? DEFAULT_SVG_BUDGET, vectorizer: vectorizer === true }
|
|
2696
|
+
: {};
|
|
2312
2697
|
return {
|
|
2313
2698
|
...media,
|
|
2699
|
+
...vector,
|
|
2314
2700
|
imgtuneVersion: IMGTUNE_VERSION,
|
|
2315
2701
|
aggressive: opts.aggressive,
|
|
2316
2702
|
quality: opts.quality,
|
|
@@ -2318,6 +2704,7 @@ export function recordOptionsOf(opts, ffmpeg) {
|
|
|
2318
2704
|
ignoreRobots: opts.ignoreRobots === true,
|
|
2319
2705
|
storageState: opts.storageState !== undefined,
|
|
2320
2706
|
scorer: opts.gpu !== undefined ? "gpu" : "cpu",
|
|
2707
|
+
resources: opts.resources !== false,
|
|
2321
2708
|
...(opts.harness === true ? { harness: true } : {}),
|
|
2322
2709
|
...(opts.survey === true ? { survey: true } : {}),
|
|
2323
2710
|
...(opts.sharedLoads === true ? { loads: "shared" } : {}),
|
|
@@ -2388,10 +2775,11 @@ function toReport(inst) {
|
|
|
2388
2775
|
* Groups by saving: the group's exact + est. bytes per page view, summed over the page's viewports and divided by their
|
|
2389
2776
|
* count (as pageNumbers does), with two sample URLs and a sizes hint.
|
|
2390
2777
|
*/
|
|
2391
|
-
export function groupReports(instances, viewportCount) {
|
|
2778
|
+
export function groupReports(instances, viewportCount, framework = false) {
|
|
2392
2779
|
const byKey = new Map();
|
|
2393
2780
|
for (const inst of instances) {
|
|
2394
|
-
|
|
2781
|
+
// A hidden file is not a group member: its saving is none, and its size is not a size the page shows.
|
|
2782
|
+
if (inst.kind === "skipped" || inst.kind === "hidden")
|
|
2395
2783
|
continue;
|
|
2396
2784
|
const list = byKey.get(inst.group);
|
|
2397
2785
|
if (list)
|
|
@@ -2402,11 +2790,15 @@ export function groupReports(instances, viewportCount) {
|
|
|
2402
2790
|
const reports = [];
|
|
2403
2791
|
for (const [key, members] of byKey) {
|
|
2404
2792
|
const sizes = members[0].img.sizes;
|
|
2793
|
+
// Plain images with no srcset: a sizes proposal does nothing there, so the group is told the variant widths instead.
|
|
2794
|
+
const plainGroup = members.every((m) => isPlainNoSrcset(m.kind, m.img.srcset, m.loader, framework));
|
|
2795
|
+
const naturals = members.map((m) => m.pixelWidth).filter((w) => w > 0);
|
|
2796
|
+
const srcsetWidths = plainGroup ? srcsetWidthsOf(members.map((m) => m.needed), naturals.length > 0 ? Math.min(...naturals) : undefined) : [];
|
|
2405
2797
|
const points = members
|
|
2406
2798
|
.filter((m) => m.renderedWidth >= 1)
|
|
2407
2799
|
.map((m) => ({ viewportWidth: m.viewport.width, renderedWidth: m.renderedWidth, mobile: m.viewport.mobile }));
|
|
2408
2800
|
// A proposal that leaves any measured viewport undersized is not a saving: it is dropped, with its exact bytes.
|
|
2409
|
-
const proposed = deriveSizes(points);
|
|
2801
|
+
const proposed = plainGroup ? undefined : deriveSizes(points);
|
|
2410
2802
|
const undersized = proposed === undefined
|
|
2411
2803
|
? undefined
|
|
2412
2804
|
: undersizedProposal(proposed, members.map((m) => ({ viewport: m.viewport.name, viewportWidth: m.viewport.width, dpr: m.viewport.dpr, renderedWidth: m.renderedWidth, srcset: m.img.srcset })), (srcset) => parseSrcset(srcset, "http://imgtune.invalid/"));
|
|
@@ -2437,6 +2829,7 @@ export function groupReports(instances, viewportCount) {
|
|
|
2437
2829
|
members: new Set(members.map((m) => m.element)).size,
|
|
2438
2830
|
samples,
|
|
2439
2831
|
hint: suggestedSizes && suggestedSizes !== sizes ? `set sizes="${suggestedSizes}"` : undefined,
|
|
2832
|
+
...(srcsetWidths.length > 0 ? { srcset: { widths: srcsetWidths } } : {}),
|
|
2440
2833
|
});
|
|
2441
2834
|
}
|
|
2442
2835
|
return reports.sort((a, b) => b.savedBytes - a.savedBytes).slice(0, TOP_GROUPS);
|
|
@@ -2446,7 +2839,7 @@ function referenceViewportName(viewports) {
|
|
|
2446
2839
|
return viewports.find((v) => v.name === "iphone-15")?.name ?? viewports.find((v) => v.mobile)?.name ?? viewports[0].name;
|
|
2447
2840
|
}
|
|
2448
2841
|
/** Turns the observations of one page into the report. Fetches and local encodes happen here. */
|
|
2449
|
-
export async function analysePage(ctx, input, base) {
|
|
2842
|
+
export async function analysePage(ctx, input, base, resourcesJob = () => resourcesOf(ctx, input)) {
|
|
2450
2843
|
// A fetched video poster is measured as an image, sized to its video's box (see video-analysis.ts).
|
|
2451
2844
|
const observations = withPosterImages(input);
|
|
2452
2845
|
const refName = referenceViewportName(observations.map((o) => o.viewport));
|
|
@@ -2514,6 +2907,19 @@ export async function analysePage(ctx, input, base) {
|
|
|
2514
2907
|
}
|
|
2515
2908
|
}
|
|
2516
2909
|
}
|
|
2910
|
+
// A hidden layer whose file a shown element also loads adds no bytes of its own: it is not a hidden copy (skipped).
|
|
2911
|
+
const shownUrls = new Set([
|
|
2912
|
+
...instances.filter((i) => i.kind !== "hidden" && i.url !== "").map((i) => i.url),
|
|
2913
|
+
...bgInstances.filter((i) => i.hidden === undefined && i.url !== "").map((i) => i.url),
|
|
2914
|
+
]);
|
|
2915
|
+
for (const inst of bgInstances) {
|
|
2916
|
+
if (inst.hidden !== undefined && shownUrls.has(inst.url)) {
|
|
2917
|
+
inst.hidden = undefined;
|
|
2918
|
+
inst.inPanel = undefined;
|
|
2919
|
+
inst.hover = undefined;
|
|
2920
|
+
inst.skipped = "hidden: file shared with a shown image";
|
|
2921
|
+
}
|
|
2922
|
+
}
|
|
2517
2923
|
const bgSources = await resolveBackgroundSources(ctx, observations);
|
|
2518
2924
|
const perViewport = {};
|
|
2519
2925
|
const totals = [];
|
|
@@ -2548,7 +2954,7 @@ export async function analysePage(ctx, input, base) {
|
|
|
2548
2954
|
const mine = instances.filter((i) => i.element === e);
|
|
2549
2955
|
if (mine.every((i) => i.kind === "skipped" && i.skipped !== "duplicate"))
|
|
2550
2956
|
skippedElements.add(e);
|
|
2551
|
-
if (mine.some((i) => i.kind !== "skipped" && i.qualityRatio === undefined))
|
|
2957
|
+
if (mine.some((i) => i.kind !== "skipped" && i.kind !== "hidden" && i.qualityRatio === undefined))
|
|
2552
2958
|
naElements.add(e);
|
|
2553
2959
|
}
|
|
2554
2960
|
const numbers = measure("aggregate", () => pageNumbers(totals));
|
|
@@ -2561,6 +2967,7 @@ export async function analysePage(ctx, input, base) {
|
|
|
2561
2967
|
refName,
|
|
2562
2968
|
resolved: sources,
|
|
2563
2969
|
currentQuality,
|
|
2970
|
+
nextjs: observations.some((o) => o.nextjs),
|
|
2564
2971
|
});
|
|
2565
2972
|
// A proposed sizes is a saving only when its predicted outcome saves more bytes than it adds (sizes-outcome.ts).
|
|
2566
2973
|
const widths = new Map(observations.map((o) => [o.viewport.name, o.viewport.width]));
|
|
@@ -2573,15 +2980,24 @@ export async function analysePage(ctx, input, base) {
|
|
|
2573
2980
|
refName,
|
|
2574
2981
|
sources: bgSources,
|
|
2575
2982
|
});
|
|
2983
|
+
// The page's loaded-but-not-shown numbers are counted over the same records the report lists (images and background layers).
|
|
2984
|
+
const hiddenTotals = hiddenTotalsOf([...images, ...backgroundRecords], observations.length);
|
|
2576
2985
|
const videoSources = await resolveVideoSources(ctx, input);
|
|
2577
2986
|
const videos = videoRecordsOf(ctx.pageUrl, input, fullBytesOf(input), "scan", ctx.maxVideoWidths ?? SCAN_MAX_VIDEO_WIDTHS, videoSources);
|
|
2987
|
+
// The resource phase runs when the page was captured with resources (see readGroup); its findings are the page's.
|
|
2988
|
+
const resourcesFound = await resourcesJob();
|
|
2989
|
+
// An image never shown on load is reported once, as loaded-but-not-shown (its bytes are that fix's): an svg or text file
|
|
2990
|
+
// behind it gets no resource finding too, so the same bytes are not claimed twice. A favicon is not an image on the page.
|
|
2991
|
+
const neverShown = new Set([...images, ...backgroundRecords].flatMap((r) => (r.hidden !== undefined && r.hidden.shownIn.length === 0 ? [r.current.src] : [])).filter((src) => src !== undefined && src !== ""));
|
|
2992
|
+
const resources = resourcesFound?.filter((finding) => finding.kind === "favicon" || !neverShown.has(finding.url));
|
|
2578
2993
|
return {
|
|
2579
2994
|
...base,
|
|
2580
2995
|
nextjs: observations.some((o) => o.nextjs),
|
|
2581
2996
|
...(videos.length > 0 ? { videos } : {}),
|
|
2997
|
+
...(resources !== undefined ? { resources } : {}),
|
|
2582
2998
|
perViewport,
|
|
2583
2999
|
page: ctx.aggressive ? { ...withoutAggressive, aggressiveExactEstPct } : withoutAggressive,
|
|
2584
|
-
groups: measure("aggregate", () => groupReports(instances, observations.length)),
|
|
3000
|
+
groups: measure("aggregate", () => groupReports(instances, observations.length, observations.some((o) => o.nextjs))),
|
|
2585
3001
|
counts: {
|
|
2586
3002
|
images: elements.size,
|
|
2587
3003
|
qualityNA: naElements.size,
|
|
@@ -2592,6 +3008,8 @@ export async function analysePage(ctx, input, base) {
|
|
|
2592
3008
|
thirdPartyBlocked: observations.reduce((sum, o) => sum + (o.thirdPartyBlocked ?? 0), 0),
|
|
2593
3009
|
images: [...images, ...backgroundRecords],
|
|
2594
3010
|
notLoaded: images.filter((r) => r.lazy === true && !r.perViewport.some((p) => p.loaded)).length,
|
|
3011
|
+
...(hiddenTotals !== undefined ? { hidden: hiddenTotals } : {}),
|
|
3012
|
+
...(revealCappedOf(input) > 0 ? { revealCapped: revealCappedOf(input) } : {}),
|
|
2595
3013
|
...(bgInstances.length > 0 ? { backgrounds: backgroundPage(bgInstances, observations.length, backgroundRecords) } : {}),
|
|
2596
3014
|
...(ctx.harness === true ? { harness: true } : {}),
|
|
2597
3015
|
};
|
|
@@ -2681,7 +3099,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2681
3099
|
const groupOutcomes = await settleLimit(groups, run.viewportConcurrency, (group) => {
|
|
2682
3100
|
count("page loads");
|
|
2683
3101
|
count("viewport loads", group.length);
|
|
2684
|
-
return observeLoads(browser, pageUrl, group, guard, run.loadCapMs, run.storageState, false, undefined, run.idleTimeoutMs);
|
|
3102
|
+
return observeLoads(browser, pageUrl, group, guard, run.loadCapMs, run.storageState, false, undefined, run.idleTimeoutMs, false, run.resources);
|
|
2685
3103
|
});
|
|
2686
3104
|
const outcomes = run.viewports.map((viewport) => {
|
|
2687
3105
|
const index = groups.findIndex((group) => group.includes(viewport));
|
|
@@ -2693,6 +3111,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2693
3111
|
let loadTimedOut = false;
|
|
2694
3112
|
const sheetsFailed = {};
|
|
2695
3113
|
const failedHosts = new Set();
|
|
3114
|
+
let loadedSheets = 0;
|
|
2696
3115
|
const documentStatus = {};
|
|
2697
3116
|
try {
|
|
2698
3117
|
for (const [index, outcome] of outcomes.entries()) {
|
|
@@ -2709,6 +3128,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2709
3128
|
sheetsFailed[viewport.name] = obs.stylesheetsFailed;
|
|
2710
3129
|
for (const host of obs.stylesheetHosts ?? [])
|
|
2711
3130
|
failedHosts.add(host);
|
|
3131
|
+
loadedSheets = Math.max(loadedSheets, obs.stylesheetsLoaded ?? 0);
|
|
2712
3132
|
if (obs.status !== undefined && obs.status >= 400)
|
|
2713
3133
|
documentStatus[viewport.name] = obs.status;
|
|
2714
3134
|
if (obs.finalUrl !== undefined && new URL(obs.finalUrl).href !== url.href) {
|
|
@@ -2735,6 +3155,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2735
3155
|
described.documentStatus = documentStatus;
|
|
2736
3156
|
if (Object.keys(sheetsFailed).length > 0) {
|
|
2737
3157
|
described.stylesheetsFailed = sheetsFailed;
|
|
3158
|
+
described.stylesheetsLoaded = loadedSheets;
|
|
2738
3159
|
run.log(`warning: ${safeText(pageUrl)}: a linked stylesheet did not load at ${Object.keys(sheetsFailed).map((name) => safeText(name)).join(", ")}; the page is not styled as the app is, so its widths and sizes verdicts are not valid (check the asset prefix the server was started with)`);
|
|
2739
3160
|
}
|
|
2740
3161
|
if (robotsIgnored)
|
|
@@ -2743,10 +3164,37 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2743
3164
|
described.robotsRule = robotsRule;
|
|
2744
3165
|
if (finalUrl !== undefined)
|
|
2745
3166
|
described.redirectNote = `the page ends on ${finalUrl}; measured there`;
|
|
2746
|
-
if (run.stage === undefined)
|
|
2747
|
-
|
|
3167
|
+
if (run.stage === undefined) {
|
|
3168
|
+
const ctx = analysisContextOf(run, pageUrl);
|
|
3169
|
+
const resources = pageResourceJob(ctx, async () => observations);
|
|
3170
|
+
return {
|
|
3171
|
+
resources,
|
|
3172
|
+
analyse: (store, withResources = true) => analysePage({ ...ctx, store }, observations, described, withResources ? resources : NO_RESOURCES),
|
|
3173
|
+
};
|
|
3174
|
+
}
|
|
2748
3175
|
return stagedMeasure(run, run.stage, pageUrl, described, observations);
|
|
2749
3176
|
}
|
|
3177
|
+
/**
|
|
3178
|
+
* The resource findings of a page's observations (see resources/), or undefined when the page was captured without resources.
|
|
3179
|
+
* Never throws: a failed analysis gives undefined, so it does not fail the page.
|
|
3180
|
+
*/
|
|
3181
|
+
async function resourcesOf(ctx, observations) {
|
|
3182
|
+
if (!observations.some((o) => o.resources !== undefined))
|
|
3183
|
+
return undefined;
|
|
3184
|
+
try {
|
|
3185
|
+
return await measure("resources", () => analyzeResources(observations.map((o) => ({ viewport: o.viewport, resources: o.resources, pageContext: o.pageContext })), ctx.pageUrl, { cache: ctx.deps.cache, fetcher: ctx.fetcher, allowPrivate: ctx.allowPrivate, thirdParty: ctx.thirdParty, run: ctx.deps.resourceRun }));
|
|
3186
|
+
}
|
|
3187
|
+
catch {
|
|
3188
|
+
return undefined;
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
/** A page's resource analysis, started at most once: the phase-1 pool starts it, and the estimate pass reads the same job. */
|
|
3192
|
+
export function pageResourceJob(ctx, observations) {
|
|
3193
|
+
let job;
|
|
3194
|
+
return () => (job ??= observations().then((obs) => resourcesOf(ctx, obs)).catch(() => undefined));
|
|
3195
|
+
}
|
|
3196
|
+
/** No resource analysis: the provisional pass of a page (its resources are replaced by the estimate pass's). */
|
|
3197
|
+
const NO_RESOURCES = () => Promise.resolve(undefined);
|
|
2750
3198
|
function analysisContextOf(run, pageUrl) {
|
|
2751
3199
|
return {
|
|
2752
3200
|
pageUrl,
|
|
@@ -2762,6 +3210,8 @@ function analysisContextOf(run, pageUrl) {
|
|
|
2762
3210
|
sourceMaps: run.sourceMaps,
|
|
2763
3211
|
projectDir: run.projectDir,
|
|
2764
3212
|
projectRoot: run.projectRoot,
|
|
3213
|
+
thirdParty: run.thirdParty,
|
|
3214
|
+
flatBuffers: run.flatBuffers,
|
|
2765
3215
|
};
|
|
2766
3216
|
}
|
|
2767
3217
|
/**
|
|
@@ -2772,19 +3222,33 @@ function stagedMeasure(run, stage, pageUrl, described, live) {
|
|
|
2772
3222
|
const file = join(stage.dir, pageFileName(new URL(pageUrl)));
|
|
2773
3223
|
let pending = live;
|
|
2774
3224
|
const ctx = analysisContextOf(run, pageUrl);
|
|
3225
|
+
// The observations the resource job reads: the live ones while the page is in memory (phase 1 starts the job before the stage
|
|
3226
|
+
// drops them), else the staged file's.
|
|
3227
|
+
const resources = pageResourceJob(ctx, async () => pending ?? (await unstagedObservations(stage.dir, file)));
|
|
2775
3228
|
return {
|
|
3229
|
+
resources,
|
|
2776
3230
|
stage: async () => {
|
|
2777
3231
|
if (pending === undefined)
|
|
2778
3232
|
return;
|
|
2779
3233
|
const staged = [];
|
|
2780
3234
|
for (const obs of pending) {
|
|
2781
|
-
const { responses, media, ...rest } = obs;
|
|
3235
|
+
const { responses, media, resources, ...rest } = obs;
|
|
2782
3236
|
const records = [];
|
|
2783
3237
|
for (const [url, record] of responses) {
|
|
2784
3238
|
const { buffer, ...meta } = record;
|
|
2785
3239
|
records.push({ url, record: meta, blob: buffer === undefined ? undefined : await putBlob(stage.dir, buffer) });
|
|
2786
3240
|
}
|
|
2787
|
-
|
|
3241
|
+
const stagedResources = [];
|
|
3242
|
+
for (const resource of resources ?? []) {
|
|
3243
|
+
const { body, ...meta } = resource;
|
|
3244
|
+
stagedResources.push({ record: meta, blob: body === undefined ? undefined : await putBlob(stage.dir, body) });
|
|
3245
|
+
}
|
|
3246
|
+
staged.push({
|
|
3247
|
+
...rest,
|
|
3248
|
+
responses: records,
|
|
3249
|
+
...(media !== undefined ? { media: [...media] } : {}),
|
|
3250
|
+
...(resources !== undefined ? { resources: stagedResources } : {}),
|
|
3251
|
+
});
|
|
2788
3252
|
}
|
|
2789
3253
|
const state = { version: 1, url: pageUrl, options: stage.options, described, observations: staged };
|
|
2790
3254
|
await writeStagedJson(stage.dir, file, state);
|
|
@@ -2795,9 +3259,9 @@ function stagedMeasure(run, stage, pageUrl, described, live) {
|
|
|
2795
3259
|
if (state !== undefined)
|
|
2796
3260
|
await writeStagedJson(stage.dir, file, { ...state, images });
|
|
2797
3261
|
},
|
|
2798
|
-
analyse: async (store) => {
|
|
3262
|
+
analyse: async (store, withResources = true) => {
|
|
2799
3263
|
const observations = pending ?? (await unstagedObservations(stage.dir, file));
|
|
2800
|
-
return analysePage({ ...ctx, store }, observations, described);
|
|
3264
|
+
return analysePage({ ...ctx, store }, observations, described, withResources ? resources : NO_RESOURCES);
|
|
2801
3265
|
},
|
|
2802
3266
|
finish: async () => {
|
|
2803
3267
|
await rm(file, { force: true });
|
|
@@ -2811,12 +3275,21 @@ async function unstagedObservations(stageDir, file) {
|
|
|
2811
3275
|
throw new Error(`staged page is missing or unreadable: ${safeText(file)}`);
|
|
2812
3276
|
const observations = [];
|
|
2813
3277
|
for (const staged of state.observations) {
|
|
2814
|
-
const { responses: records, media, ...rest } = staged;
|
|
3278
|
+
const { responses: records, media, resources, ...rest } = staged;
|
|
2815
3279
|
const responses = new Map();
|
|
2816
3280
|
for (const { url, record, blob } of records) {
|
|
2817
3281
|
responses.set(url, blob === undefined ? { ...record } : { ...record, buffer: await getBlob(stageDir, blob) });
|
|
2818
3282
|
}
|
|
2819
|
-
|
|
3283
|
+
const resourceRecords = [];
|
|
3284
|
+
for (const { record, blob } of resources ?? []) {
|
|
3285
|
+
resourceRecords.push(blob === undefined ? { ...record } : { ...record, body: await getBlob(stageDir, blob) });
|
|
3286
|
+
}
|
|
3287
|
+
observations.push({
|
|
3288
|
+
...rest,
|
|
3289
|
+
responses,
|
|
3290
|
+
...(media !== undefined ? { media: new Map(media) } : {}),
|
|
3291
|
+
...(resources !== undefined ? { resources: resourceRecords } : {}),
|
|
3292
|
+
});
|
|
2820
3293
|
}
|
|
2821
3294
|
return observations;
|
|
2822
3295
|
}
|
|
@@ -2832,10 +3305,11 @@ async function resumeStaged(run, url) {
|
|
|
2832
3305
|
if (state === undefined || state.url !== url || state.images === undefined || !sameRecordOptions(state.options, stage.options))
|
|
2833
3306
|
return undefined;
|
|
2834
3307
|
for (const obs of state.observations) {
|
|
2835
|
-
|
|
2836
|
-
|
|
3308
|
+
const blobs = [...obs.responses, ...(obs.resources ?? [])].map((entry) => entry.blob);
|
|
3309
|
+
for (const blob of blobs) {
|
|
3310
|
+
if (blob === undefined)
|
|
2837
3311
|
continue;
|
|
2838
|
-
if (!(await stat(join(stage.dir, "blobs",
|
|
3312
|
+
if (!(await stat(join(stage.dir, "blobs", blob)).then(() => true, () => false)))
|
|
2839
3313
|
return undefined;
|
|
2840
3314
|
}
|
|
2841
3315
|
}
|
|
@@ -2870,6 +3344,7 @@ async function withBrowser(opts, gpu, body) {
|
|
|
2870
3344
|
const deps = {
|
|
2871
3345
|
score: makeScorer("ssimulacra2", { projectDir: opts.cwd, gpu }),
|
|
2872
3346
|
cache,
|
|
3347
|
+
resourceRun: newResourceRun((opts.resourceTime ?? DEFAULT_RESOURCE_TIME_S) * 1000),
|
|
2873
3348
|
scorerId: gpu !== undefined ? `gpu:${gpu.deviceName}` : "cpu",
|
|
2874
3349
|
cpuScored: gpu !== undefined ? () => gpu.cpuScores : undefined,
|
|
2875
3350
|
};
|
|
@@ -2885,6 +3360,8 @@ async function withBrowser(opts, gpu, body) {
|
|
|
2885
3360
|
ignoreRobots: opts.ignoreRobots === true,
|
|
2886
3361
|
harness: opts.harness === true,
|
|
2887
3362
|
sourceMaps: new Map(),
|
|
3363
|
+
// The browser's own bytes of the flat images the vector check may trace (see FlatBuffers); none without the check.
|
|
3364
|
+
flatBuffers: new FlatBuffers(opts.quality && opts.survey !== true ? (opts.svgBudget ?? DEFAULT_SVG_BUDGET) : 0),
|
|
2888
3365
|
storageState: opts.storageState,
|
|
2889
3366
|
thirdParty: opts.thirdParty ?? DEFAULT_THIRD_PARTY,
|
|
2890
3367
|
qualityFor: opts.qualityFor,
|
|
@@ -2894,6 +3371,7 @@ async function withBrowser(opts, gpu, body) {
|
|
|
2894
3371
|
viewportConcurrency: opts.viewportConcurrency ?? VIEWPORT_CONCURRENCY,
|
|
2895
3372
|
idleTimeoutMs: opts.idleTimeoutMs ?? NETWORK_IDLE_TIMEOUT_MS,
|
|
2896
3373
|
sharedLoads: opts.sharedLoads === true,
|
|
3374
|
+
resources: opts.resources !== false,
|
|
2897
3375
|
estimates: opts.survey !== true,
|
|
2898
3376
|
log: opts.log,
|
|
2899
3377
|
gpu,
|
|
@@ -2980,7 +3458,9 @@ async function mediaPhaseFor(assets, run, opts, urls) {
|
|
|
2980
3458
|
if (opts.quality !== true || opts.survey === true) {
|
|
2981
3459
|
return { ...skippedMedia(sized, opts.survey === true ? "survey" : "quality off"), lengths };
|
|
2982
3460
|
}
|
|
2983
|
-
|
|
3461
|
+
// A video no viewport shows is not encoded: its record keeps the probe's bytes, and the reason is printed (see splitNeverShown).
|
|
3462
|
+
const { encode, neverShown } = splitNeverShown(sized);
|
|
3463
|
+
const encoded = await runMediaPhase(sized, {
|
|
2984
3464
|
budget: opts.mediaBudget ?? DEFAULT_MEDIA_BUDGET,
|
|
2985
3465
|
timeS: opts.mediaTime ?? DEFAULT_MEDIA_TIME,
|
|
2986
3466
|
vmafBar: opts.videoBar ?? DEFAULT_VIDEO_BAR,
|
|
@@ -2998,7 +3478,8 @@ async function mediaPhaseFor(assets, run, opts, urls) {
|
|
|
2998
3478
|
thirdParty: run.thirdParty,
|
|
2999
3479
|
}, {
|
|
3000
3480
|
fetch: (url, maxBytes, hop) => run.fetcher.get(url, { maxBytes: maxBytes ?? MAX_IMAGE_BYTES, allowPrivate, allowHop: hop?.allowHop, allowHopKey: hop?.page }),
|
|
3001
|
-
})
|
|
3481
|
+
});
|
|
3482
|
+
return { ...encoded, byKey: new Map([...encoded.byKey, ...skippedMedia(neverShown, NEVER_SHOWN).byKey]), lengths };
|
|
3002
3483
|
}
|
|
3003
3484
|
catch (err) {
|
|
3004
3485
|
const message = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 200);
|
|
@@ -3024,13 +3505,16 @@ export async function runScan(urls, opts) {
|
|
|
3024
3505
|
return withScanRun(opts, async (browser, run, robotsFor) => {
|
|
3025
3506
|
await mkdir(opts.outDir, { recursive: true });
|
|
3026
3507
|
// Whether ffmpeg is installed decides whether the media encodes run, so it is part of the record (see recordOptionsOf).
|
|
3027
|
-
const
|
|
3508
|
+
const vectorizer = (await loadVectorizer()) !== undefined;
|
|
3509
|
+
const options = recordOptionsOf(opts, (await probeFfmpeg()).ffmpeg, vectorizer);
|
|
3028
3510
|
run.stage = { dir: stageDirOf(opts.outDir), options };
|
|
3029
3511
|
const results = new Array(urls.length);
|
|
3030
3512
|
// The finished pages in finishing order: the progress line counts them.
|
|
3031
3513
|
const finished = [];
|
|
3032
3514
|
const registry = new EstimateRegistry();
|
|
3033
3515
|
const collector = registry.collector();
|
|
3516
|
+
// The resource analyses run in this pool while later pages are measured (see resources/pool.ts).
|
|
3517
|
+
const resourcePool = new ResourcePool(RESOURCE_PARALLEL, (error) => opts.log(`resources: a page's analysis failed: ${safeText(String(error))}`));
|
|
3034
3518
|
const measured = [];
|
|
3035
3519
|
// The media assets (videos and animated images) of every measured page, from its collecting pass (see media-phase.ts).
|
|
3036
3520
|
const mediaAssets = [];
|
|
@@ -3058,7 +3542,7 @@ export async function runScan(urls, opts) {
|
|
|
3058
3542
|
// A page measured by an interrupted run: its estimates only. Its requests are collected as a measured page's are.
|
|
3059
3543
|
const staged = opts.resume ? await resumeStaged(run, url) : undefined;
|
|
3060
3544
|
if (staged !== undefined) {
|
|
3061
|
-
const stagedProvisional = await staged.page.analyse(collector);
|
|
3545
|
+
const stagedProvisional = await staged.page.analyse(collector, false);
|
|
3062
3546
|
mediaAssets.push(...mediaAssetsOf(stagedProvisional));
|
|
3063
3547
|
measured.push({ index, url, page: staged.page });
|
|
3064
3548
|
images += staged.images;
|
|
@@ -3079,8 +3563,14 @@ export async function runScan(urls, opts) {
|
|
|
3079
3563
|
}
|
|
3080
3564
|
// The collecting pass: the page's estimate requests are recorded; its own numbers are provisional and replaced below.
|
|
3081
3565
|
// Staged as soon as it is measured: the collecting pass reads it back from the stage, so no bytes are held meanwhile.
|
|
3566
|
+
// The resource analysis starts now, pooled, so it overlaps the next pages' measurement. It reads the observations before the
|
|
3567
|
+
// stage drops them from memory.
|
|
3568
|
+
if (outcome.resources !== undefined) {
|
|
3569
|
+
await resourcePool.admit();
|
|
3570
|
+
void resourcePool.start(outcome.resources);
|
|
3571
|
+
}
|
|
3082
3572
|
await outcome.stage?.();
|
|
3083
|
-
const provisional = await outcome.analyse(collector);
|
|
3573
|
+
const provisional = await outcome.analyse(collector, false);
|
|
3084
3574
|
mediaAssets.push(...mediaAssetsOf(provisional));
|
|
3085
3575
|
await outcome.countImages?.(provisional.counts?.images ?? 0);
|
|
3086
3576
|
measured.push({ index, url, page: outcome });
|
|
@@ -3104,6 +3594,22 @@ export async function runScan(urls, opts) {
|
|
|
3104
3594
|
analysedPages.push({ index, url, page, analysed, fallbacks: tally.fallbacks });
|
|
3105
3595
|
opts.log(progressLine([...finished, ...analysedPages.map((p) => p.analysed)], urls.length));
|
|
3106
3596
|
}
|
|
3597
|
+
// The vector check runs once over the flat images of every page (see vector-phase.ts), before the media phase.
|
|
3598
|
+
const allowPrivate = urls.length > 0 && urls.every((url) => isLocalTarget(url));
|
|
3599
|
+
const vector = await runVectorPhase(analysedPages.map((p) => p.analysed), {
|
|
3600
|
+
budget: opts.svgBudget ?? DEFAULT_SVG_BUDGET,
|
|
3601
|
+
survey: opts.survey === true,
|
|
3602
|
+
quality: opts.quality,
|
|
3603
|
+
outDir: resolve(opts.outDir),
|
|
3604
|
+
cwd: opts.cwd,
|
|
3605
|
+
cache: run.deps.cache,
|
|
3606
|
+
log: opts.log,
|
|
3607
|
+
}, {
|
|
3608
|
+
// The browser's own bytes when they were kept (the same file the page got); else fetched, as before.
|
|
3609
|
+
fetch: async (url) => run.flatBuffers.get(url) ?? run.fetcher.get(url, { maxBytes: MAX_IMAGE_BYTES, allowPrivate }),
|
|
3610
|
+
});
|
|
3611
|
+
for (const entry of analysedPages)
|
|
3612
|
+
entry.analysed = applyVector(entry.analysed, vector);
|
|
3107
3613
|
// The media phase runs once for every page's assets (see runMediaFor); each page is then written again, complete.
|
|
3108
3614
|
const media = await runMediaFor(mediaAssets, run, opts, urls);
|
|
3109
3615
|
for (const { index, url, page, analysed, fallbacks } of analysedPages) {
|
|
@@ -3245,7 +3751,20 @@ export function renderPage(result, aggressive, hintsFile) {
|
|
|
3245
3751
|
if (bg) {
|
|
3246
3752
|
lines.push(` CSS backgrounds (not in the lines above): ${bg.layers} layer(s), ${kb(bg.servedBytes)}/view served, ${kb(bg.estSaved)}/view est. saving, ${bg.undersized} undersized`);
|
|
3247
3753
|
}
|
|
3248
|
-
|
|
3754
|
+
// The Loaded but not shown group: the images the browser downloaded that no viewport shows in full, by bytes per view.
|
|
3755
|
+
if (result.hidden !== undefined) {
|
|
3756
|
+
lines.push(` ${hiddenSummaryLine(result.hidden) ?? ""}`);
|
|
3757
|
+
const hiddenRecords = (result.images ?? []).filter((r) => r.hidden !== undefined);
|
|
3758
|
+
const rows = hiddenRecords.sort((a, b) => (b.hidden?.perViewBytes ?? 0) - (a.hidden?.perViewBytes ?? 0)).slice(0, HIDDEN_ROWS);
|
|
3759
|
+
for (const record of rows) {
|
|
3760
|
+
const hidden = record.hidden;
|
|
3761
|
+
const where = record.locator.file !== undefined ? `${record.locator.file}:${record.locator.line ?? 0}` : record.locator.selector;
|
|
3762
|
+
lines.push(` ${JSON.stringify(safeText(where))} hidden at ${hidden.viewports.length} of ${hidden.of} ${kb(hidden.perViewBytes)}/view ${hidden.action}`);
|
|
3763
|
+
}
|
|
3764
|
+
if (hiddenRecords.length > rows.length)
|
|
3765
|
+
lines.push(` (+${hiddenRecords.length - rows.length} more: see imgtune hints)`);
|
|
3766
|
+
}
|
|
3767
|
+
const hinted = (result.images ?? []).filter((r) => hasFix(r) || r.hidden !== undefined).length;
|
|
3249
3768
|
if (hinted > 0 && hintsFile !== undefined)
|
|
3250
3769
|
lines.push(` ${hinted} images with fix hints: run \`imgtune hints ${safeText(hintsFile)}\``);
|
|
3251
3770
|
if (result.groups.length > 0) {
|
|
@@ -3256,7 +3775,8 @@ export function renderPage(result, aggressive, hintsFile) {
|
|
|
3256
3775
|
const loader = g.loader ? ` loader: ${safeText(g.loader)}` : "";
|
|
3257
3776
|
const sizes = g.sizes === undefined ? "no sizes" : safeText(g.sizes);
|
|
3258
3777
|
const suggested = g.suggestedSizes === undefined ? "-" : safeText(g.suggestedSizes);
|
|
3259
|
-
|
|
3778
|
+
const head = g.srcset !== undefined ? srcsetGroupText(g.srcset.widths) : `${sizes} -> ${suggested}`;
|
|
3779
|
+
lines.push(` ${head} saves ${kb(g.savedBytes)}/view${loader}${hint}${qualityHint}`);
|
|
3260
3780
|
}
|
|
3261
3781
|
}
|
|
3262
3782
|
// The Video section: the page's <video> files, per view and per play, with each one's encode or the reason it has none.
|
|
@@ -3267,8 +3787,16 @@ export function renderPage(result, aggressive, hintsFile) {
|
|
|
3267
3787
|
for (const video of videos)
|
|
3268
3788
|
lines.push(` ${videoRowOf(video, (text) => JSON.stringify(safeText(text)))}`);
|
|
3269
3789
|
}
|
|
3790
|
+
// The Resources section: the page's non-image findings, ranked by saving (see resources/).
|
|
3791
|
+
lines.push(...resourceLinesOf(result.resources ?? [], (text) => safeText(text)));
|
|
3270
3792
|
return lines;
|
|
3271
3793
|
}
|
|
3794
|
+
/** The most loaded-but-not-shown images the page's text output lists (the rest are in the hints). */
|
|
3795
|
+
const HIDDEN_ROWS = 10;
|
|
3796
|
+
/** The top-groups line of a plain image group with no srcset: what to serve, in place of a sizes change. */
|
|
3797
|
+
export function srcsetGroupText(widths) {
|
|
3798
|
+
return `no srcset -> serve ${srcsetWidthLabel(widths)} + srcset/sizes`;
|
|
3799
|
+
}
|
|
3272
3800
|
/** Per-site line for hosts with two or more scanned pages. */
|
|
3273
3801
|
export function renderScan(results, aggressive, hintsFile, maxQualities) {
|
|
3274
3802
|
const lines = [];
|
|
@@ -3282,6 +3810,9 @@ export function renderScan(results, aggressive, hintsFile, maxQualities) {
|
|
|
3282
3810
|
const media = results.find((r) => r.mediaNote !== undefined)?.mediaNote;
|
|
3283
3811
|
if (media !== undefined)
|
|
3284
3812
|
lines.push(safeText(media), "");
|
|
3813
|
+
const vectorNote = results.find((r) => r.vectorNote !== undefined)?.vectorNote;
|
|
3814
|
+
if (vectorNote !== undefined)
|
|
3815
|
+
lines.push(safeText(vectorNote), "");
|
|
3285
3816
|
for (const result of results)
|
|
3286
3817
|
lines.push(...renderPage(result, aggressive, hintsFile?.(result)), "");
|
|
3287
3818
|
const byHost = new Map();
|
|
@@ -3365,6 +3896,9 @@ function markdownBody(results, aggressive, withHints, maxQualities) {
|
|
|
3365
3896
|
const media = results.find((r) => r.mediaNote !== undefined)?.mediaNote;
|
|
3366
3897
|
if (media !== undefined)
|
|
3367
3898
|
lines.push(mdText(media), "");
|
|
3899
|
+
const vectorNote = results.find((r) => r.vectorNote !== undefined)?.vectorNote;
|
|
3900
|
+
if (vectorNote !== undefined)
|
|
3901
|
+
lines.push(mdText(vectorNote), "");
|
|
3368
3902
|
lines.push(mdRow(head, width), mdRow(align, width));
|
|
3369
3903
|
for (const result of results) {
|
|
3370
3904
|
const label = pageLabel(result.url);
|
|
@@ -3394,8 +3928,8 @@ function markdownBody(results, aggressive, withHints, maxQualities) {
|
|
|
3394
3928
|
lines.push(mdRow(["current sizes", "suggested", "saves KB/view", "loader", "hint"], 5));
|
|
3395
3929
|
lines.push(mdRow(["---", "---", "---:", "---", "---"], 5));
|
|
3396
3930
|
for (const g of result.groups.slice(0, SUMMARY_GROUPS)) {
|
|
3397
|
-
const current = g.sizes === undefined ? "no sizes" : mdCode(g.sizes);
|
|
3398
|
-
const suggested = g.suggestedSizes === undefined ? "-" : mdCode(g.suggestedSizes);
|
|
3931
|
+
const current = g.srcset !== undefined ? "no srcset" : g.sizes === undefined ? "no sizes" : mdCode(g.sizes);
|
|
3932
|
+
const suggested = g.srcset !== undefined ? mdText(`serve ${srcsetWidthLabel(g.srcset.widths)} + srcset/sizes`) : g.suggestedSizes === undefined ? "-" : mdCode(g.suggestedSizes);
|
|
3399
3933
|
const loader = g.loader ? mdCode(g.loader) : "-";
|
|
3400
3934
|
const hint = [g.hint, g.qualityHint].filter((h) => h !== undefined).map((h) => mdCode(h)).join("; ");
|
|
3401
3935
|
lines.push(mdRow([current, suggested, `${kbNumber(g.savedBytes)}${g.estApprox === true ? APPROX_MARK : ""}`, loader, hint || "-"], 5));
|
|
@@ -3408,9 +3942,25 @@ function markdownBody(results, aggressive, withHints, maxQualities) {
|
|
|
3408
3942
|
lines.push(`- ${mdCode(videoRowOf(video, (text) => JSON.stringify(safeText(text))))}`);
|
|
3409
3943
|
}
|
|
3410
3944
|
}
|
|
3945
|
+
// Resources (see resources/): the totals, then the largest findings. They are not in the image totals above.
|
|
3946
|
+
const resourceRows = results.filter((r) => r.status === "ok").flatMap((r) => r.resources ?? []);
|
|
3947
|
+
const resourceTotals = resourceTotalsLine(resourceRows);
|
|
3948
|
+
if (resourceTotals !== undefined) {
|
|
3949
|
+
lines.push("", mdText(resourceTotals));
|
|
3950
|
+
const largest = [...resourceRows].sort((a, b) => b.savedBytes - a.savedBytes).slice(0, RESOURCE_MARKDOWN_ROWS);
|
|
3951
|
+
for (const finding of largest) {
|
|
3952
|
+
const saved = finding.savedBytes > 0 ? ` saves ${kbNumber(finding.savedBytes)} KB${finding.estimate ? APPROX_MARK : ""}` : "";
|
|
3953
|
+
lines.push(`- ${mdCode(`${finding.kind}: ${safeText(finding.title)}`)}${saved}`);
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
3411
3956
|
const plan = planQualityCap(results, maxQualities);
|
|
3412
3957
|
if (plan !== undefined)
|
|
3413
3958
|
lines.push("", mdCode(qualitiesLine(plan)));
|
|
3959
|
+
const hiddenLine = hiddenPagesLine(results);
|
|
3960
|
+
const hiddenVideoLine = hiddenVideoPagesLine(results);
|
|
3961
|
+
const hiddenRows = [hiddenLine, hiddenVideoLine].flatMap((line) => (line !== undefined ? [mdCode(line)] : []));
|
|
3962
|
+
if (hiddenRows.length > 0)
|
|
3963
|
+
lines.push("", ...hiddenRows);
|
|
3414
3964
|
if (withHints) {
|
|
3415
3965
|
const hints = commentHints(results, undefined, maxQualities);
|
|
3416
3966
|
if (hints.lines.length > 0) {
|
|
@@ -3530,6 +4080,8 @@ export async function scanCommand(argv, cwd, log, signal) {
|
|
|
3530
4080
|
maxPages: args.maxPages,
|
|
3531
4081
|
maxImages: args.maxImages,
|
|
3532
4082
|
mediaBudget: args.mediaBudget,
|
|
4083
|
+
svgBudget: args.svgBudget,
|
|
4084
|
+
resourceTime: args.resourceTime,
|
|
3533
4085
|
mediaTime: args.mediaTime,
|
|
3534
4086
|
videoBar: args.videoBar,
|
|
3535
4087
|
videoStillBar: args.videoStillBar,
|
|
@@ -3541,6 +4093,7 @@ export async function scanCommand(argv, cwd, log, signal) {
|
|
|
3541
4093
|
gpu: args.gpu,
|
|
3542
4094
|
gpuEncode: args.gpuEncode,
|
|
3543
4095
|
cache: args.noCache !== true,
|
|
4096
|
+
resources: args.resources !== false,
|
|
3544
4097
|
report,
|
|
3545
4098
|
thirdParty: args.thirdParty,
|
|
3546
4099
|
});
|