imgtune 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +172 -0
- package/README.md +109 -5
- package/dist/animated.js +176 -0
- package/dist/audit.js +1 -0
- package/dist/backgrounds.js +42 -7
- package/dist/broken.js +5 -1
- package/dist/browser.js +4 -4
- package/dist/candidates.js +25 -0
- package/dist/check-report.js +56 -6
- package/dist/check.js +26 -5
- package/dist/cli.js +86 -8
- package/dist/egress-proxy.js +2 -1
- package/dist/fetch.js +9 -4
- package/dist/ffmpeg.js +530 -0
- package/dist/gpu.js +12 -0
- package/dist/hints.js +704 -16
- package/dist/image-file.js +154 -34
- package/dist/image-open.js +9 -0
- package/dist/images.js +142 -8
- package/dist/layout.js +7 -7
- package/dist/lcp.js +2 -1
- package/dist/locate-join.js +86 -0
- package/dist/locate-only.js +14 -3
- package/dist/locate.js +6 -2
- package/dist/measure.js +39 -1
- package/dist/media-encode.js +1054 -0
- package/dist/media-file.js +284 -0
- package/dist/media-limits.js +31 -0
- package/dist/media-model.js +251 -0
- package/dist/media-phase.js +675 -0
- package/dist/media-prepare.js +31 -0
- package/dist/px.js +11 -0
- package/dist/recheck.js +12 -4
- package/dist/remote.js +1065 -107
- package/dist/replay.js +20 -0
- package/dist/resources/audio.js +248 -0
- package/dist/resources/brotli.js +22 -0
- package/dist/resources/cache.js +59 -0
- package/dist/resources/capture.js +225 -0
- package/dist/resources/compression.js +122 -0
- package/dist/resources/favicon.js +130 -0
- package/dist/resources/font.js +440 -0
- package/dist/resources/index.js +8 -0
- package/dist/resources/lottie.js +491 -0
- package/dist/resources/model.js +478 -0
- package/dist/resources/phase.js +171 -0
- package/dist/resources/pool.js +47 -0
- package/dist/resources/report.js +57 -0
- package/dist/resources/sfnt.js +305 -0
- package/dist/resources/svg.js +178 -0
- package/dist/resources/types.js +1 -0
- package/dist/resources/zip.js +67 -0
- package/dist/result-cache.js +56 -3
- package/dist/run.js +32 -8
- package/dist/sizes.js +7 -5
- package/dist/summary.js +69 -3
- package/dist/vector-art.js +294 -0
- package/dist/vector-phase.js +222 -0
- package/dist/video-analysis.js +371 -0
- package/dist/webp-animation.js +283 -0
- package/package.json +25 -4
- package/skills/imgtune/SKILL.md +8 -4
- package/skills/imgtune/reference.md +1 -1
package/dist/remote.js
CHANGED
|
@@ -5,14 +5,18 @@ import { dirname, join, resolve } from "node:path";
|
|
|
5
5
|
import { launchChromium } from "./browser.js";
|
|
6
6
|
import sharp from "sharp";
|
|
7
7
|
import { writeFileAtomic, sha1 } from "./atomic.js";
|
|
8
|
+
import { MAX_INPUT_PIXELS, openImage } from "./image-open.js";
|
|
8
9
|
import { errorLine, mdCode, mdText, safeText } from "./text.js";
|
|
9
|
-
import { commentHints } from "./hints.js";
|
|
10
|
+
import { hiddenPagesLine, hiddenVideoPagesLine, commentHints, hiddenSummaryLine, videoRowOf, videoTotalsLine } from "./hints.js";
|
|
10
11
|
import { lcpLinesOf } from "./lcp.js";
|
|
11
12
|
import { MAX_MAX_QUALITIES, MIN_MAX_QUALITIES, planQualityCap, qualitiesLine } from "./qualities-cap.js";
|
|
12
13
|
export { mdCode, mdText, safeText };
|
|
14
|
+
export { MAX_INPUT_PIXELS, openImage };
|
|
13
15
|
import { isLocalTarget, systemResolver } from "./address.js";
|
|
16
|
+
import { loadVectorizer, stillFlatness } from "./vector-art.js";
|
|
17
|
+
import { applyVector, FlatBuffers, runVectorPhase } from "./vector-phase.js";
|
|
14
18
|
import { startEgressProxy } from "./egress-proxy.js";
|
|
15
|
-
import { makeFetcher } from "./fetch.js";
|
|
19
|
+
import { makeFetcher, MAX_IMAGE_BYTES } from "./fetch.js";
|
|
16
20
|
import { sameSite } from "./site.js";
|
|
17
21
|
import { confirmMatch, matchLocalQuality } from "./calibrate.js";
|
|
18
22
|
import { DEFAULT_THIRD_PARTY, isThirdPartyBlocked, parseAllowHosts, thirdPartyFromFlags } from "./third-party.js";
|
|
@@ -20,17 +24,29 @@ import { DEFAULT_VIEWPORTS } from "./config.js";
|
|
|
20
24
|
import { encode, encodedBytes, ENCODE_THREADS, ENCODER_ID, FAST_AVIF_EFFORT } from "./encode.js";
|
|
21
25
|
import { adapterFor, formatAutoOf, originalOf } from "./loaders.js";
|
|
22
26
|
import { COMPONENT_NAME, MAX_SOURCE_READ, imageAtLine, parseSourceMap, resolveFunctionLocations, resolveOwnerChain, resolveOwners } from "./locate.js";
|
|
23
|
-
import { buildImageRecords, hasFix, rungOfPage } from "./images.js";
|
|
27
|
+
import { buildImageRecords, hasFix, hiddenTotalsOf, isPlainNoSrcset, rungOfPage } from "./images.js";
|
|
28
|
+
import { animationOf } from "./animated.js";
|
|
29
|
+
import { fullBytesOf, SCAN_MAX_VIDEO_WIDTHS, videoRecordsOf, withPosterImages } from "./video-analysis.js";
|
|
24
30
|
import { backgroundPage, backgroundTotals, buildBackgroundRecords, displayedCssWidth, MAX_BACKGROUND_ELEMENTS, MAX_BACKGROUND_VALUE, backgroundTooLarge, neededDevicePixels, parseBackgroundImage, parseBackgroundSize, pickLoaded, } from "./backgrounds.js";
|
|
25
31
|
import { applySizesOutcome } from "./sizes-outcome.js";
|
|
26
32
|
import { sampleByPattern, urlsFromSource } from "./urls.js";
|
|
27
33
|
import { groupRoutes, parseRouteMap } from "./routes.js";
|
|
28
34
|
import { countDocumentNavigations, gotoParsed, NETWORK_IDLE_TIMEOUT_MS, settlePage, settleWithin, waitForLoadWithin, BODY_READ_TIMEOUT_MS } from "./measure.js";
|
|
29
35
|
import { makeScorer } from "./metric.js";
|
|
30
|
-
import { gpuNote,
|
|
36
|
+
import { gpuNote, startGpuScorer, gpuSettingOf, gpuSettingOne } from "./gpu.js";
|
|
31
37
|
import { bisectQuality, confirmBisect, minPassingQuality } from "./search.js";
|
|
32
38
|
import { SURVEY_IDLE_TIMEOUT_MS, SURVEY_PAGE_CONCURRENCY, SURVEY_VIEWPORTS, surveyNote } from "./survey.js";
|
|
33
39
|
import { cacheScope, CACHE_DIR, CACHE_VERSION, contentHash, ResultCache, resizeScope } from "./result-cache.js";
|
|
40
|
+
import { attachMediaResults, mediaAssetsOf, NEVER_SHOWN, partitionByPolicy, probeLengths, refineAssetLength, runMediaPhase, skippedMedia, splitNeverShown, } from "./media-phase.js";
|
|
41
|
+
import { analyzeResources } from "./resources/phase.js";
|
|
42
|
+
import { replayHarPath } from "./replay.js";
|
|
43
|
+
import { ResourcePool } from "./resources/pool.js";
|
|
44
|
+
import { RESOURCE_BODY_SETTLE_MS } from "./resources/capture.js";
|
|
45
|
+
import { DEFAULT_RESOURCE_BUDGET_MS, newResourceRun } from "./resources/cache.js";
|
|
46
|
+
import { resourceLinesOf, resourceTotalsLine } from "./resources/report.js";
|
|
47
|
+
/** Resource findings listed in the PR-comment Markdown (the JSON and the text output have more). */
|
|
48
|
+
const RESOURCE_MARKDOWN_ROWS = 5;
|
|
49
|
+
import { readPageContext, readResourceBody, resourceRecordOf } from "./resources/capture.js";
|
|
34
50
|
import { BROKEN_EXIT, withBroken } from "./broken.js";
|
|
35
51
|
import { EstimateRegistry, hashOf, withEstimates } from "./estimate-jobs.js";
|
|
36
52
|
import { isFolder, sourceRootOf, unreadSourceNote } from "./source-root.js";
|
|
@@ -41,6 +57,7 @@ import { count, measure, renderTiming, resetTiming, TIMING_ENABLED } from "./tim
|
|
|
41
57
|
import { mapLimit, settleLimit } from "./concurrency.js";
|
|
42
58
|
import { getBlob, putBlob, readStagedJson, removeStage, stageDirOf, writeStagedJson } from "./stage.js";
|
|
43
59
|
import { loadGroups } from "./loads.js";
|
|
60
|
+
import { probeFfmpeg } from "./ffmpeg.js";
|
|
44
61
|
export { ipv6Bytes, isBlockedAddress, isLocalTarget, isPrivateAddress, refusedAddress, guardedLookup } from "./address.js";
|
|
45
62
|
export { makeFetcher, MAX_IMAGE_BYTES } from "./fetch.js";
|
|
46
63
|
export { registrableDomain, sameSite } from "./site.js";
|
|
@@ -71,7 +88,15 @@ export const MAX_PAGE_CONCURRENCY = 6;
|
|
|
71
88
|
/** The longest --idle-timeout: ten minutes (a timer past 2^31 ms would fire at once). */
|
|
72
89
|
export const MAX_IDLE_TIMEOUT_MS = 600_000;
|
|
73
90
|
const CHALLENGE_STATUS = [403, 429, 503];
|
|
74
|
-
const CHALLENGE_MARKERS = ["cf-chl", "captcha"];
|
|
91
|
+
const CHALLENGE_MARKERS = ["cf-chl", "captcha", "just a moment", "captcha-delivery", "awswaf"];
|
|
92
|
+
/**
|
|
93
|
+
* Response headers a WAF or CDN sends with its challenge page. They decide whatever the page has in it: a challenge page can
|
|
94
|
+
* carry a few images (S071 had 3 of a page's 148), so the image count does not clear them.
|
|
95
|
+
*/
|
|
96
|
+
const CHALLENGE_HEADERS = [
|
|
97
|
+
{ name: "x-amzn-waf-action", values: ["challenge", "captcha"] },
|
|
98
|
+
{ name: "cf-mitigated", values: ["challenge"] },
|
|
99
|
+
];
|
|
75
100
|
/** Quality assumed for an image whose URL does not say (next/image's default). */
|
|
76
101
|
export const ASSUMED_QUALITY = 75;
|
|
77
102
|
/** Rules longer than this are ignored; a robots.txt is read up to MAX_ROBOTS_BYTES. */
|
|
@@ -161,7 +186,7 @@ function decidingRule(robotsTxt, target) {
|
|
|
161
186
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
162
187
|
// Resolution: srcset candidates and the right one
|
|
163
188
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
164
|
-
import { UNDERSIZED_RATIO, isUndersized, neededWidth, rightCandidate, undersizedProposal } from "./candidates.js";
|
|
189
|
+
import { UNDERSIZED_RATIO, isUndersized, neededWidth, rightCandidate, srcsetWidthLabel, srcsetWidthsOf, undersizedProposal } from "./candidates.js";
|
|
165
190
|
export { isUndersized, neededWidth, rightCandidate } from "./candidates.js";
|
|
166
191
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
167
192
|
// References and the current quality of a URL
|
|
@@ -304,7 +329,11 @@ export function groupKey(sizes, classes, refWidth) {
|
|
|
304
329
|
return [sizes ?? "-", cls || "-", bucket].join("|");
|
|
305
330
|
}
|
|
306
331
|
/** A page is a challenge when the document is refused or throttled, or it has no image and shows a challenge marker. */
|
|
307
|
-
export function isChallenge(status, imageCount, html) {
|
|
332
|
+
export function isChallenge(status, imageCount, html, headers = {}) {
|
|
333
|
+
for (const { name, values } of CHALLENGE_HEADERS) {
|
|
334
|
+
if (values.includes((headers[name] ?? "").toLowerCase()))
|
|
335
|
+
return true;
|
|
336
|
+
}
|
|
308
337
|
if (status !== undefined && CHALLENGE_STATUS.includes(status))
|
|
309
338
|
return true;
|
|
310
339
|
if (imageCount > 0)
|
|
@@ -369,6 +398,12 @@ const VALUE_OPTIONS = [
|
|
|
369
398
|
"--max-qualities",
|
|
370
399
|
"--page-concurrency",
|
|
371
400
|
"--idle-timeout",
|
|
401
|
+
"--media-budget",
|
|
402
|
+
"--svg-budget",
|
|
403
|
+
"--media-time",
|
|
404
|
+
"--video-bar",
|
|
405
|
+
"--video-still-bar",
|
|
406
|
+
"--max-video-widths",
|
|
372
407
|
"--urls-from",
|
|
373
408
|
"--sample-per-pattern",
|
|
374
409
|
"--route-map",
|
|
@@ -390,6 +425,22 @@ export function parseRange(name, text, min, max) {
|
|
|
390
425
|
}
|
|
391
426
|
return value;
|
|
392
427
|
}
|
|
428
|
+
/** The media phase's limits and defaults (see media-phase.ts and the --media-* flags). */
|
|
429
|
+
export const DEFAULT_MEDIA_BUDGET = 20;
|
|
430
|
+
const MAX_MEDIA_BUDGET = 1000;
|
|
431
|
+
export const DEFAULT_MEDIA_TIME = 600;
|
|
432
|
+
const MAX_MEDIA_TIME = 3600;
|
|
433
|
+
export const DEFAULT_VIDEO_BAR = 93;
|
|
434
|
+
export const DEFAULT_VIDEO_STILL_BAR = 65;
|
|
435
|
+
/** The vector check's budget: the flat images traced per run, heaviest first (--svg-budget). */
|
|
436
|
+
export const DEFAULT_SVG_BUDGET = 50;
|
|
437
|
+
export const MAX_SVG_BUDGET = 1000;
|
|
438
|
+
/** The longest --resource-time, in seconds. */
|
|
439
|
+
export const MAX_RESOURCE_TIME = 3600;
|
|
440
|
+
/** The resource analyses of a scan that run at once, in the phase-1 pool (see resources/pool.ts). */
|
|
441
|
+
const RESOURCE_PARALLEL = 2;
|
|
442
|
+
/** The default --resource-time, in seconds: the heavy resource work of a run (see resources/cache.ts). */
|
|
443
|
+
export const DEFAULT_RESOURCE_TIME_S = DEFAULT_RESOURCE_BUDGET_MS / 1000;
|
|
393
444
|
/** The arguments after `scan`: one or more page URLs and the options. Throws with a message the CLI prints. */
|
|
394
445
|
export function parseScanArgs(argv) {
|
|
395
446
|
const args = { urls: [], out: DEFAULT_OUT, aggressive: false, quality: true, thirdParty: DEFAULT_THIRD_PARTY };
|
|
@@ -411,9 +462,13 @@ export function parseScanArgs(argv) {
|
|
|
411
462
|
else if (arg === "--resume")
|
|
412
463
|
args.resume = true;
|
|
413
464
|
else if (arg === "--gpu")
|
|
414
|
-
args.gpu ??=
|
|
465
|
+
args.gpu ??= gpuSettingOne(0);
|
|
415
466
|
else if (arg === "--no-cache")
|
|
416
467
|
args.noCache = true;
|
|
468
|
+
else if (arg === "--no-resources")
|
|
469
|
+
args.resources = false;
|
|
470
|
+
else if (arg === "--gpu-encode")
|
|
471
|
+
args.gpuEncode = true;
|
|
417
472
|
else if (arg === "--harness")
|
|
418
473
|
args.harness = true;
|
|
419
474
|
else if (arg === "--survey")
|
|
@@ -447,6 +502,20 @@ export function parseScanArgs(argv) {
|
|
|
447
502
|
args.pageConcurrency = parseRange(arg, value, 1, MAX_PAGE_CONCURRENCY);
|
|
448
503
|
else if (arg === "--idle-timeout")
|
|
449
504
|
args.idleTimeoutMs = parseRange(arg, value, 0, MAX_IDLE_TIMEOUT_MS);
|
|
505
|
+
else if (arg === "--media-budget")
|
|
506
|
+
args.mediaBudget = parseRange(arg, value, 0, MAX_MEDIA_BUDGET);
|
|
507
|
+
else if (arg === "--svg-budget")
|
|
508
|
+
args.svgBudget = parseRange(arg, value, 0, MAX_SVG_BUDGET);
|
|
509
|
+
else if (arg === "--resource-time")
|
|
510
|
+
args.resourceTime = parseRange(arg, value, 0, MAX_RESOURCE_TIME);
|
|
511
|
+
else if (arg === "--media-time")
|
|
512
|
+
args.mediaTime = parseRange(arg, value, 1, MAX_MEDIA_TIME);
|
|
513
|
+
else if (arg === "--video-bar")
|
|
514
|
+
args.videoBar = parseRange(arg, value, 0, 100);
|
|
515
|
+
else if (arg === "--video-still-bar")
|
|
516
|
+
args.videoStillBar = parseRange(arg, value, 0, 100);
|
|
517
|
+
else if (arg === "--max-video-widths")
|
|
518
|
+
args.maxVideoWidths = parseRange(arg, value, 1, 3);
|
|
450
519
|
else if (arg === "--urls-from")
|
|
451
520
|
args.urlsFrom = value;
|
|
452
521
|
else if (arg === "--sample-per-pattern")
|
|
@@ -456,7 +525,7 @@ export function parseScanArgs(argv) {
|
|
|
456
525
|
else if (arg === "--storage-state")
|
|
457
526
|
args.storageState = value;
|
|
458
527
|
else if (arg === "--gpu-device")
|
|
459
|
-
args.gpu =
|
|
528
|
+
args.gpu = gpuSettingOf(value);
|
|
460
529
|
else if (arg === "--allow-host")
|
|
461
530
|
allowHosts = parseAllowHosts(value);
|
|
462
531
|
else
|
|
@@ -476,16 +545,10 @@ export function parseScanArgs(argv) {
|
|
|
476
545
|
args.thirdParty = thirdPartyFromFlags(blockThirdParty, allowHosts);
|
|
477
546
|
return args;
|
|
478
547
|
}
|
|
479
|
-
/** The most input pixels sharp accepts for a downloaded file. */
|
|
480
|
-
export const MAX_INPUT_PIXELS = 100_000_000;
|
|
481
548
|
/** A width clamped to 1..MAX_WIDTH. */
|
|
482
549
|
export function clampWidth(width) {
|
|
483
550
|
return Math.max(1, Math.min(MAX_WIDTH, Math.floor(width)));
|
|
484
551
|
}
|
|
485
|
-
/** sharp over a downloaded file, refusing inputs above `limit` pixels. */
|
|
486
|
-
export function openImage(buffer, limit = MAX_INPUT_PIXELS) {
|
|
487
|
-
return sharp(buffer, { limitInputPixels: limit });
|
|
488
|
-
}
|
|
489
552
|
/**
|
|
490
553
|
* sharp names an AVIF file "heif" (its compression is "av1"). Every other use in this file says "avif", so that is the
|
|
491
554
|
* name a pixel size carries. A HEIF with another compression (HEVC) keeps sharp's name and gets no estimate.
|
|
@@ -510,6 +573,26 @@ export async function pixelSize(buffer) {
|
|
|
510
573
|
return undefined;
|
|
511
574
|
}
|
|
512
575
|
}
|
|
576
|
+
/** The animation of a response's bytes, read once per buffer (see animated.ts). Undefined for a still image. */
|
|
577
|
+
const animationMemo = new WeakMap();
|
|
578
|
+
export function animationOfBuffer(buffer) {
|
|
579
|
+
let info = animationMemo.get(buffer);
|
|
580
|
+
if (info === undefined) {
|
|
581
|
+
info = animationOf(buffer) ?? null;
|
|
582
|
+
animationMemo.set(buffer, info);
|
|
583
|
+
}
|
|
584
|
+
return info ?? undefined;
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* The format the pixel rules judge a file by. A static GIF is a lossless palette file, so it takes the PNG rules (a
|
|
588
|
+
* local resize, and the AVIF that meets the file's own score). An animated GIF takes none of them: its saving is an
|
|
589
|
+
* encode to video or animated AVIF (see video-analysis.ts), so it keeps its own name and gets no estimate here. The
|
|
590
|
+
* bytes must really be a GIF (its signature), so a label alone never routes a file to the PNG rules.
|
|
591
|
+
*/
|
|
592
|
+
export function pixelRuleFormat(format, buffer) {
|
|
593
|
+
const isGif = buffer.toString("latin1", 0, 4) === "GIF8";
|
|
594
|
+
return format === "gif" && isGif && animationOfBuffer(buffer) === undefined ? "png" : format;
|
|
595
|
+
}
|
|
513
596
|
/** The file decoded to 8-bit RGBA; resized to `size` when given and different (the scorer needs equal dimensions). */
|
|
514
597
|
export async function decodeRaw(buffer, size) {
|
|
515
598
|
const { data, info } = await openImage(buffer).rotate().ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
@@ -613,7 +696,7 @@ export async function matchedQualityOf(buffer, format) {
|
|
|
613
696
|
*/
|
|
614
697
|
export async function resizedEstimate(buffer, format, requestedWidth, quality) {
|
|
615
698
|
const targetWidth = clampWidth(requestedWidth);
|
|
616
|
-
if (format === "png") {
|
|
699
|
+
if (pixelRuleFormat(format, buffer) === "png") {
|
|
617
700
|
const out = await openImage(buffer).rotate().resize({ width: targetWidth, withoutEnlargement: true }).png().toBuffer();
|
|
618
701
|
return { bytes: out.length, buffer: out };
|
|
619
702
|
}
|
|
@@ -784,7 +867,8 @@ export async function estimateQuality(input, deltas, deps) {
|
|
|
784
867
|
}
|
|
785
868
|
return { ratio, target };
|
|
786
869
|
}
|
|
787
|
-
|
|
870
|
+
const lossless = pixelRuleFormat(input.format, input.served);
|
|
871
|
+
if (lossless === "jpeg" || lossless === "png") {
|
|
788
872
|
const servedRaw = await measure("decode: served", () => decodeRaw(input.served, { width: reference.width, height: reference.height }));
|
|
789
873
|
const ownScore = await score(reference, servedRaw);
|
|
790
874
|
for (const delta of deltas) {
|
|
@@ -806,9 +890,91 @@ function isNavigationError(err) {
|
|
|
806
890
|
export const LAYOUT_DEPTH = 15;
|
|
807
891
|
/** React owners read per image, nearest first. */
|
|
808
892
|
export const MAX_OWNERS = 8;
|
|
809
|
-
/**
|
|
810
|
-
|
|
811
|
-
|
|
893
|
+
/**
|
|
894
|
+
* Whether a <video> source is out of scope: a blob or media-stream URL (made by the page, not a file), an adaptive stream
|
|
895
|
+
* (HLS .m3u8 or DASH .mpd, with or without a query string), or no source at all. readView applies the same rule in the
|
|
896
|
+
* browser, before a video leaves the page, so the rule is written there too.
|
|
897
|
+
*/
|
|
898
|
+
export function isOutOfScopeVideo(src) {
|
|
899
|
+
return src.trim() === "" || /^(blob|mediastream):/i.test(src) || /\.(m3u8|mpd)(?:[?#]|$)/i.test(src);
|
|
900
|
+
}
|
|
901
|
+
/**
|
|
902
|
+
* The bytes a media response carries, from its headers: Content-Length, else the span of Content-Range (`bytes a-b/total`
|
|
903
|
+
* is b - a + 1). Undefined when neither gives a length. The body is never read.
|
|
904
|
+
*/
|
|
905
|
+
export function mediaBytesOf(headers) {
|
|
906
|
+
const length = headers["content-length"];
|
|
907
|
+
if (length !== undefined && /^\d+$/.test(length.trim()))
|
|
908
|
+
return Number(length);
|
|
909
|
+
const range = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i.exec((headers["content-range"] ?? "").trim());
|
|
910
|
+
if (range && Number(range[2]) >= Number(range[1]))
|
|
911
|
+
return Number(range[2]) - Number(range[1]) + 1;
|
|
912
|
+
return undefined;
|
|
913
|
+
}
|
|
914
|
+
/**
|
|
915
|
+
* The start and the total of a Content-Range header (`bytes a-b/total`), when it has them. Undefined fields when the
|
|
916
|
+
* header is absent or does not parse (a 200 has none, so its bytes start at 0).
|
|
917
|
+
*/
|
|
918
|
+
export function contentRangeOf(headers) {
|
|
919
|
+
const range = /^bytes (\d+)-(\d+)\/(\d+|\*)$/i.exec((headers["content-range"] ?? "").trim());
|
|
920
|
+
if (!range)
|
|
921
|
+
return {};
|
|
922
|
+
const total = range[3] === "*" ? undefined : Number(range[3]);
|
|
923
|
+
return { start: Number(range[1]), ...(total !== undefined ? { total } : {}) };
|
|
924
|
+
}
|
|
925
|
+
/** Merges [start, end) ranges into the ascending, disjoint list that covers the same bytes. */
|
|
926
|
+
export function mergeRanges(ranges) {
|
|
927
|
+
const sorted = ranges.filter(([a, b]) => b > a).sort((x, y) => x[0] - y[0] || x[1] - y[1]);
|
|
928
|
+
const merged = [];
|
|
929
|
+
for (const [start, end] of sorted) {
|
|
930
|
+
const last = merged[merged.length - 1];
|
|
931
|
+
if (last && start <= last[1])
|
|
932
|
+
last[1] = Math.max(last[1], end);
|
|
933
|
+
else
|
|
934
|
+
merged.push([start, end]);
|
|
935
|
+
}
|
|
936
|
+
return merged;
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Adds one media response's bytes to the page load's map (see readGroup). The response covers [start, start + bytes) (a
|
|
940
|
+
* 200 starts at 0), so a re-request of bytes already received adds nothing: the figure is the union of the ranges, capped
|
|
941
|
+
* at the file's total when one is known. The status and content type are the last ones seen. Every media response counts,
|
|
942
|
+
* a 200 as well as a 206 (a server that ignores Range sends the whole file, and the bytes the page moved are the union).
|
|
943
|
+
*/
|
|
944
|
+
export function addMediaResponse(media, response) {
|
|
945
|
+
const prev = media.get(response.url);
|
|
946
|
+
const start = response.start ?? 0;
|
|
947
|
+
const ranges = mergeRanges([...(prev?.ranges ?? []), [start, start + response.bytes]]);
|
|
948
|
+
const total = response.total ?? prev?.total;
|
|
949
|
+
const union = ranges.reduce((sum, [a, b]) => sum + (b - a), 0);
|
|
950
|
+
media.set(response.url, {
|
|
951
|
+
url: response.url,
|
|
952
|
+
contentType: response.contentType ?? prev?.contentType,
|
|
953
|
+
bytes: total !== undefined ? Math.min(union, total) : union,
|
|
954
|
+
status: response.status,
|
|
955
|
+
ranges,
|
|
956
|
+
...(total !== undefined ? { total } : {}),
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
/**
|
|
960
|
+
* The most candidates per viewport that are re-read after a scroll (see readView): an opacity, clipped or zero-size copy.
|
|
961
|
+
* A candidate past the cap keeps its snapshot reason, and the page counts it (ViewportObservation.revealCapped).
|
|
962
|
+
*/
|
|
963
|
+
export const REVEAL_CAP = 60;
|
|
964
|
+
/** The wait between the snapshot and the second read of a hidden candidate (see readView): it must span a toggle period. */
|
|
965
|
+
export const REVEAL_STABLE_MS = 1000;
|
|
966
|
+
/** The wait after the scroll pass, before the last read: a fade the scroll started has finished by then. */
|
|
967
|
+
export const REVEAL_SETTLE_MS = 800;
|
|
968
|
+
/** The candidates of a page's viewports that were past REVEAL_CAP (not re-read after a scroll), summed. */
|
|
969
|
+
function revealCappedOf(observations) {
|
|
970
|
+
return observations.reduce((sum, obs) => sum + (obs.revealCapped ?? 0), 0);
|
|
971
|
+
}
|
|
972
|
+
/**
|
|
973
|
+
* One read of the page as it stands: its images and whether it is a Next.js app. `quick` takes no stability pass (see
|
|
974
|
+
* settledView): every hidden copy it finds is unverified.
|
|
975
|
+
*/
|
|
976
|
+
async function readView(page, light = false, withVideos = false, quick = false) {
|
|
977
|
+
const seen = await page.evaluate(async ({ depth, maxOwners, maxBackgrounds, maxValue, light, withVideos, revealCap, revealWait, revealSettle, reveal }) => {
|
|
812
978
|
const num = (value) => {
|
|
813
979
|
const n = parseFloat(value);
|
|
814
980
|
return Number.isFinite(n) ? n : 0;
|
|
@@ -953,8 +1119,86 @@ async function readView(page, light = false) {
|
|
|
953
1119
|
return { nodes, skipped, truncated: el !== null };
|
|
954
1120
|
};
|
|
955
1121
|
const nextjs = document.querySelector('img[data-nimg], script[src*="/_next/"], link[href*="/_next/"]') !== null;
|
|
1122
|
+
// Why an image is not shown, if it is not (see HiddenReason). Read in full reads only: a light read has no use for it.
|
|
1123
|
+
// The selectors of the :hover rules that swap what is shown (a rule setting display, visibility or opacity), with the
|
|
1124
|
+
// :hover taken out: an element they match is shown on hover, not on load.
|
|
1125
|
+
const hoverSelectors = [];
|
|
1126
|
+
const collectHover = (rules) => {
|
|
1127
|
+
for (const rule of Array.from(rules)) {
|
|
1128
|
+
if (hoverSelectors.length >= 2000)
|
|
1129
|
+
return;
|
|
1130
|
+
if (rule instanceof CSSStyleRule) {
|
|
1131
|
+
if (rule.selectorText.includes(":hover") && (rule.style.display || rule.style.visibility || rule.style.opacity)) {
|
|
1132
|
+
hoverSelectors.push(rule.selectorText.replace(/:hover/g, ""));
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
else if ("cssRules" in rule) {
|
|
1136
|
+
collectHover(rule.cssRules);
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
};
|
|
1140
|
+
for (const sheet of Array.from(document.styleSheets)) {
|
|
1141
|
+
try {
|
|
1142
|
+
collectHover(sheet.cssRules);
|
|
1143
|
+
}
|
|
1144
|
+
catch {
|
|
1145
|
+
// A cross-origin sheet has no readable rules.
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
const hoverSwap = (el) => hoverSelectors.some((selector) => {
|
|
1149
|
+
try {
|
|
1150
|
+
return el.matches(selector);
|
|
1151
|
+
}
|
|
1152
|
+
catch {
|
|
1153
|
+
return false;
|
|
1154
|
+
}
|
|
1155
|
+
});
|
|
1156
|
+
const hiddenReasonOf = (img, rect) => {
|
|
1157
|
+
let displayNone = false;
|
|
1158
|
+
let opacityZero = false;
|
|
1159
|
+
let inPanel = false;
|
|
1160
|
+
let clipped = false;
|
|
1161
|
+
for (let el = img; el !== null; el = el.parentElement) {
|
|
1162
|
+
const s = getComputedStyle(el);
|
|
1163
|
+
// content-visibility: hidden keeps the box (a closed <details> does) but skips the rendering: not shown either.
|
|
1164
|
+
if (s.display === "none" || s.getPropertyValue("content-visibility") === "hidden")
|
|
1165
|
+
displayNone = true;
|
|
1166
|
+
if (s.opacity === "0")
|
|
1167
|
+
opacityZero = true;
|
|
1168
|
+
// A tab panel, a hidden subtree, an aria-hidden one, or the content of a closed <details> (rendered when it opens).
|
|
1169
|
+
if (el.getAttribute("role") === "tabpanel" || el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true")
|
|
1170
|
+
inPanel = true;
|
|
1171
|
+
// The content of a closed <details> is not rendered (Chrome skips it with a pseudo-element, so the box is laid out
|
|
1172
|
+
// and only the content is not shown); the summary stays visible.
|
|
1173
|
+
if (el.tagName === "DETAILS" && !el.hasAttribute("open") && img.closest("summary") === null) {
|
|
1174
|
+
inPanel = true;
|
|
1175
|
+
displayNone = true;
|
|
1176
|
+
}
|
|
1177
|
+
// The root's overflow is the viewport's (it propagates from html or body), so it clips nothing: only the ancestors between.
|
|
1178
|
+
const root = el === document.documentElement || el === document.body;
|
|
1179
|
+
if (!root && el !== img && (s.overflowX === "hidden" || s.overflowY === "hidden" || s.overflowX === "clip" || s.overflowY === "clip")) {
|
|
1180
|
+
const box = el.getBoundingClientRect();
|
|
1181
|
+
if (rect.width > 0 && rect.height > 0 && (rect.right <= box.left || rect.left >= box.right || rect.bottom <= box.top || rect.top >= box.bottom))
|
|
1182
|
+
clipped = true;
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
const own = getComputedStyle(img);
|
|
1186
|
+
const reason = displayNone
|
|
1187
|
+
? "display"
|
|
1188
|
+
: own.visibility === "hidden" || own.visibility === "collapse"
|
|
1189
|
+
? "visibility"
|
|
1190
|
+
: opacityZero
|
|
1191
|
+
? "opacity"
|
|
1192
|
+
: rect.width === 0 || rect.height === 0
|
|
1193
|
+
? "zero-size"
|
|
1194
|
+
: clipped
|
|
1195
|
+
? "clipped"
|
|
1196
|
+
: undefined;
|
|
1197
|
+
return reason === undefined ? undefined : { reason, inPanel, hover: hoverSwap(img) };
|
|
1198
|
+
};
|
|
956
1199
|
const images = Array.from(document.querySelectorAll("img"), (img, index) => {
|
|
957
1200
|
const rect = img.getBoundingClientRect();
|
|
1201
|
+
const hidden = light ? undefined : hiddenReasonOf(img, rect);
|
|
958
1202
|
return {
|
|
959
1203
|
index,
|
|
960
1204
|
currentSrc: img.currentSrc,
|
|
@@ -966,6 +1210,9 @@ async function readView(page, light = false) {
|
|
|
966
1210
|
nextImage: img.hasAttribute("data-nimg"),
|
|
967
1211
|
inFirstViewport: rect.bottom > 0 && rect.top < window.innerHeight,
|
|
968
1212
|
lazy: img.loading === "lazy",
|
|
1213
|
+
...(hidden !== undefined
|
|
1214
|
+
? { hidden: hidden.reason, ...(hidden.inPanel ? { inPanel: true } : {}), ...(hidden.hover ? { hover: true } : {}) }
|
|
1215
|
+
: {}),
|
|
969
1216
|
layout: light ? undefined : layoutOf(img),
|
|
970
1217
|
locator: {
|
|
971
1218
|
testid: img.closest("[data-testid]")?.getAttribute("data-testid") ?? undefined,
|
|
@@ -990,6 +1237,7 @@ async function readView(page, light = false) {
|
|
|
990
1237
|
if (!value || value === "none" || !/url\(|image-set\(/.test(value))
|
|
991
1238
|
continue;
|
|
992
1239
|
const rect = el.getBoundingClientRect();
|
|
1240
|
+
const hiddenInfo = hiddenReasonOf(el, rect);
|
|
993
1241
|
// The element's own fiber only: an ancestor's owner would name a component that does not draw this background.
|
|
994
1242
|
const fiberKey = Object.keys(el).find((k) => k.startsWith("__reactFiber$"));
|
|
995
1243
|
backgrounds.push({
|
|
@@ -1000,15 +1248,218 @@ async function readView(page, light = false) {
|
|
|
1000
1248
|
owners: ownersOf(fiberKey ? el[fiberKey] : undefined),
|
|
1001
1249
|
width: rect.width,
|
|
1002
1250
|
height: rect.height,
|
|
1003
|
-
|
|
1251
|
+
// Hidden: the reason (see hiddenReasonOf). A hidden layer is measured only when its file was downloaded.
|
|
1252
|
+
hidden: hiddenInfo !== undefined,
|
|
1253
|
+
...(hiddenInfo !== undefined
|
|
1254
|
+
? { hiddenReason: hiddenInfo.reason, ...(hiddenInfo.inPanel ? { inPanel: true } : {}), ...(hiddenInfo.hover ? { hover: true } : {}) }
|
|
1255
|
+
: {}),
|
|
1004
1256
|
// Capped here as well: one character over the cap is enough for Node to skip the element.
|
|
1005
1257
|
backgroundImage: (value.length > maxValue ? value.slice(0, maxValue + 1) : value).replace(/url\((["']?)data:[^)]*\)/g, 'url("data:")'),
|
|
1006
1258
|
backgroundSize: cs.backgroundSize.slice(0, maxValue + 1),
|
|
1007
1259
|
});
|
|
1008
1260
|
}
|
|
1009
1261
|
}
|
|
1010
|
-
|
|
1011
|
-
|
|
1262
|
+
// <video> elements with a file source. Blob, media-stream and adaptive sources, and an empty src, are dropped here
|
|
1263
|
+
// (the rule of isOutOfScopeVideo, written out because this function cannot import it). A data: URI is replaced
|
|
1264
|
+
// before it leaves the browser. A light read takes none unless it asks (a locate joins the dev page's videos: the DOM
|
|
1265
|
+
// read is cheap, and no media is read). Only the top document is read: a video in an iframe is not.
|
|
1266
|
+
const videos = [];
|
|
1267
|
+
if (!light || withVideos) {
|
|
1268
|
+
const OUT_OF_SCOPE = (src) => src.trim() === "" || /^(blob|mediastream):/i.test(src) || /\.(m3u8|mpd)(?:[?#]|$)/i.test(src);
|
|
1269
|
+
const shortUrl = (url) => (url.startsWith("data:") ? "data:" : url);
|
|
1270
|
+
document.querySelectorAll("video").forEach((video, index) => {
|
|
1271
|
+
// A src attribute wins: the browser ignores the <source> children then, so they are not read at all.
|
|
1272
|
+
const hasSrcAttribute = (video.getAttribute("src") ?? "").trim() !== "";
|
|
1273
|
+
const sourceEls = hasSrcAttribute ? [] : Array.from(video.querySelectorAll("source"));
|
|
1274
|
+
// The source the browser plays. When it chose none (every source refused, so currentSrc is ""), the src attribute,
|
|
1275
|
+
// else the first in-scope <source>. Dropped only when none is in scope.
|
|
1276
|
+
const inScope = sourceEls.map((source) => source.src).filter((src) => !OUT_OF_SCOPE(src));
|
|
1277
|
+
const playing = video.currentSrc !== "" ? video.currentSrc : hasSrcAttribute ? video.src : (inScope[0] ?? video.src);
|
|
1278
|
+
if (OUT_OF_SCOPE(playing))
|
|
1279
|
+
return;
|
|
1280
|
+
const cs = getComputedStyle(video);
|
|
1281
|
+
const rect = video.getBoundingClientRect();
|
|
1282
|
+
const hiddenInfo = light ? undefined : hiddenReasonOf(video, rect);
|
|
1283
|
+
videos.push({
|
|
1284
|
+
index,
|
|
1285
|
+
currentSrc: shortUrl(playing),
|
|
1286
|
+
// The out-of-scope <source> children are left out, as the video itself is when its source is out of scope.
|
|
1287
|
+
sources: sourceEls
|
|
1288
|
+
.filter((source) => !OUT_OF_SCOPE(source.src))
|
|
1289
|
+
.map((source) => ({
|
|
1290
|
+
src: shortUrl(source.src),
|
|
1291
|
+
type: source.getAttribute("type") ?? undefined,
|
|
1292
|
+
media: source.getAttribute("media") ?? undefined,
|
|
1293
|
+
})),
|
|
1294
|
+
autoplay: video.autoplay,
|
|
1295
|
+
muted: video.muted,
|
|
1296
|
+
loop: video.loop,
|
|
1297
|
+
playsinline: video.hasAttribute("playsinline"),
|
|
1298
|
+
controls: video.controls,
|
|
1299
|
+
preload: video.getAttribute("preload") ?? "",
|
|
1300
|
+
poster: video.poster ? shortUrl(video.poster) : undefined,
|
|
1301
|
+
paused: video.paused,
|
|
1302
|
+
...(video.error ? { errorCode: video.error.code } : {}),
|
|
1303
|
+
renderedWidth: rect.width,
|
|
1304
|
+
renderedHeight: rect.height,
|
|
1305
|
+
...(video.videoWidth > 0 ? { videoWidth: video.videoWidth } : {}),
|
|
1306
|
+
inFirstViewport: rect.bottom > 0 && rect.top < window.innerHeight,
|
|
1307
|
+
// A full read uses the same check as an image (hiddenReasonOf); a light read keeps the display and box check only.
|
|
1308
|
+
hidden: hiddenInfo !== undefined || (light && (cs.display === "none" || cs.visibility === "hidden" || rect.width === 0 || rect.height === 0)),
|
|
1309
|
+
...(hiddenInfo !== undefined
|
|
1310
|
+
? { hiddenReason: hiddenInfo.reason, ...(hiddenInfo.inPanel ? { inPanel: true } : {}), ...(hiddenInfo.hover ? { hover: true } : {}) }
|
|
1311
|
+
: {}),
|
|
1312
|
+
locator: {
|
|
1313
|
+
testid: video.closest("[data-testid]")?.getAttribute("data-testid") ?? undefined,
|
|
1314
|
+
selector: cssPath(video),
|
|
1315
|
+
owners: ownersOf(fiberOf(video)),
|
|
1316
|
+
},
|
|
1317
|
+
});
|
|
1318
|
+
});
|
|
1319
|
+
}
|
|
1320
|
+
// Stability: a copy is hidden only when it reads hidden on every read: the snapshot, a read REVEAL_STABLE_MS later with no
|
|
1321
|
+
// scroll, and a read after a scroll to it (the same reason each time). A copy that reads shown at any read is shown: a
|
|
1322
|
+
// carousel slide that auto-advances, a crossfade, or a fade-in a scroll brings in. The candidates are every hidden copy
|
|
1323
|
+
// (images, then videos, in document order); past revealCap per viewport a candidate keeps its snapshot reason and is
|
|
1324
|
+
// flagged unstable: "unknown". The scroll pass is one pass over the candidates: each is brought to the centre of the
|
|
1325
|
+
// window (the window only, so an overflow box such as a carousel track is not scrolled), read after two frames, and
|
|
1326
|
+
// the window is put back at the end. A light read does none of this.
|
|
1327
|
+
let revealCapped = 0;
|
|
1328
|
+
if (!light) {
|
|
1329
|
+
const candidates = [];
|
|
1330
|
+
// A copy the scan's scroll pass saw visible at some step is shown (see MARK_VISIBLE_SCRIPT in measure.ts).
|
|
1331
|
+
const seenVisible = window.__imgtuneSeen;
|
|
1332
|
+
document.querySelectorAll("img").forEach((img, index) => {
|
|
1333
|
+
const obj = images[index];
|
|
1334
|
+
if (obj === undefined || obj.hidden === undefined)
|
|
1335
|
+
return;
|
|
1336
|
+
if (seenVisible?.has(img) === true) {
|
|
1337
|
+
delete obj.hidden;
|
|
1338
|
+
delete obj.inPanel;
|
|
1339
|
+
delete obj.hover;
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
candidates.push({
|
|
1343
|
+
el: img,
|
|
1344
|
+
isVideo: false,
|
|
1345
|
+
snap: { reason: obj.hidden, inPanel: obj.inPanel === true, hover: obj.hover === true },
|
|
1346
|
+
apply: (info, unstable) => {
|
|
1347
|
+
delete obj.hidden;
|
|
1348
|
+
delete obj.inPanel;
|
|
1349
|
+
delete obj.hover;
|
|
1350
|
+
if (unstable)
|
|
1351
|
+
obj.unstable = "unknown";
|
|
1352
|
+
if (info !== undefined) {
|
|
1353
|
+
obj.hidden = info.reason;
|
|
1354
|
+
if (info.inPanel)
|
|
1355
|
+
obj.inPanel = true;
|
|
1356
|
+
if (info.hover)
|
|
1357
|
+
obj.hover = true;
|
|
1358
|
+
}
|
|
1359
|
+
},
|
|
1360
|
+
});
|
|
1361
|
+
});
|
|
1362
|
+
document.querySelectorAll("video").forEach((video, index) => {
|
|
1363
|
+
const obj = videos.find((v) => v.index === index);
|
|
1364
|
+
if (obj === undefined || !obj.hidden || obj.hiddenReason === undefined)
|
|
1365
|
+
return;
|
|
1366
|
+
if (seenVisible?.has(video) === true) {
|
|
1367
|
+
delete obj.hiddenReason;
|
|
1368
|
+
delete obj.inPanel;
|
|
1369
|
+
delete obj.hover;
|
|
1370
|
+
obj.hidden = false;
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
candidates.push({
|
|
1374
|
+
el: video,
|
|
1375
|
+
isVideo: true,
|
|
1376
|
+
snap: { reason: obj.hiddenReason, inPanel: obj.inPanel === true, hover: obj.hover === true },
|
|
1377
|
+
apply: (info, unstable) => {
|
|
1378
|
+
delete obj.hiddenReason;
|
|
1379
|
+
delete obj.inPanel;
|
|
1380
|
+
delete obj.hover;
|
|
1381
|
+
obj.hidden = info !== undefined;
|
|
1382
|
+
if (unstable)
|
|
1383
|
+
obj.unstable = "unknown";
|
|
1384
|
+
if (info !== undefined) {
|
|
1385
|
+
obj.hiddenReason = info.reason;
|
|
1386
|
+
if (info.inPanel)
|
|
1387
|
+
obj.inPanel = true;
|
|
1388
|
+
if (info.hover)
|
|
1389
|
+
obj.hover = true;
|
|
1390
|
+
}
|
|
1391
|
+
},
|
|
1392
|
+
});
|
|
1393
|
+
});
|
|
1394
|
+
// Videos first: they are the heavy copies, so the cap never leaves a video unverified while images are checked.
|
|
1395
|
+
candidates.sort((a, b) => Number(b.isVideo) - Number(a.isVideo));
|
|
1396
|
+
// No candidate, or a quick read (a read taken after the page navigated during a full read): no stability pass, so no
|
|
1397
|
+
// sleeps. A hidden copy the pass would have verified is unverified here (unstable: "unknown"), as past revealCap.
|
|
1398
|
+
if (candidates.length === 0 || !reveal) {
|
|
1399
|
+
for (const c of candidates)
|
|
1400
|
+
c.apply(c.snap, true);
|
|
1401
|
+
return { nextjs, images, backgrounds, videos, revealCapped: 0 };
|
|
1402
|
+
}
|
|
1403
|
+
const checked = candidates.slice(0, revealCap);
|
|
1404
|
+
revealCapped = candidates.length - checked.length;
|
|
1405
|
+
const x0 = window.scrollX;
|
|
1406
|
+
const y0 = window.scrollY;
|
|
1407
|
+
const frames = () => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(() => resolve())));
|
|
1408
|
+
// Read 2: a second read, revealWait ms after the snapshot, with no scroll. Attribute changes in that window are recorded
|
|
1409
|
+
// too: a toggle whose two reads happen to agree (a period that fits twice in the window) still shows as a change.
|
|
1410
|
+
const touched = [];
|
|
1411
|
+
const watch = new MutationObserver((records) => {
|
|
1412
|
+
for (const record of records)
|
|
1413
|
+
touched.push(record.target);
|
|
1414
|
+
});
|
|
1415
|
+
watch.observe(document.documentElement, { subtree: true, attributes: true, attributeFilter: ["style", "class", "hidden", "open", "aria-hidden"] });
|
|
1416
|
+
await new Promise((resolve) => setTimeout(resolve, revealWait));
|
|
1417
|
+
watch.disconnect();
|
|
1418
|
+
// A change on the copy, its descendants, or one of its three nearest ancestors (a carousel's track or slide wrapper).
|
|
1419
|
+
const changedNear = (el) => touched.some((node) => {
|
|
1420
|
+
if (node === el || el.contains(node))
|
|
1421
|
+
return true;
|
|
1422
|
+
let up = el;
|
|
1423
|
+
for (let level = 0; level < 3 && up !== null; level++, up = up.parentElement)
|
|
1424
|
+
if (up === node)
|
|
1425
|
+
return true;
|
|
1426
|
+
return false;
|
|
1427
|
+
});
|
|
1428
|
+
const second = checked.map((c) => hiddenReasonOf(c.el, c.el.getBoundingClientRect()));
|
|
1429
|
+
// Read 3: one pass, each candidate read after a scroll to it.
|
|
1430
|
+
// Read 3a: each candidate right after its own scroll (a display that a scroll shows is read at its own position).
|
|
1431
|
+
const third = [];
|
|
1432
|
+
for (const c of checked) {
|
|
1433
|
+
const box = c.el.getBoundingClientRect();
|
|
1434
|
+
window.scrollBy(box.left + box.width / 2 - window.innerWidth / 2, box.top + box.height / 2 - window.innerHeight / 2);
|
|
1435
|
+
await frames();
|
|
1436
|
+
third.push(hiddenReasonOf(c.el, c.el.getBoundingClientRect()));
|
|
1437
|
+
}
|
|
1438
|
+
// Read 3b: once, after the pass, so a CSS fade started by the scroll has finished (one wait for the whole pass).
|
|
1439
|
+
await new Promise((resolve) => setTimeout(resolve, revealSettle));
|
|
1440
|
+
const fourth = checked.map((c) => hiddenReasonOf(c.el, c.el.getBoundingClientRect()));
|
|
1441
|
+
window.scrollTo(x0, y0);
|
|
1442
|
+
checked.forEach((c, i) => {
|
|
1443
|
+
const a = c.snap;
|
|
1444
|
+
const b = second[i];
|
|
1445
|
+
const t = third[i];
|
|
1446
|
+
const f = fourth[i];
|
|
1447
|
+
// Hidden only when every read is hidden with the same reason; any read that shows it, or a change of reason, is
|
|
1448
|
+
// shown (a toggle, a fade, or a box a scroll brings in).
|
|
1449
|
+
const stable = b !== undefined &&
|
|
1450
|
+
t !== undefined &&
|
|
1451
|
+
f !== undefined &&
|
|
1452
|
+
b.reason === a.reason &&
|
|
1453
|
+
t.reason === a.reason &&
|
|
1454
|
+
f.reason === a.reason &&
|
|
1455
|
+
!changedNear(c.el);
|
|
1456
|
+
c.apply(stable ? f : undefined, false);
|
|
1457
|
+
});
|
|
1458
|
+
for (const c of candidates.slice(revealCap))
|
|
1459
|
+
c.apply(c.snap, true);
|
|
1460
|
+
}
|
|
1461
|
+
return { nextjs, images, backgrounds, videos, revealCapped };
|
|
1462
|
+
}, { depth: LAYOUT_DEPTH, maxOwners: MAX_OWNERS, maxBackgrounds: MAX_BACKGROUND_ELEMENTS, maxValue: MAX_BACKGROUND_VALUE, light, withVideos, revealCap: REVEAL_CAP, revealWait: REVEAL_STABLE_MS, revealSettle: REVEAL_SETTLE_MS, reveal: !quick });
|
|
1012
1463
|
const html = seen.images.length === 0 ? await page.content() : "";
|
|
1013
1464
|
return { seen, html };
|
|
1014
1465
|
}
|
|
@@ -1093,8 +1544,12 @@ export async function readLcp(page) {
|
|
|
1093
1544
|
* location change after load). At most MAX_POST_LOAD_NAVIGATIONS repeats; past that, the last complete read is used.
|
|
1094
1545
|
* `readLcpFirst`: the LCP is read before the first scroll, and kept with the read it belongs to (fresh loads only).
|
|
1095
1546
|
*/
|
|
1096
|
-
async function settledView(page, navigations, loadCapMs, light = false, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, afterSettle, readLcpFirst = false) {
|
|
1547
|
+
async function settledView(page, navigations, loadCapMs, light = false, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, afterSettle, readLcpFirst = false, withVideos = false) {
|
|
1097
1548
|
let last;
|
|
1549
|
+
// Once a read was cut off by a navigation, the later reads are quick ones. A full read holds the page for its stability
|
|
1550
|
+
// pass (about 1.8 s of waits and a scroll), and a page that reloads itself on a timer would cut every full read. The
|
|
1551
|
+
// quick read's hidden copies are unverified, not claimed.
|
|
1552
|
+
let quick = false;
|
|
1098
1553
|
for (let repeat = 0;; repeat++) {
|
|
1099
1554
|
const start = navigations();
|
|
1100
1555
|
let view;
|
|
@@ -1112,7 +1567,7 @@ async function settledView(page, navigations, loadCapMs, light = false, idleTime
|
|
|
1112
1567
|
});
|
|
1113
1568
|
if (afterSettle !== undefined)
|
|
1114
1569
|
await afterSettle();
|
|
1115
|
-
view = await measure("read view", () => readView(page, light));
|
|
1570
|
+
view = await measure("read view", () => readView(page, light, withVideos, quick));
|
|
1116
1571
|
view.lcp = lcp;
|
|
1117
1572
|
}
|
|
1118
1573
|
catch (err) {
|
|
@@ -1128,6 +1583,7 @@ async function settledView(page, navigations, loadCapMs, light = false, idleTime
|
|
|
1128
1583
|
return last;
|
|
1129
1584
|
throw new Error(`the page kept navigating after load (${MAX_POST_LOAD_NAVIGATIONS + 1} attempts)`);
|
|
1130
1585
|
}
|
|
1586
|
+
quick = true;
|
|
1131
1587
|
// The new document settles first: its load event, or the cap if it never comes.
|
|
1132
1588
|
await waitForLoadWithin(page, loadCapMs).catch(() => false);
|
|
1133
1589
|
}
|
|
@@ -1274,8 +1730,6 @@ function responsesOfStep(fetched, answered, seen, dpr) {
|
|
|
1274
1730
|
if (img.currentSrc)
|
|
1275
1731
|
keep(img.currentSrc);
|
|
1276
1732
|
for (const bg of seen.backgrounds) {
|
|
1277
|
-
if (bg.hidden)
|
|
1278
|
-
continue;
|
|
1279
1733
|
for (const layer of parseBackgroundImage(bg.backgroundImage)) {
|
|
1280
1734
|
if (layer.dataUri)
|
|
1281
1735
|
continue;
|
|
@@ -1295,17 +1749,45 @@ function responsesOfStep(fetched, answered, seen, dpr) {
|
|
|
1295
1749
|
*
|
|
1296
1750
|
* `light` (the locate and sizes-only paths) reads no image bodies and no layout chain, and is always a single fresh load.
|
|
1297
1751
|
*/
|
|
1298
|
-
async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs) {
|
|
1752
|
+
async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos = false, captureResources = false) {
|
|
1299
1753
|
const first = viewports[0];
|
|
1300
1754
|
const outcomes = [];
|
|
1301
1755
|
let fresh = [];
|
|
1302
1756
|
// Every image response of the document, by URL (the latest one for a URL fetched twice).
|
|
1303
1757
|
const fetched = new Map();
|
|
1758
|
+
// Every other response of the page load (not media, not in a light read), by URL: the resource analyzers' records. The
|
|
1759
|
+
// records are kept across the viewport steps, since a resized viewport is the same page and its resources stay loaded.
|
|
1760
|
+
const others = new Map();
|
|
1761
|
+
// Every media (video) response of the page load, by URL: its bytes from the headers, never from the body.
|
|
1762
|
+
const media = new Map();
|
|
1304
1763
|
const bodyReads = [];
|
|
1764
|
+
// The resource bodies (see resources/): settled on their own short wait, so one stream that never ends costs seconds, not the
|
|
1765
|
+
// image bodies' wait. A body not read in time is fetched again by the resource phase, when an analyzer wants it.
|
|
1766
|
+
const resourceReads = [];
|
|
1767
|
+
// The media bodies, awaited on their own: they share one deadline for the whole page load (see mediaDeadline below).
|
|
1768
|
+
const mediaReads = [];
|
|
1769
|
+
// Per media response: counts the bytes that arrived, for a response still streaming when the media reads settle.
|
|
1770
|
+
const mediaFallbacks = [];
|
|
1771
|
+
// The media requests in flight, by request id: the bytes received so far (CDP dataReceived), for the fallback above.
|
|
1772
|
+
const inFlightMedia = new Map();
|
|
1773
|
+
const streamedBytesOf = (url) => {
|
|
1774
|
+
let bytes = 0;
|
|
1775
|
+
for (const entry of inFlightMedia.values())
|
|
1776
|
+
if (entry.url === url)
|
|
1777
|
+
bytes += entry.bytes;
|
|
1778
|
+
return bytes;
|
|
1779
|
+
};
|
|
1780
|
+
// One deadline for the media reads of the whole page load: the viewport steps share it, so a stream that never ends
|
|
1781
|
+
// costs BODY_READ_TIMEOUT_MS once per page, not once per viewport.
|
|
1782
|
+
let mediaDeadline;
|
|
1305
1783
|
let blockedRequests = 0;
|
|
1784
|
+
// The names the egress proxy could not resolve during this page load (see startEgressProxy).
|
|
1785
|
+
const unresolvedHosts = new Set();
|
|
1306
1786
|
let thirdPartyBlocked = 0;
|
|
1307
1787
|
// Stylesheet URLs the browser could not apply (refused, failed or blocked): one entry per URL.
|
|
1308
1788
|
const failedSheets = new Set();
|
|
1789
|
+
// Stylesheet URLs the browser applied (answered below 400): with failedSheets, it tells a partly styled page from an unstyled one.
|
|
1790
|
+
const loadedSheets = new Set();
|
|
1309
1791
|
// Requests the browser has not finished, by request id (a redirect keeps its id).
|
|
1310
1792
|
const busy = new Set();
|
|
1311
1793
|
// The image URLs answered during the current step (see responsesOfStep).
|
|
@@ -1313,8 +1795,11 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1313
1795
|
const decoded = new WeakSet();
|
|
1314
1796
|
// Every request the browser makes goes through this proxy, so each redirect hop and each tunnel is checked at connect
|
|
1315
1797
|
// time. Chromium gives the route handler no route for a redirect hop, so the route guard alone would miss them.
|
|
1316
|
-
const egress = await measure("egress proxy start", () => startEgressProxy(guard.policy, () => {
|
|
1798
|
+
const egress = await measure("egress proxy start", () => startEgressProxy(guard.policy, (host) => {
|
|
1317
1799
|
blockedRequests++;
|
|
1800
|
+
// A name the proxy could not resolve (not a refused address): the page's own scripts for it may not run here.
|
|
1801
|
+
if (host !== undefined)
|
|
1802
|
+
unresolvedHosts.add(host);
|
|
1318
1803
|
}));
|
|
1319
1804
|
try {
|
|
1320
1805
|
const context = await measure("context start", () => browser.newContext({
|
|
@@ -1351,6 +1836,14 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1351
1836
|
}
|
|
1352
1837
|
catch {
|
|
1353
1838
|
blocked = true;
|
|
1839
|
+
// The guard could not resolve the name (it throws on a failed lookup, not on a refused address): a name a browser
|
|
1840
|
+
// may still load (see unresolvedHosts).
|
|
1841
|
+
try {
|
|
1842
|
+
unresolvedHosts.add(new URL(url).hostname);
|
|
1843
|
+
}
|
|
1844
|
+
catch {
|
|
1845
|
+
// Not a URL the guard looked at.
|
|
1846
|
+
}
|
|
1354
1847
|
}
|
|
1355
1848
|
if (blocked) {
|
|
1356
1849
|
blockedRequests++;
|
|
@@ -1380,15 +1873,70 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1380
1873
|
socket.connectToServer();
|
|
1381
1874
|
}
|
|
1382
1875
|
});
|
|
1876
|
+
// Replay for development (IMGTUNE_REPLAY_HAR, see replay.ts): registered last, so the recorded page answers first.
|
|
1877
|
+
const replayHar = replayHarPath(pageUrl, first.name);
|
|
1878
|
+
if (replayHar !== undefined)
|
|
1879
|
+
await context.routeFromHAR(replayHar, { notFound: "abort" });
|
|
1383
1880
|
context.on("requestfailed", (request) => {
|
|
1384
1881
|
if (request.resourceType() === "stylesheet")
|
|
1385
1882
|
failedSheets.add(request.url());
|
|
1386
1883
|
});
|
|
1387
1884
|
context.on("response", (response) => {
|
|
1388
1885
|
// A stylesheet the server refused is not applied (Chromium still makes an empty sheet for it): counted for the scan's warning.
|
|
1389
|
-
if (response.request().resourceType() === "stylesheet"
|
|
1390
|
-
|
|
1886
|
+
if (response.request().resourceType() === "stylesheet") {
|
|
1887
|
+
if (response.status() >= 400)
|
|
1888
|
+
failedSheets.add(response.url());
|
|
1889
|
+
else
|
|
1890
|
+
loadedSheets.add(response.url());
|
|
1891
|
+
}
|
|
1391
1892
|
const contentType = response.headers()["content-type"];
|
|
1893
|
+
// A media response is sized once it has finished, from the browser's count of its body (sizes): the body is never
|
|
1894
|
+
// read, since a video can be far too big to buffer. Chunked answers have no Content-Length, so the headers are the
|
|
1895
|
+
// fallback only: when sizes fails or reports 0, or when the response is still streaming at the settle (see below).
|
|
1896
|
+
// A light read reads no media at all: it waits on none of it.
|
|
1897
|
+
if (response.request().resourceType() === "media" || (contentType ?? "").toLowerCase().startsWith("video/")) {
|
|
1898
|
+
if (light)
|
|
1899
|
+
return;
|
|
1900
|
+
const url = response.url();
|
|
1901
|
+
const status = response.status();
|
|
1902
|
+
const headers = response.headers();
|
|
1903
|
+
const headerBytes = mediaBytesOf(headers) ?? 0;
|
|
1904
|
+
const { start, total } = contentRangeOf(headers);
|
|
1905
|
+
// A media file is a resource record too, without a body: the resource phase fetches it again when it is wanted
|
|
1906
|
+
// (Chromium gives no body for a range response, see resources/phase.ts).
|
|
1907
|
+
const resourceRecord = captureResources ? resourceRecordOf(response, pageUrl, "", contentType) : undefined;
|
|
1908
|
+
if (resourceRecord !== undefined)
|
|
1909
|
+
others.set(url, resourceRecord);
|
|
1910
|
+
let counted = false;
|
|
1911
|
+
const count = (bytes) => {
|
|
1912
|
+
if (counted)
|
|
1913
|
+
return;
|
|
1914
|
+
counted = true;
|
|
1915
|
+
addMediaResponse(media, { url, status, contentType, bytes, start, total });
|
|
1916
|
+
if (resourceRecord !== undefined)
|
|
1917
|
+
resourceRecord.bytes = bytes;
|
|
1918
|
+
};
|
|
1919
|
+
mediaReads.push((async () => {
|
|
1920
|
+
try {
|
|
1921
|
+
// An aborted range request also finishes (with an error): its sizes still count what was received.
|
|
1922
|
+
await response.finished();
|
|
1923
|
+
const { responseBodySize } = await response.request().sizes();
|
|
1924
|
+
count(responseBodySize > 0 ? responseBodySize : headerBytes);
|
|
1925
|
+
}
|
|
1926
|
+
catch {
|
|
1927
|
+
count(headerBytes);
|
|
1928
|
+
}
|
|
1929
|
+
})());
|
|
1930
|
+
// Still streaming at the settle: the bytes that have arrived so far (or the header bytes, when those are more).
|
|
1931
|
+
mediaFallbacks.push(() => count(Math.max(headerBytes, streamedBytesOf(url))));
|
|
1932
|
+
return;
|
|
1933
|
+
}
|
|
1934
|
+
// Every other response is a resource record (see resources/): its body is read only when an analyzer wants it.
|
|
1935
|
+
if (captureResources && !light) {
|
|
1936
|
+
const record = resourceRecordOf(response, pageUrl, "", contentType);
|
|
1937
|
+
others.set(response.url(), record);
|
|
1938
|
+
resourceReads.push(readResourceBody(record, response).catch(() => { }));
|
|
1939
|
+
}
|
|
1392
1940
|
const isImage = response.request().resourceType() === "image" || (contentType ?? "").toLowerCase().startsWith("image/");
|
|
1393
1941
|
if (!isImage)
|
|
1394
1942
|
return;
|
|
@@ -1427,13 +1975,28 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1427
1975
|
navigations = await countDocumentNavigations(context, page);
|
|
1428
1976
|
}
|
|
1429
1977
|
// The requests in flight, for the network-idle wait after a resize (the browser's own events).
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1978
|
+
// Always opened: the media bytes that arrive before a settle are read from its data events (see inFlightMedia).
|
|
1979
|
+
const cdp = await context.newCDPSession(page);
|
|
1980
|
+
// A media request is not counted as busy: a streaming autoplay video never finishes, and would hold every resize's
|
|
1981
|
+
// network-idle wait to its cap. Its bytes are still counted per response (see the media branch above).
|
|
1982
|
+
cdp.on("Network.requestWillBeSent", (event) => {
|
|
1983
|
+
if (event.type === "Media")
|
|
1984
|
+
inFlightMedia.set(event.requestId, { url: event.request?.url ?? "", bytes: 0 });
|
|
1985
|
+
else
|
|
1986
|
+
busy.add(event.requestId);
|
|
1987
|
+
});
|
|
1988
|
+
cdp.on("Network.dataReceived", (event) => {
|
|
1989
|
+
const entry = inFlightMedia.get(event.requestId);
|
|
1990
|
+
if (entry !== undefined)
|
|
1991
|
+
entry.bytes += event.dataLength ?? 0;
|
|
1992
|
+
});
|
|
1993
|
+
const done = (event) => {
|
|
1994
|
+
busy.delete(event.requestId);
|
|
1995
|
+
inFlightMedia.delete(event.requestId);
|
|
1996
|
+
};
|
|
1997
|
+
cdp.on("Network.loadingFinished", done);
|
|
1998
|
+
cdp.on("Network.loadingFailed", done);
|
|
1999
|
+
await cdp.send("Network.enable");
|
|
1437
2000
|
const resizeState = { userAgent: first.userAgent };
|
|
1438
2001
|
let loaded;
|
|
1439
2002
|
for (const [step, viewport] of viewports.entries()) {
|
|
@@ -1449,11 +2012,16 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1449
2012
|
await measure(`resize ${viewport.name}`, () => resizeTo(cdp, page, viewport, resizeState));
|
|
1450
2013
|
}
|
|
1451
2014
|
// The LCP belongs to the page's own load: a fresh load's viewport reads it (a resized one shares that load's LCP).
|
|
1452
|
-
const { seen, html, lcp } = await settledView(page, navigations, loadCapMs, light, idleTimeoutMs,
|
|
2015
|
+
const { seen, html, lcp } = await settledView(page, navigations, loadCapMs, light, idleTimeoutMs,
|
|
2016
|
+
// Every full read waits on the busy set, fresh load included: Playwright's networkidle counts the media requests
|
|
2017
|
+
// too, so a streaming video would hold a fresh load to the cap. A light fresh load waits for nothing, as before.
|
|
2018
|
+
step === 0 && light ? undefined : () => waitNetworkQuiet(() => busy.size, idleTimeoutMs), !light && step === 0, withVideos);
|
|
1453
2019
|
if (step > 0 && navigations() !== navigatedBefore) {
|
|
1454
2020
|
fresh = viewports.slice(step);
|
|
1455
2021
|
break;
|
|
1456
2022
|
}
|
|
2023
|
+
// What the page shows, once it has settled (the resource analyzers read it; a light read takes none).
|
|
2024
|
+
const pageContext = captureResources && !light ? await measure("page context", () => readPageContext(page)) : undefined;
|
|
1457
2025
|
const status = loaded?.response?.status();
|
|
1458
2026
|
if (debug !== undefined && functionsWanted !== undefined) {
|
|
1459
2027
|
for (const img of seen.images) {
|
|
@@ -1466,6 +2034,12 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1466
2034
|
await debug.session.detach().catch(() => undefined);
|
|
1467
2035
|
}
|
|
1468
2036
|
await measure("body reads", () => settleWithin(bodyReads, BODY_READ_TIMEOUT_MS));
|
|
2037
|
+
await measure("resource bodies", () => settleWithin(resourceReads, RESOURCE_BODY_SETTLE_MS));
|
|
2038
|
+
mediaDeadline ??= Date.now() + BODY_READ_TIMEOUT_MS;
|
|
2039
|
+
await measure("media reads", () => settleWithin(mediaReads, Math.max(0, mediaDeadline - Date.now())));
|
|
2040
|
+
// A media response still streaming at the settle counts the bytes that arrived; it is not waited for.
|
|
2041
|
+
for (const fallback of mediaFallbacks)
|
|
2042
|
+
fallback();
|
|
1469
2043
|
const responses = step === 0 ? new Map(fetched) : responsesOfStep(fetched, answered, seen, viewport.dpr);
|
|
1470
2044
|
count("image responses", responses.size);
|
|
1471
2045
|
for (const record of responses.values()) {
|
|
@@ -1474,24 +2048,43 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1474
2048
|
decoded.add(record);
|
|
1475
2049
|
record.pixels = await measure("decode: pixel size", () => pixelSize(record.buffer));
|
|
1476
2050
|
}
|
|
2051
|
+
// A page with a name the proxy could not resolve may not run its own scripts as a visitor's browser runs them: its
|
|
2052
|
+
// hidden copies are not verified (unstable: "unknown"), and the page says which names (unresolvedHosts).
|
|
2053
|
+
if (unresolvedHosts.size > 0) {
|
|
2054
|
+
for (const image of seen.images)
|
|
2055
|
+
if (image.hidden !== undefined)
|
|
2056
|
+
image.unstable = "unknown";
|
|
2057
|
+
for (const video of seen.videos)
|
|
2058
|
+
if (video.hidden && video.hiddenReason !== undefined)
|
|
2059
|
+
video.unstable = "unknown";
|
|
2060
|
+
}
|
|
1477
2061
|
outcomes.push({
|
|
1478
2062
|
ok: true,
|
|
1479
2063
|
value: {
|
|
1480
2064
|
viewport,
|
|
1481
2065
|
load: step === 0 ? "fresh" : "resized",
|
|
2066
|
+
...(unresolvedHosts.size > 0 ? { unresolvedHosts: [...unresolvedHosts].slice(0, 20) } : {}),
|
|
1482
2067
|
status,
|
|
1483
2068
|
finalUrl: page.url(),
|
|
1484
2069
|
...(step === 0 && loaded !== undefined ? { loadTimedOut: loaded.loadTimedOut } : {}),
|
|
1485
2070
|
...(lcp !== undefined ? { lcp } : {}),
|
|
1486
|
-
blocked: isChallenge(status, seen.images.length, html),
|
|
2071
|
+
blocked: isChallenge(status, seen.images.length, html, loaded?.response?.headers()),
|
|
1487
2072
|
blockedRequests: blockedRequests - blockedBefore,
|
|
1488
2073
|
thirdPartyBlocked: thirdPartyBlocked - thirdBefore,
|
|
1489
2074
|
nextjs: seen.nextjs,
|
|
1490
2075
|
images: seen.images,
|
|
2076
|
+
revealCapped: seen.revealCapped,
|
|
1491
2077
|
backgrounds: seen.backgrounds,
|
|
2078
|
+
videos: seen.videos,
|
|
2079
|
+
// A copy: the page load's records keep growing for the viewports after this one.
|
|
2080
|
+
media: new Map(media),
|
|
1492
2081
|
stylesheetsFailed: failedSheets.size,
|
|
1493
2082
|
stylesheetHosts: hostsOf(failedSheets),
|
|
2083
|
+
stylesheetsLoaded: loadedSheets.size,
|
|
1494
2084
|
responses,
|
|
2085
|
+
...(captureResources && !light
|
|
2086
|
+
? { resources: [...others.values()].map((record) => ({ ...record, viewport: viewport.name })), ...(pageContext !== undefined ? { pageContext } : {}) }
|
|
2087
|
+
: {}),
|
|
1495
2088
|
},
|
|
1496
2089
|
});
|
|
1497
2090
|
}
|
|
@@ -1516,17 +2109,17 @@ async function readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageS
|
|
|
1516
2109
|
* The observations of one load group (see readGroup), one outcome per viewport in the order given. A group of one
|
|
1517
2110
|
* viewport is a fresh load: the default. `--shared-loads` groups the viewports of a class into one load.
|
|
1518
2111
|
*/
|
|
1519
|
-
export async function observeLoads(browser, pageUrl, viewports, guard, loadCapMs = LOAD_EVENT_CAP_MS, storageState, light = false, functionsWanted, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS) {
|
|
1520
|
-
const { outcomes, fresh } = await readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs);
|
|
2112
|
+
export async function observeLoads(browser, pageUrl, viewports, guard, loadCapMs = LOAD_EVENT_CAP_MS, storageState, light = false, functionsWanted, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, withVideos = false, captureResources = false) {
|
|
2113
|
+
const { outcomes, fresh } = await readGroup(browser, pageUrl, viewports, guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos, captureResources);
|
|
1521
2114
|
for (const viewport of fresh) {
|
|
1522
|
-
const [outcome] = await observeLoads(browser, pageUrl, [viewport], guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs);
|
|
2115
|
+
const [outcome] = await observeLoads(browser, pageUrl, [viewport], guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos, captureResources);
|
|
1523
2116
|
outcomes.push(outcome);
|
|
1524
2117
|
}
|
|
1525
2118
|
return outcomes;
|
|
1526
2119
|
}
|
|
1527
2120
|
/** One viewport's observation by a fresh load (the locate and sizes-only paths). */
|
|
1528
|
-
async function observeViewport(browser, pageUrl, viewport, guard, loadCapMs = LOAD_EVENT_CAP_MS, storageState, light = false, functionsWanted, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS) {
|
|
1529
|
-
const [outcome] = await observeLoads(browser, pageUrl, [viewport], guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs);
|
|
2121
|
+
async function observeViewport(browser, pageUrl, viewport, guard, loadCapMs = LOAD_EVENT_CAP_MS, storageState, light = false, functionsWanted, idleTimeoutMs = NETWORK_IDLE_TIMEOUT_MS, withVideos = false) {
|
|
2122
|
+
const [outcome] = await observeLoads(browser, pageUrl, [viewport], guard, loadCapMs, storageState, light, functionsWanted, idleTimeoutMs, withVideos);
|
|
1530
2123
|
if (!outcome.ok)
|
|
1531
2124
|
throw outcome.error;
|
|
1532
2125
|
return outcome.value;
|
|
@@ -1612,22 +2205,43 @@ async function resolveSources(ctx, observations, wanted) {
|
|
|
1612
2205
|
for (const img of obs.images) {
|
|
1613
2206
|
if (wanted && !wanted(img))
|
|
1614
2207
|
continue;
|
|
1615
|
-
|
|
1616
|
-
const chain = owners.length > 0 ? await resolveOwnerChain(owners, (chunk) => loadSourceMap(ctx, chunk)) : [];
|
|
1617
|
-
const dev = chain[0] !== undefined ? { file: chain[0].file, line: chain[0].line } : undefined;
|
|
1618
|
-
// A production build has no owner source: the function locations of the owners stand in (prod-sourcemap).
|
|
1619
|
-
const prod = dev === undefined && img.locator?.functionLocations?.length
|
|
1620
|
-
? await resolveFunctionLocations(img.locator.functionLocations, (chunk) => loadSourceMap(ctx, chunk))
|
|
1621
|
-
: undefined;
|
|
1622
|
-
const found = dev ?? prod;
|
|
1623
|
-
const via = dev ? "react-dev" : prod ? "prod-sourcemap" : undefined;
|
|
1624
|
-
const component = componentNamesOf(owners);
|
|
1625
|
-
const check = found ? await checkSource(ctx, texts, found.file, found.line, component[0]) : undefined;
|
|
1626
|
-
resolved.set(img, { file: found?.file, line: found?.line, component, check, via, ownerChain: chain.length > 0 ? chain : undefined });
|
|
2208
|
+
resolved.set(img, await resolveLocator(ctx, texts, img.locator));
|
|
1627
2209
|
}
|
|
1628
2210
|
}
|
|
1629
2211
|
return resolved;
|
|
1630
2212
|
}
|
|
2213
|
+
/**
|
|
2214
|
+
* The owner source of each observed <video> (the same rule as an image's, see resolveLocator): its file:line on a dev build,
|
|
2215
|
+
* or the function location on a production build, and the check of that line. Used by the scan's video records and by a
|
|
2216
|
+
* locate, which joins the dev page's videos to the measured ones.
|
|
2217
|
+
*/
|
|
2218
|
+
async function resolveVideoSources(ctx, observations) {
|
|
2219
|
+
const resolved = new Map();
|
|
2220
|
+
const texts = new Map();
|
|
2221
|
+
for (const obs of observations) {
|
|
2222
|
+
for (const video of obs.videos ?? [])
|
|
2223
|
+
resolved.set(video, await resolveLocator(ctx, texts, video.locator));
|
|
2224
|
+
}
|
|
2225
|
+
return resolved;
|
|
2226
|
+
}
|
|
2227
|
+
/**
|
|
2228
|
+
* The source of one observed element from its locator: the React owner chain (dev builds, through the source map), else the
|
|
2229
|
+
* owner's function location (production builds), and the check of the file:line against the project file.
|
|
2230
|
+
*/
|
|
2231
|
+
async function resolveLocator(ctx, texts, locator) {
|
|
2232
|
+
const owners = locator?.owners ?? [];
|
|
2233
|
+
const chain = owners.length > 0 ? await resolveOwnerChain(owners, (chunk) => loadSourceMap(ctx, chunk)) : [];
|
|
2234
|
+
const dev = chain[0] !== undefined ? { file: chain[0].file, line: chain[0].line } : undefined;
|
|
2235
|
+
// A production build has no owner source: the function locations of the owners stand in (prod-sourcemap).
|
|
2236
|
+
const prod = dev === undefined && locator?.functionLocations?.length
|
|
2237
|
+
? await resolveFunctionLocations(locator.functionLocations, (chunk) => loadSourceMap(ctx, chunk))
|
|
2238
|
+
: undefined;
|
|
2239
|
+
const found = dev ?? prod;
|
|
2240
|
+
const via = dev ? "react-dev" : prod ? "prod-sourcemap" : undefined;
|
|
2241
|
+
const component = componentNamesOf(owners);
|
|
2242
|
+
const check = found ? await checkSource(ctx, texts, found.file, found.line, component[0]) : undefined;
|
|
2243
|
+
return { file: found?.file, line: found?.line, component, check, via, ownerChain: chain.length > 0 ? chain : undefined };
|
|
2244
|
+
}
|
|
1631
2245
|
const formatName = (format) => (format === "jpg" ? "jpeg" : format);
|
|
1632
2246
|
/**
|
|
1633
2247
|
* The owner of each CSS background element, by its index (the first viewport that has it). Owners give the component
|
|
@@ -1679,8 +2293,23 @@ async function measureBackground(ctx, obs, bg, layerIndex, layer, size) {
|
|
|
1679
2293
|
size,
|
|
1680
2294
|
};
|
|
1681
2295
|
const skip = (reason) => ({ ...instance, skipped: reason });
|
|
1682
|
-
if (bg.hidden)
|
|
1683
|
-
|
|
2296
|
+
if (bg.hidden) {
|
|
2297
|
+
// Not shown on load. Its file is measured only if the browser downloaded it (display:none usually does not download).
|
|
2298
|
+
const loaded = layer.dataUri ? undefined : pickLoaded(layer.candidates, (url) => obs.responses.get(url)?.buffer !== undefined, dpr);
|
|
2299
|
+
if (!loaded)
|
|
2300
|
+
return skip("hidden");
|
|
2301
|
+
const downloaded = obs.responses.get(loaded.url);
|
|
2302
|
+
return {
|
|
2303
|
+
...instance,
|
|
2304
|
+
url: loaded.url,
|
|
2305
|
+
bytes: downloaded.bytes,
|
|
2306
|
+
pixelWidth: downloaded.pixels?.width ?? 0,
|
|
2307
|
+
format: formatName(downloaded.pixels?.format),
|
|
2308
|
+
hidden: bg.hiddenReason,
|
|
2309
|
+
...(bg.inPanel === true ? { inPanel: true } : {}),
|
|
2310
|
+
...(bg.hover === true ? { hover: true } : {}),
|
|
2311
|
+
};
|
|
2312
|
+
}
|
|
1684
2313
|
if (bg.width < MIN_RENDERED_PX || bg.height < MIN_RENDERED_PX)
|
|
1685
2314
|
return skip("under 16 px");
|
|
1686
2315
|
if (layer.dataUri)
|
|
@@ -1708,7 +2337,15 @@ async function measureBackground(ctx, obs, bg, layerIndex, layer, size) {
|
|
|
1708
2337
|
needed,
|
|
1709
2338
|
undersized: served.width < UNDERSIZED_RATIO * needed,
|
|
1710
2339
|
});
|
|
1711
|
-
|
|
2340
|
+
const animated = animationOfBuffer(response.buffer);
|
|
2341
|
+
if (animated !== undefined)
|
|
2342
|
+
instance.animated = animated;
|
|
2343
|
+
const flatness = await stillFlatness(response, served, animated !== undefined);
|
|
2344
|
+
if (flatness !== undefined)
|
|
2345
|
+
instance.flatness = flatness;
|
|
2346
|
+
if (flatness?.flat === true)
|
|
2347
|
+
ctx.flatBuffers?.offer(chosen.url, response.buffer, response.bytes);
|
|
2348
|
+
if (ctx.estimates !== false && animated === undefined && served.width > PLAIN_RESIZE_RATIO * needed && served.format) {
|
|
1712
2349
|
const resized = await measure("background resize estimate", () => resizeEstimateOf(ctx, response.buffer, served.format, needed));
|
|
1713
2350
|
if (resized)
|
|
1714
2351
|
Object.assign(instance, approxEst(Math.max(0, response.bytes - resized.bytes), resized.approx));
|
|
@@ -1740,7 +2377,8 @@ async function baseline(ctx, obs, img) {
|
|
|
1740
2377
|
...loaderOf(img.currentSrc, ctx.pageUrl),
|
|
1741
2378
|
};
|
|
1742
2379
|
const skip = (reason) => ({ ...instance, skipped: reason });
|
|
1743
|
-
|
|
2380
|
+
// A hidden element is measured whatever its box: its file was downloaded, and that is what the page spends (see hidden).
|
|
2381
|
+
if (img.renderedWidth < MIN_RENDERED_PX && img.hidden === undefined)
|
|
1744
2382
|
return skip("under 16 px");
|
|
1745
2383
|
if (!img.currentSrc || img.currentSrc.startsWith("data:"))
|
|
1746
2384
|
return skip("data URI");
|
|
@@ -1748,11 +2386,28 @@ async function baseline(ctx, obs, img) {
|
|
|
1748
2386
|
if (!response?.buffer)
|
|
1749
2387
|
return skip("not loaded");
|
|
1750
2388
|
const served = response.pixels;
|
|
2389
|
+
if (img.hidden !== undefined) {
|
|
2390
|
+
// Loaded but not shown at this viewport: the bytes, and nothing to fix in the file itself (no sizes, no quality, no saving).
|
|
2391
|
+
instance.kind = "hidden";
|
|
2392
|
+
Object.assign(instance, { bytes: response.bytes, pixelWidth: served?.width ?? 0, format: served?.format ? formatName(served.format) : undefined });
|
|
2393
|
+
const hiddenAnimated = animationOfBuffer(response.buffer);
|
|
2394
|
+
if (hiddenAnimated !== undefined)
|
|
2395
|
+
instance.animated = hiddenAnimated;
|
|
2396
|
+
return instance;
|
|
2397
|
+
}
|
|
1751
2398
|
if (response.contentType?.toLowerCase().includes("svg") || served?.format === "svg")
|
|
1752
2399
|
return skip("vector");
|
|
1753
2400
|
if (!served)
|
|
1754
2401
|
return skip("not decodable");
|
|
1755
2402
|
Object.assign(instance, { bytes: response.bytes, pixelWidth: served.width, format: formatName(served.format) });
|
|
2403
|
+
const animated = animationOfBuffer(response.buffer);
|
|
2404
|
+
if (animated !== undefined)
|
|
2405
|
+
instance.animated = animated;
|
|
2406
|
+
const flatness = await stillFlatness(response, served, animated !== undefined);
|
|
2407
|
+
if (flatness !== undefined)
|
|
2408
|
+
instance.flatness = flatness;
|
|
2409
|
+
if (flatness?.flat === true)
|
|
2410
|
+
ctx.flatBuffers?.offer(img.currentSrc, response.buffer, response.bytes);
|
|
1756
2411
|
const candidates = parseSrcset(img.srcset, ctx.pageUrl);
|
|
1757
2412
|
if (candidates.length > 0) {
|
|
1758
2413
|
const largest = Math.max(...candidates.map((c) => c.width));
|
|
@@ -1790,9 +2445,10 @@ async function baseline(ctx, obs, img) {
|
|
|
1790
2445
|
}
|
|
1791
2446
|
return instance;
|
|
1792
2447
|
}
|
|
1793
|
-
// No w-descriptor srcset: a wider file than needed is estimated by a local resize at its own format (est.).
|
|
2448
|
+
// No w-descriptor srcset: a wider file than needed is estimated by a local resize at its own format (est.). An animated
|
|
2449
|
+
// file gets no resize estimate: its saving is an encode.
|
|
1794
2450
|
instance.kind = "plain";
|
|
1795
|
-
if (ctx.estimates !== false && served.width > PLAIN_RESIZE_RATIO * needed && served.format) {
|
|
2451
|
+
if (ctx.estimates !== false && instance.animated === undefined && served.width > PLAIN_RESIZE_RATIO * needed && served.format) {
|
|
1796
2452
|
const resized = await measure("plain resize estimate", () => resizeEstimateOf(ctx, response.buffer, served.format, needed));
|
|
1797
2453
|
if (resized)
|
|
1798
2454
|
Object.assign(instance, approxEst(Math.max(0, response.bytes - resized.bytes), resized.approx));
|
|
@@ -1839,14 +2495,15 @@ export const CDN_AUTO_QUALITY_SHARE = 0.1;
|
|
|
1839
2495
|
*/
|
|
1840
2496
|
export function applyCdnPolicies(instances, viewportCount) {
|
|
1841
2497
|
for (const inst of instances) {
|
|
1842
|
-
|
|
2498
|
+
const lossless = inst.format === "jpeg" || inst.format === "png" || (inst.format === "gif" && inst.animated === undefined);
|
|
2499
|
+
if (inst.formatAuto === true && inst.kind !== "skipped" && inst.kind !== "hidden" && lossless) {
|
|
1843
2500
|
inst.est = 0;
|
|
1844
2501
|
inst.estAggressive = 0;
|
|
1845
2502
|
}
|
|
1846
2503
|
}
|
|
1847
2504
|
const groups = new Map();
|
|
1848
2505
|
for (const inst of instances) {
|
|
1849
|
-
if (inst.qualityParam !== "auto" || inst.kind === "skipped")
|
|
2506
|
+
if (inst.qualityParam !== "auto" || inst.kind === "skipped" || inst.kind === "hidden")
|
|
1850
2507
|
continue;
|
|
1851
2508
|
const list = groups.get(inst.group);
|
|
1852
2509
|
if (list)
|
|
@@ -1881,7 +2538,8 @@ function median(values) {
|
|
|
1881
2538
|
async function estimateQualities(ctx, all, refName) {
|
|
1882
2539
|
const groups = new Map();
|
|
1883
2540
|
for (const inst of all) {
|
|
1884
|
-
|
|
2541
|
+
// An animated file has no quality sweep: its saving is an encode (see video-analysis.ts), not a re-encode of the still.
|
|
2542
|
+
if (inst.kind !== "srcset" || !inst.right || inst.animated !== undefined)
|
|
1885
2543
|
continue;
|
|
1886
2544
|
if (ctx.qualityFor && !ctx.qualityFor(inst))
|
|
1887
2545
|
continue;
|
|
@@ -2018,9 +2676,27 @@ export function applyQuality(inst) {
|
|
|
2018
2676
|
/** The `robots` value recorded for a page on a loopback or private host: its owner's own server is not checked. */
|
|
2019
2677
|
export const ROBOTS_SKIPPED_LOCAL = "skipped (local host)";
|
|
2020
2678
|
/** The options of one page's output, for the completeness check of --resume. */
|
|
2021
|
-
export function recordOptionsOf(opts) {
|
|
2679
|
+
export function recordOptionsOf(opts, ffmpeg, vectorizer) {
|
|
2022
2680
|
const thirdParty = opts.thirdParty ?? DEFAULT_THIRD_PARTY;
|
|
2681
|
+
// The media settings are recorded when encodes can run, so a page scanned under other settings (or without ffmpeg) is
|
|
2682
|
+
// not reused by --resume. Without encodes they are left out, and the record keeps its shape.
|
|
2683
|
+
const encodes = opts.quality === true && opts.survey !== true && ffmpeg !== undefined;
|
|
2684
|
+
const media = encodes
|
|
2685
|
+
? {
|
|
2686
|
+
mediaBudget: opts.mediaBudget ?? DEFAULT_MEDIA_BUDGET,
|
|
2687
|
+
mediaTime: opts.mediaTime ?? DEFAULT_MEDIA_TIME,
|
|
2688
|
+
videoBar: opts.videoBar ?? DEFAULT_VIDEO_BAR,
|
|
2689
|
+
videoStillBar: opts.videoStillBar ?? DEFAULT_VIDEO_STILL_BAR,
|
|
2690
|
+
maxVideoWidths: opts.maxVideoWidths ?? SCAN_MAX_VIDEO_WIDTHS,
|
|
2691
|
+
ffmpeg,
|
|
2692
|
+
}
|
|
2693
|
+
: {};
|
|
2694
|
+
const vector = opts.quality === true && opts.survey !== true
|
|
2695
|
+
? { svgBudget: opts.svgBudget ?? DEFAULT_SVG_BUDGET, vectorizer: vectorizer === true }
|
|
2696
|
+
: {};
|
|
2023
2697
|
return {
|
|
2698
|
+
...media,
|
|
2699
|
+
...vector,
|
|
2024
2700
|
imgtuneVersion: IMGTUNE_VERSION,
|
|
2025
2701
|
aggressive: opts.aggressive,
|
|
2026
2702
|
quality: opts.quality,
|
|
@@ -2028,6 +2704,7 @@ export function recordOptionsOf(opts) {
|
|
|
2028
2704
|
ignoreRobots: opts.ignoreRobots === true,
|
|
2029
2705
|
storageState: opts.storageState !== undefined,
|
|
2030
2706
|
scorer: opts.gpu !== undefined ? "gpu" : "cpu",
|
|
2707
|
+
resources: opts.resources !== false,
|
|
2031
2708
|
...(opts.harness === true ? { harness: true } : {}),
|
|
2032
2709
|
...(opts.survey === true ? { survey: true } : {}),
|
|
2033
2710
|
...(opts.sharedLoads === true ? { loads: "shared" } : {}),
|
|
@@ -2098,10 +2775,11 @@ function toReport(inst) {
|
|
|
2098
2775
|
* Groups by saving: the group's exact + est. bytes per page view, summed over the page's viewports and divided by their
|
|
2099
2776
|
* count (as pageNumbers does), with two sample URLs and a sizes hint.
|
|
2100
2777
|
*/
|
|
2101
|
-
export function groupReports(instances, viewportCount) {
|
|
2778
|
+
export function groupReports(instances, viewportCount, framework = false) {
|
|
2102
2779
|
const byKey = new Map();
|
|
2103
2780
|
for (const inst of instances) {
|
|
2104
|
-
|
|
2781
|
+
// A hidden file is not a group member: its saving is none, and its size is not a size the page shows.
|
|
2782
|
+
if (inst.kind === "skipped" || inst.kind === "hidden")
|
|
2105
2783
|
continue;
|
|
2106
2784
|
const list = byKey.get(inst.group);
|
|
2107
2785
|
if (list)
|
|
@@ -2112,11 +2790,15 @@ export function groupReports(instances, viewportCount) {
|
|
|
2112
2790
|
const reports = [];
|
|
2113
2791
|
for (const [key, members] of byKey) {
|
|
2114
2792
|
const sizes = members[0].img.sizes;
|
|
2793
|
+
// Plain images with no srcset: a sizes proposal does nothing there, so the group is told the variant widths instead.
|
|
2794
|
+
const plainGroup = members.every((m) => isPlainNoSrcset(m.kind, m.img.srcset, m.loader, framework));
|
|
2795
|
+
const naturals = members.map((m) => m.pixelWidth).filter((w) => w > 0);
|
|
2796
|
+
const srcsetWidths = plainGroup ? srcsetWidthsOf(members.map((m) => m.needed), naturals.length > 0 ? Math.min(...naturals) : undefined) : [];
|
|
2115
2797
|
const points = members
|
|
2116
2798
|
.filter((m) => m.renderedWidth >= 1)
|
|
2117
2799
|
.map((m) => ({ viewportWidth: m.viewport.width, renderedWidth: m.renderedWidth, mobile: m.viewport.mobile }));
|
|
2118
2800
|
// A proposal that leaves any measured viewport undersized is not a saving: it is dropped, with its exact bytes.
|
|
2119
|
-
const proposed = deriveSizes(points);
|
|
2801
|
+
const proposed = plainGroup ? undefined : deriveSizes(points);
|
|
2120
2802
|
const undersized = proposed === undefined
|
|
2121
2803
|
? undefined
|
|
2122
2804
|
: undersizedProposal(proposed, members.map((m) => ({ viewport: m.viewport.name, viewportWidth: m.viewport.width, dpr: m.viewport.dpr, renderedWidth: m.renderedWidth, srcset: m.img.srcset })), (srcset) => parseSrcset(srcset, "http://imgtune.invalid/"));
|
|
@@ -2147,6 +2829,7 @@ export function groupReports(instances, viewportCount) {
|
|
|
2147
2829
|
members: new Set(members.map((m) => m.element)).size,
|
|
2148
2830
|
samples,
|
|
2149
2831
|
hint: suggestedSizes && suggestedSizes !== sizes ? `set sizes="${suggestedSizes}"` : undefined,
|
|
2832
|
+
...(srcsetWidths.length > 0 ? { srcset: { widths: srcsetWidths } } : {}),
|
|
2150
2833
|
});
|
|
2151
2834
|
}
|
|
2152
2835
|
return reports.sort((a, b) => b.savedBytes - a.savedBytes).slice(0, TOP_GROUPS);
|
|
@@ -2156,7 +2839,9 @@ function referenceViewportName(viewports) {
|
|
|
2156
2839
|
return viewports.find((v) => v.name === "iphone-15")?.name ?? viewports.find((v) => v.mobile)?.name ?? viewports[0].name;
|
|
2157
2840
|
}
|
|
2158
2841
|
/** Turns the observations of one page into the report. Fetches and local encodes happen here. */
|
|
2159
|
-
export async function analysePage(ctx,
|
|
2842
|
+
export async function analysePage(ctx, input, base, resourcesJob = () => resourcesOf(ctx, input)) {
|
|
2843
|
+
// A fetched video poster is measured as an image, sized to its video's box (see video-analysis.ts).
|
|
2844
|
+
const observations = withPosterImages(input);
|
|
2160
2845
|
const refName = referenceViewportName(observations.map((o) => o.viewport));
|
|
2161
2846
|
const instances = [];
|
|
2162
2847
|
for (const obs of observations) {
|
|
@@ -2222,6 +2907,19 @@ export async function analysePage(ctx, observations, base) {
|
|
|
2222
2907
|
}
|
|
2223
2908
|
}
|
|
2224
2909
|
}
|
|
2910
|
+
// A hidden layer whose file a shown element also loads adds no bytes of its own: it is not a hidden copy (skipped).
|
|
2911
|
+
const shownUrls = new Set([
|
|
2912
|
+
...instances.filter((i) => i.kind !== "hidden" && i.url !== "").map((i) => i.url),
|
|
2913
|
+
...bgInstances.filter((i) => i.hidden === undefined && i.url !== "").map((i) => i.url),
|
|
2914
|
+
]);
|
|
2915
|
+
for (const inst of bgInstances) {
|
|
2916
|
+
if (inst.hidden !== undefined && shownUrls.has(inst.url)) {
|
|
2917
|
+
inst.hidden = undefined;
|
|
2918
|
+
inst.inPanel = undefined;
|
|
2919
|
+
inst.hover = undefined;
|
|
2920
|
+
inst.skipped = "hidden: file shared with a shown image";
|
|
2921
|
+
}
|
|
2922
|
+
}
|
|
2225
2923
|
const bgSources = await resolveBackgroundSources(ctx, observations);
|
|
2226
2924
|
const perViewport = {};
|
|
2227
2925
|
const totals = [];
|
|
@@ -2256,7 +2954,7 @@ export async function analysePage(ctx, observations, base) {
|
|
|
2256
2954
|
const mine = instances.filter((i) => i.element === e);
|
|
2257
2955
|
if (mine.every((i) => i.kind === "skipped" && i.skipped !== "duplicate"))
|
|
2258
2956
|
skippedElements.add(e);
|
|
2259
|
-
if (mine.some((i) => i.kind !== "skipped" && i.qualityRatio === undefined))
|
|
2957
|
+
if (mine.some((i) => i.kind !== "skipped" && i.kind !== "hidden" && i.qualityRatio === undefined))
|
|
2260
2958
|
naElements.add(e);
|
|
2261
2959
|
}
|
|
2262
2960
|
const numbers = measure("aggregate", () => pageNumbers(totals));
|
|
@@ -2269,6 +2967,7 @@ export async function analysePage(ctx, observations, base) {
|
|
|
2269
2967
|
refName,
|
|
2270
2968
|
resolved: sources,
|
|
2271
2969
|
currentQuality,
|
|
2970
|
+
nextjs: observations.some((o) => o.nextjs),
|
|
2272
2971
|
});
|
|
2273
2972
|
// A proposed sizes is a saving only when its predicted outcome saves more bytes than it adds (sizes-outcome.ts).
|
|
2274
2973
|
const widths = new Map(observations.map((o) => [o.viewport.name, o.viewport.width]));
|
|
@@ -2281,12 +2980,24 @@ export async function analysePage(ctx, observations, base) {
|
|
|
2281
2980
|
refName,
|
|
2282
2981
|
sources: bgSources,
|
|
2283
2982
|
});
|
|
2983
|
+
// The page's loaded-but-not-shown numbers are counted over the same records the report lists (images and background layers).
|
|
2984
|
+
const hiddenTotals = hiddenTotalsOf([...images, ...backgroundRecords], observations.length);
|
|
2985
|
+
const videoSources = await resolveVideoSources(ctx, input);
|
|
2986
|
+
const videos = videoRecordsOf(ctx.pageUrl, input, fullBytesOf(input), "scan", ctx.maxVideoWidths ?? SCAN_MAX_VIDEO_WIDTHS, videoSources);
|
|
2987
|
+
// The resource phase runs when the page was captured with resources (see readGroup); its findings are the page's.
|
|
2988
|
+
const resourcesFound = await resourcesJob();
|
|
2989
|
+
// An image never shown on load is reported once, as loaded-but-not-shown (its bytes are that fix's): an svg or text file
|
|
2990
|
+
// behind it gets no resource finding too, so the same bytes are not claimed twice. A favicon is not an image on the page.
|
|
2991
|
+
const neverShown = new Set([...images, ...backgroundRecords].flatMap((r) => (r.hidden !== undefined && r.hidden.shownIn.length === 0 ? [r.current.src] : [])).filter((src) => src !== undefined && src !== ""));
|
|
2992
|
+
const resources = resourcesFound?.filter((finding) => finding.kind === "favicon" || !neverShown.has(finding.url));
|
|
2284
2993
|
return {
|
|
2285
2994
|
...base,
|
|
2286
2995
|
nextjs: observations.some((o) => o.nextjs),
|
|
2996
|
+
...(videos.length > 0 ? { videos } : {}),
|
|
2997
|
+
...(resources !== undefined ? { resources } : {}),
|
|
2287
2998
|
perViewport,
|
|
2288
2999
|
page: ctx.aggressive ? { ...withoutAggressive, aggressiveExactEstPct } : withoutAggressive,
|
|
2289
|
-
groups: measure("aggregate", () => groupReports(instances, observations.length)),
|
|
3000
|
+
groups: measure("aggregate", () => groupReports(instances, observations.length, observations.some((o) => o.nextjs))),
|
|
2290
3001
|
counts: {
|
|
2291
3002
|
images: elements.size,
|
|
2292
3003
|
qualityNA: naElements.size,
|
|
@@ -2297,6 +3008,8 @@ export async function analysePage(ctx, observations, base) {
|
|
|
2297
3008
|
thirdPartyBlocked: observations.reduce((sum, o) => sum + (o.thirdPartyBlocked ?? 0), 0),
|
|
2298
3009
|
images: [...images, ...backgroundRecords],
|
|
2299
3010
|
notLoaded: images.filter((r) => r.lazy === true && !r.perViewport.some((p) => p.loaded)).length,
|
|
3011
|
+
...(hiddenTotals !== undefined ? { hidden: hiddenTotals } : {}),
|
|
3012
|
+
...(revealCappedOf(input) > 0 ? { revealCapped: revealCappedOf(input) } : {}),
|
|
2300
3013
|
...(bgInstances.length > 0 ? { backgrounds: backgroundPage(bgInstances, observations.length, backgroundRecords) } : {}),
|
|
2301
3014
|
...(ctx.harness === true ? { harness: true } : {}),
|
|
2302
3015
|
};
|
|
@@ -2386,7 +3099,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2386
3099
|
const groupOutcomes = await settleLimit(groups, run.viewportConcurrency, (group) => {
|
|
2387
3100
|
count("page loads");
|
|
2388
3101
|
count("viewport loads", group.length);
|
|
2389
|
-
return observeLoads(browser, pageUrl, group, guard, run.loadCapMs, run.storageState, false, undefined, run.idleTimeoutMs);
|
|
3102
|
+
return observeLoads(browser, pageUrl, group, guard, run.loadCapMs, run.storageState, false, undefined, run.idleTimeoutMs, false, run.resources);
|
|
2390
3103
|
});
|
|
2391
3104
|
const outcomes = run.viewports.map((viewport) => {
|
|
2392
3105
|
const index = groups.findIndex((group) => group.includes(viewport));
|
|
@@ -2398,6 +3111,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2398
3111
|
let loadTimedOut = false;
|
|
2399
3112
|
const sheetsFailed = {};
|
|
2400
3113
|
const failedHosts = new Set();
|
|
3114
|
+
let loadedSheets = 0;
|
|
2401
3115
|
const documentStatus = {};
|
|
2402
3116
|
try {
|
|
2403
3117
|
for (const [index, outcome] of outcomes.entries()) {
|
|
@@ -2414,6 +3128,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2414
3128
|
sheetsFailed[viewport.name] = obs.stylesheetsFailed;
|
|
2415
3129
|
for (const host of obs.stylesheetHosts ?? [])
|
|
2416
3130
|
failedHosts.add(host);
|
|
3131
|
+
loadedSheets = Math.max(loadedSheets, obs.stylesheetsLoaded ?? 0);
|
|
2417
3132
|
if (obs.status !== undefined && obs.status >= 400)
|
|
2418
3133
|
documentStatus[viewport.name] = obs.status;
|
|
2419
3134
|
if (obs.finalUrl !== undefined && new URL(obs.finalUrl).href !== url.href) {
|
|
@@ -2440,6 +3155,7 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2440
3155
|
described.documentStatus = documentStatus;
|
|
2441
3156
|
if (Object.keys(sheetsFailed).length > 0) {
|
|
2442
3157
|
described.stylesheetsFailed = sheetsFailed;
|
|
3158
|
+
described.stylesheetsLoaded = loadedSheets;
|
|
2443
3159
|
run.log(`warning: ${safeText(pageUrl)}: a linked stylesheet did not load at ${Object.keys(sheetsFailed).map((name) => safeText(name)).join(", ")}; the page is not styled as the app is, so its widths and sizes verdicts are not valid (check the asset prefix the server was started with)`);
|
|
2444
3160
|
}
|
|
2445
3161
|
if (robotsIgnored)
|
|
@@ -2448,13 +3164,41 @@ async function scanPage(browser, run, pageUrl, robotsFor) {
|
|
|
2448
3164
|
described.robotsRule = robotsRule;
|
|
2449
3165
|
if (finalUrl !== undefined)
|
|
2450
3166
|
described.redirectNote = `the page ends on ${finalUrl}; measured there`;
|
|
2451
|
-
if (run.stage === undefined)
|
|
2452
|
-
|
|
3167
|
+
if (run.stage === undefined) {
|
|
3168
|
+
const ctx = analysisContextOf(run, pageUrl);
|
|
3169
|
+
const resources = pageResourceJob(ctx, async () => observations);
|
|
3170
|
+
return {
|
|
3171
|
+
resources,
|
|
3172
|
+
analyse: (store, withResources = true) => analysePage({ ...ctx, store }, observations, described, withResources ? resources : NO_RESOURCES),
|
|
3173
|
+
};
|
|
3174
|
+
}
|
|
2453
3175
|
return stagedMeasure(run, run.stage, pageUrl, described, observations);
|
|
2454
3176
|
}
|
|
3177
|
+
/**
|
|
3178
|
+
* The resource findings of a page's observations (see resources/), or undefined when the page was captured without resources.
|
|
3179
|
+
* Never throws: a failed analysis gives undefined, so it does not fail the page.
|
|
3180
|
+
*/
|
|
3181
|
+
async function resourcesOf(ctx, observations) {
|
|
3182
|
+
if (!observations.some((o) => o.resources !== undefined))
|
|
3183
|
+
return undefined;
|
|
3184
|
+
try {
|
|
3185
|
+
return await measure("resources", () => analyzeResources(observations.map((o) => ({ viewport: o.viewport, resources: o.resources, pageContext: o.pageContext })), ctx.pageUrl, { cache: ctx.deps.cache, fetcher: ctx.fetcher, allowPrivate: ctx.allowPrivate, thirdParty: ctx.thirdParty, run: ctx.deps.resourceRun }));
|
|
3186
|
+
}
|
|
3187
|
+
catch {
|
|
3188
|
+
return undefined;
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
/** A page's resource analysis, started at most once: the phase-1 pool starts it, and the estimate pass reads the same job. */
|
|
3192
|
+
export function pageResourceJob(ctx, observations) {
|
|
3193
|
+
let job;
|
|
3194
|
+
return () => (job ??= observations().then((obs) => resourcesOf(ctx, obs)).catch(() => undefined));
|
|
3195
|
+
}
|
|
3196
|
+
/** No resource analysis: the provisional pass of a page (its resources are replaced by the estimate pass's). */
|
|
3197
|
+
const NO_RESOURCES = () => Promise.resolve(undefined);
|
|
2455
3198
|
function analysisContextOf(run, pageUrl) {
|
|
2456
3199
|
return {
|
|
2457
3200
|
pageUrl,
|
|
3201
|
+
maxVideoWidths: run.maxVideoWidths,
|
|
2458
3202
|
fetcher: run.fetcher,
|
|
2459
3203
|
allowPrivate: isLocalTarget(pageUrl),
|
|
2460
3204
|
deps: run.deps,
|
|
@@ -2466,6 +3210,8 @@ function analysisContextOf(run, pageUrl) {
|
|
|
2466
3210
|
sourceMaps: run.sourceMaps,
|
|
2467
3211
|
projectDir: run.projectDir,
|
|
2468
3212
|
projectRoot: run.projectRoot,
|
|
3213
|
+
thirdParty: run.thirdParty,
|
|
3214
|
+
flatBuffers: run.flatBuffers,
|
|
2469
3215
|
};
|
|
2470
3216
|
}
|
|
2471
3217
|
/**
|
|
@@ -2476,19 +3222,33 @@ function stagedMeasure(run, stage, pageUrl, described, live) {
|
|
|
2476
3222
|
const file = join(stage.dir, pageFileName(new URL(pageUrl)));
|
|
2477
3223
|
let pending = live;
|
|
2478
3224
|
const ctx = analysisContextOf(run, pageUrl);
|
|
3225
|
+
// The observations the resource job reads: the live ones while the page is in memory (phase 1 starts the job before the stage
|
|
3226
|
+
// drops them), else the staged file's.
|
|
3227
|
+
const resources = pageResourceJob(ctx, async () => pending ?? (await unstagedObservations(stage.dir, file)));
|
|
2479
3228
|
return {
|
|
3229
|
+
resources,
|
|
2480
3230
|
stage: async () => {
|
|
2481
3231
|
if (pending === undefined)
|
|
2482
3232
|
return;
|
|
2483
3233
|
const staged = [];
|
|
2484
3234
|
for (const obs of pending) {
|
|
2485
|
-
const { responses, ...rest } = obs;
|
|
3235
|
+
const { responses, media, resources, ...rest } = obs;
|
|
2486
3236
|
const records = [];
|
|
2487
3237
|
for (const [url, record] of responses) {
|
|
2488
3238
|
const { buffer, ...meta } = record;
|
|
2489
3239
|
records.push({ url, record: meta, blob: buffer === undefined ? undefined : await putBlob(stage.dir, buffer) });
|
|
2490
3240
|
}
|
|
2491
|
-
|
|
3241
|
+
const stagedResources = [];
|
|
3242
|
+
for (const resource of resources ?? []) {
|
|
3243
|
+
const { body, ...meta } = resource;
|
|
3244
|
+
stagedResources.push({ record: meta, blob: body === undefined ? undefined : await putBlob(stage.dir, body) });
|
|
3245
|
+
}
|
|
3246
|
+
staged.push({
|
|
3247
|
+
...rest,
|
|
3248
|
+
responses: records,
|
|
3249
|
+
...(media !== undefined ? { media: [...media] } : {}),
|
|
3250
|
+
...(resources !== undefined ? { resources: stagedResources } : {}),
|
|
3251
|
+
});
|
|
2492
3252
|
}
|
|
2493
3253
|
const state = { version: 1, url: pageUrl, options: stage.options, described, observations: staged };
|
|
2494
3254
|
await writeStagedJson(stage.dir, file, state);
|
|
@@ -2499,9 +3259,9 @@ function stagedMeasure(run, stage, pageUrl, described, live) {
|
|
|
2499
3259
|
if (state !== undefined)
|
|
2500
3260
|
await writeStagedJson(stage.dir, file, { ...state, images });
|
|
2501
3261
|
},
|
|
2502
|
-
analyse: async (store) => {
|
|
3262
|
+
analyse: async (store, withResources = true) => {
|
|
2503
3263
|
const observations = pending ?? (await unstagedObservations(stage.dir, file));
|
|
2504
|
-
return analysePage({ ...ctx, store }, observations, described);
|
|
3264
|
+
return analysePage({ ...ctx, store }, observations, described, withResources ? resources : NO_RESOURCES);
|
|
2505
3265
|
},
|
|
2506
3266
|
finish: async () => {
|
|
2507
3267
|
await rm(file, { force: true });
|
|
@@ -2515,12 +3275,21 @@ async function unstagedObservations(stageDir, file) {
|
|
|
2515
3275
|
throw new Error(`staged page is missing or unreadable: ${safeText(file)}`);
|
|
2516
3276
|
const observations = [];
|
|
2517
3277
|
for (const staged of state.observations) {
|
|
2518
|
-
const { responses: records, ...rest } = staged;
|
|
3278
|
+
const { responses: records, media, resources, ...rest } = staged;
|
|
2519
3279
|
const responses = new Map();
|
|
2520
3280
|
for (const { url, record, blob } of records) {
|
|
2521
3281
|
responses.set(url, blob === undefined ? { ...record } : { ...record, buffer: await getBlob(stageDir, blob) });
|
|
2522
3282
|
}
|
|
2523
|
-
|
|
3283
|
+
const resourceRecords = [];
|
|
3284
|
+
for (const { record, blob } of resources ?? []) {
|
|
3285
|
+
resourceRecords.push(blob === undefined ? { ...record } : { ...record, body: await getBlob(stageDir, blob) });
|
|
3286
|
+
}
|
|
3287
|
+
observations.push({
|
|
3288
|
+
...rest,
|
|
3289
|
+
responses,
|
|
3290
|
+
...(media !== undefined ? { media: new Map(media) } : {}),
|
|
3291
|
+
...(resources !== undefined ? { resources: resourceRecords } : {}),
|
|
3292
|
+
});
|
|
2524
3293
|
}
|
|
2525
3294
|
return observations;
|
|
2526
3295
|
}
|
|
@@ -2536,10 +3305,11 @@ async function resumeStaged(run, url) {
|
|
|
2536
3305
|
if (state === undefined || state.url !== url || state.images === undefined || !sameRecordOptions(state.options, stage.options))
|
|
2537
3306
|
return undefined;
|
|
2538
3307
|
for (const obs of state.observations) {
|
|
2539
|
-
|
|
2540
|
-
|
|
3308
|
+
const blobs = [...obs.responses, ...(obs.resources ?? [])].map((entry) => entry.blob);
|
|
3309
|
+
for (const blob of blobs) {
|
|
3310
|
+
if (blob === undefined)
|
|
2541
3311
|
continue;
|
|
2542
|
-
if (!(await stat(join(stage.dir, "blobs",
|
|
3312
|
+
if (!(await stat(join(stage.dir, "blobs", blob)).then(() => true, () => false)))
|
|
2543
3313
|
return undefined;
|
|
2544
3314
|
}
|
|
2545
3315
|
}
|
|
@@ -2574,6 +3344,7 @@ async function withBrowser(opts, gpu, body) {
|
|
|
2574
3344
|
const deps = {
|
|
2575
3345
|
score: makeScorer("ssimulacra2", { projectDir: opts.cwd, gpu }),
|
|
2576
3346
|
cache,
|
|
3347
|
+
resourceRun: newResourceRun((opts.resourceTime ?? DEFAULT_RESOURCE_TIME_S) * 1000),
|
|
2577
3348
|
scorerId: gpu !== undefined ? `gpu:${gpu.deviceName}` : "cpu",
|
|
2578
3349
|
cpuScored: gpu !== undefined ? () => gpu.cpuScores : undefined,
|
|
2579
3350
|
};
|
|
@@ -2589,14 +3360,18 @@ async function withBrowser(opts, gpu, body) {
|
|
|
2589
3360
|
ignoreRobots: opts.ignoreRobots === true,
|
|
2590
3361
|
harness: opts.harness === true,
|
|
2591
3362
|
sourceMaps: new Map(),
|
|
3363
|
+
// The browser's own bytes of the flat images the vector check may trace (see FlatBuffers); none without the check.
|
|
3364
|
+
flatBuffers: new FlatBuffers(opts.quality && opts.survey !== true ? (opts.svgBudget ?? DEFAULT_SVG_BUDGET) : 0),
|
|
2592
3365
|
storageState: opts.storageState,
|
|
2593
3366
|
thirdParty: opts.thirdParty ?? DEFAULT_THIRD_PARTY,
|
|
2594
3367
|
qualityFor: opts.qualityFor,
|
|
3368
|
+
maxVideoWidths: opts.maxVideoWidths,
|
|
2595
3369
|
projectDir: opts.cwd,
|
|
2596
3370
|
projectRoot: opts.root,
|
|
2597
3371
|
viewportConcurrency: opts.viewportConcurrency ?? VIEWPORT_CONCURRENCY,
|
|
2598
3372
|
idleTimeoutMs: opts.idleTimeoutMs ?? NETWORK_IDLE_TIMEOUT_MS,
|
|
2599
3373
|
sharedLoads: opts.sharedLoads === true,
|
|
3374
|
+
resources: opts.resources !== false,
|
|
2600
3375
|
estimates: opts.survey !== true,
|
|
2601
3376
|
log: opts.log,
|
|
2602
3377
|
gpu,
|
|
@@ -2650,6 +3425,70 @@ export function progressLine(done, total) {
|
|
|
2650
3425
|
const status = last.status === "ok" ? "" : ` (${last.status})`;
|
|
2651
3426
|
return `page ${done.length}/${total} done${status} · ${(measured / (1024 * 1024)).toFixed(1)} MB measured · est. ${estimated.toFixed(0)}%`;
|
|
2652
3427
|
}
|
|
3428
|
+
/**
|
|
3429
|
+
* The media phase of a run (see media-phase.ts), run once after every page is measured. Each asset is first checked against
|
|
3430
|
+
* the third-party policy from the page it was found on: a blocked one is never fetched, read or proxied. The rest go to
|
|
3431
|
+
* runMediaPhase (see mediaPhaseFor).
|
|
3432
|
+
*/
|
|
3433
|
+
async function runMediaFor(assets, run, opts, urls) {
|
|
3434
|
+
const { allowed, blocked } = partitionByPolicy(assets, run.thirdParty);
|
|
3435
|
+
const results = await mediaPhaseFor(allowed, run, opts, urls);
|
|
3436
|
+
return blocked.size > 0 ? { ...results, blocked } : results;
|
|
3437
|
+
}
|
|
3438
|
+
/**
|
|
3439
|
+
* The media phase for the assets the policy allows. The videos are read through an egress proxy started for the phase,
|
|
3440
|
+
* which allows private destinations only when every scanned URL is local. The length of every video URL is probed first,
|
|
3441
|
+
* whatever the settings (a click-to-play video's per-play bytes need it, and no ffmpeg); the budget ranks the videos by
|
|
3442
|
+
* that length. A survey or a scan with quality off encodes nothing, and each asset says so. Never throws.
|
|
3443
|
+
*/
|
|
3444
|
+
async function mediaPhaseFor(assets, run, opts, urls) {
|
|
3445
|
+
if (assets.length === 0)
|
|
3446
|
+
return { byKey: new Map() };
|
|
3447
|
+
const allowPrivate = urls.length > 0 && urls.every((url) => isLocalTarget(url));
|
|
3448
|
+
let proxy;
|
|
3449
|
+
// Declared outside the try: a media error after the probe keeps the lengths, so a click-to-play video's per-play bytes stay.
|
|
3450
|
+
let lengths = new Map();
|
|
3451
|
+
try {
|
|
3452
|
+
const videoUrls = assets.filter((asset) => asset.kind === "video").map((asset) => asset.url);
|
|
3453
|
+
if (videoUrls.length > 0) {
|
|
3454
|
+
proxy = await startEgressProxy({ resolver: run.resolver, allowPrivate, connectAddress: run.connectAddress });
|
|
3455
|
+
lengths = await probeLengths(assets.filter((asset) => asset.kind === "video").map((asset) => ({ url: asset.url, page: asset.page })), proxy.url, { thirdParty: run.thirdParty });
|
|
3456
|
+
}
|
|
3457
|
+
const sized = assets.map((asset) => refineAssetLength(asset, lengths.get(asset.url)));
|
|
3458
|
+
if (opts.quality !== true || opts.survey === true) {
|
|
3459
|
+
return { ...skippedMedia(sized, opts.survey === true ? "survey" : "quality off"), lengths };
|
|
3460
|
+
}
|
|
3461
|
+
// A video no viewport shows is not encoded: its record keeps the probe's bytes, and the reason is printed (see splitNeverShown).
|
|
3462
|
+
const { encode, neverShown } = splitNeverShown(sized);
|
|
3463
|
+
const encoded = await runMediaPhase(sized, {
|
|
3464
|
+
budget: opts.mediaBudget ?? DEFAULT_MEDIA_BUDGET,
|
|
3465
|
+
timeS: opts.mediaTime ?? DEFAULT_MEDIA_TIME,
|
|
3466
|
+
vmafBar: opts.videoBar ?? DEFAULT_VIDEO_BAR,
|
|
3467
|
+
stillBar: opts.videoStillBar ?? DEFAULT_VIDEO_STILL_BAR,
|
|
3468
|
+
survey: false,
|
|
3469
|
+
workDir: join(resolve(opts.outDir), ".media-work"),
|
|
3470
|
+
cache: run.deps.cache,
|
|
3471
|
+
log: opts.log,
|
|
3472
|
+
proxy: proxy?.url,
|
|
3473
|
+
score: run.deps.score,
|
|
3474
|
+
scorerId: run.deps.scorerId,
|
|
3475
|
+
// The media is encoded on the CPU unless --gpu-encode asks for the GPUs: the --gpu-device list, else GPU 0.
|
|
3476
|
+
gpus: opts.gpuEncode === true ? (opts.gpu?.devices ?? [0]) : undefined,
|
|
3477
|
+
signal: opts.signal,
|
|
3478
|
+
thirdParty: run.thirdParty,
|
|
3479
|
+
}, {
|
|
3480
|
+
fetch: (url, maxBytes, hop) => run.fetcher.get(url, { maxBytes: maxBytes ?? MAX_IMAGE_BYTES, allowPrivate, allowHop: hop?.allowHop, allowHopKey: hop?.page }),
|
|
3481
|
+
});
|
|
3482
|
+
return { ...encoded, byKey: new Map([...encoded.byKey, ...skippedMedia(neverShown, NEVER_SHOWN).byKey]), lengths };
|
|
3483
|
+
}
|
|
3484
|
+
catch (err) {
|
|
3485
|
+
const message = (err instanceof Error ? err.message : String(err)).replace(/\s+/g, " ").slice(0, 200);
|
|
3486
|
+
return { ...skippedMedia(assets, `media error: ${message}`), lengths };
|
|
3487
|
+
}
|
|
3488
|
+
finally {
|
|
3489
|
+
await proxy?.close();
|
|
3490
|
+
}
|
|
3491
|
+
}
|
|
2653
3492
|
/**
|
|
2654
3493
|
* Scans the pages in two phases, with a progress line after each. Phase 1 measures every page (up to `pageConcurrency` in
|
|
2655
3494
|
* flight at once on the shared browser; pages start in input order) and collects the estimate jobs they need, so no
|
|
@@ -2665,14 +3504,20 @@ export function progressLine(done, total) {
|
|
|
2665
3504
|
export async function runScan(urls, opts) {
|
|
2666
3505
|
return withScanRun(opts, async (browser, run, robotsFor) => {
|
|
2667
3506
|
await mkdir(opts.outDir, { recursive: true });
|
|
2668
|
-
|
|
3507
|
+
// Whether ffmpeg is installed decides whether the media encodes run, so it is part of the record (see recordOptionsOf).
|
|
3508
|
+
const vectorizer = (await loadVectorizer()) !== undefined;
|
|
3509
|
+
const options = recordOptionsOf(opts, (await probeFfmpeg()).ffmpeg, vectorizer);
|
|
2669
3510
|
run.stage = { dir: stageDirOf(opts.outDir), options };
|
|
2670
3511
|
const results = new Array(urls.length);
|
|
2671
3512
|
// The finished pages in finishing order: the progress line counts them.
|
|
2672
3513
|
const finished = [];
|
|
2673
3514
|
const registry = new EstimateRegistry();
|
|
2674
3515
|
const collector = registry.collector();
|
|
3516
|
+
// The resource analyses run in this pool while later pages are measured (see resources/pool.ts).
|
|
3517
|
+
const resourcePool = new ResourcePool(RESOURCE_PARALLEL, (error) => opts.log(`resources: a page's analysis failed: ${safeText(String(error))}`));
|
|
2675
3518
|
const measured = [];
|
|
3519
|
+
// The media assets (videos and animated images) of every measured page, from its collecting pass (see media-phase.ts).
|
|
3520
|
+
const mediaAssets = [];
|
|
2676
3521
|
let started = 0;
|
|
2677
3522
|
let images = 0;
|
|
2678
3523
|
let measuredCount = 0;
|
|
@@ -2697,7 +3542,8 @@ export async function runScan(urls, opts) {
|
|
|
2697
3542
|
// A page measured by an interrupted run: its estimates only. Its requests are collected as a measured page's are.
|
|
2698
3543
|
const staged = opts.resume ? await resumeStaged(run, url) : undefined;
|
|
2699
3544
|
if (staged !== undefined) {
|
|
2700
|
-
await staged.page.analyse(collector);
|
|
3545
|
+
const stagedProvisional = await staged.page.analyse(collector, false);
|
|
3546
|
+
mediaAssets.push(...mediaAssetsOf(stagedProvisional));
|
|
2701
3547
|
measured.push({ index, url, page: staged.page });
|
|
2702
3548
|
images += staged.images;
|
|
2703
3549
|
measuredCount++;
|
|
@@ -2717,8 +3563,15 @@ export async function runScan(urls, opts) {
|
|
|
2717
3563
|
}
|
|
2718
3564
|
// The collecting pass: the page's estimate requests are recorded; its own numbers are provisional and replaced below.
|
|
2719
3565
|
// Staged as soon as it is measured: the collecting pass reads it back from the stage, so no bytes are held meanwhile.
|
|
3566
|
+
// The resource analysis starts now, pooled, so it overlaps the next pages' measurement. It reads the observations before the
|
|
3567
|
+
// stage drops them from memory.
|
|
3568
|
+
if (outcome.resources !== undefined) {
|
|
3569
|
+
await resourcePool.admit();
|
|
3570
|
+
void resourcePool.start(outcome.resources);
|
|
3571
|
+
}
|
|
2720
3572
|
await outcome.stage?.();
|
|
2721
|
-
const provisional = await outcome.analyse(collector);
|
|
3573
|
+
const provisional = await outcome.analyse(collector, false);
|
|
3574
|
+
mediaAssets.push(...mediaAssetsOf(provisional));
|
|
2722
3575
|
await outcome.countImages?.(provisional.counts?.images ?? 0);
|
|
2723
3576
|
measured.push({ index, url, page: outcome });
|
|
2724
3577
|
images += provisional.counts?.images ?? 0;
|
|
@@ -2731,15 +3584,41 @@ export async function runScan(urls, opts) {
|
|
|
2731
3584
|
const counts = registry.counts();
|
|
2732
3585
|
opts.log(`estimates: ${counts.unique} unique of ${counts.requests}, ${counts.skipped} skipped`);
|
|
2733
3586
|
await registry.runJobs(QUALITY_PARALLEL);
|
|
3587
|
+
// Each page's image analysis is written as soon as its estimates are in, with no media results and no completeness
|
|
3588
|
+
// marker: a crash or Ctrl-C during the long media phase keeps it (--resume scans it again and readers skip it).
|
|
3589
|
+
const analysedPages = [];
|
|
2734
3590
|
for (const { index, url, page } of measured) {
|
|
2735
3591
|
const tally = { fallbacks: 0 };
|
|
2736
3592
|
const analysed = await page.analyse(registry.reader(tally));
|
|
2737
|
-
|
|
3593
|
+
await writePage(url, withBroken({ ...analysed, options, scorer: scorerOf(run.gpu, tally.fallbacks) }, opts.expectImages?.(url)));
|
|
3594
|
+
analysedPages.push({ index, url, page, analysed, fallbacks: tally.fallbacks });
|
|
3595
|
+
opts.log(progressLine([...finished, ...analysedPages.map((p) => p.analysed)], urls.length));
|
|
3596
|
+
}
|
|
3597
|
+
// The vector check runs once over the flat images of every page (see vector-phase.ts), before the media phase.
|
|
3598
|
+
const allowPrivate = urls.length > 0 && urls.every((url) => isLocalTarget(url));
|
|
3599
|
+
const vector = await runVectorPhase(analysedPages.map((p) => p.analysed), {
|
|
3600
|
+
budget: opts.svgBudget ?? DEFAULT_SVG_BUDGET,
|
|
3601
|
+
survey: opts.survey === true,
|
|
3602
|
+
quality: opts.quality,
|
|
3603
|
+
outDir: resolve(opts.outDir),
|
|
3604
|
+
cwd: opts.cwd,
|
|
3605
|
+
cache: run.deps.cache,
|
|
3606
|
+
log: opts.log,
|
|
3607
|
+
}, {
|
|
3608
|
+
// The browser's own bytes when they were kept (the same file the page got); else fetched, as before.
|
|
3609
|
+
fetch: async (url) => run.flatBuffers.get(url) ?? run.fetcher.get(url, { maxBytes: MAX_IMAGE_BYTES, allowPrivate }),
|
|
3610
|
+
});
|
|
3611
|
+
for (const entry of analysedPages)
|
|
3612
|
+
entry.analysed = applyVector(entry.analysed, vector);
|
|
3613
|
+
// The media phase runs once for every page's assets (see runMediaFor); each page is then written again, complete.
|
|
3614
|
+
const media = await runMediaFor(mediaAssets, run, opts, urls);
|
|
3615
|
+
for (const { index, url, page, analysed, fallbacks } of analysedPages) {
|
|
3616
|
+
const withMedia = attachMediaResults(analysed, media, { quality: opts.quality, survey: opts.survey === true });
|
|
3617
|
+
const result = withBroken({ ...withMedia, complete: true, options, scorer: scorerOf(run.gpu, fallbacks) }, opts.expectImages?.(url));
|
|
2738
3618
|
results[index] = result;
|
|
2739
3619
|
finished.push(result);
|
|
2740
3620
|
await writePage(url, result);
|
|
2741
3621
|
await page.finish?.();
|
|
2742
|
-
opts.log(progressLine(finished, urls.length));
|
|
2743
3622
|
}
|
|
2744
3623
|
}
|
|
2745
3624
|
// Every measured page is complete now: the stage's blobs go with it (an interrupted run never gets here).
|
|
@@ -2791,7 +3670,7 @@ export async function observeLight(pageUrl, opts) {
|
|
|
2791
3670
|
}, async (browser, run, robotsFor) => {
|
|
2792
3671
|
const url = new URL(pageUrl);
|
|
2793
3672
|
const allowPrivate = isLocalTarget(pageUrl);
|
|
2794
|
-
const empty = (status, error) => ({ status, error, observations: [], resolved: new Map() });
|
|
3673
|
+
const empty = (status, error) => ({ status, error, observations: [], resolved: new Map(), resolvedVideos: new Map() });
|
|
2795
3674
|
const robots = await checkRobots(pageUrl, allowPrivate, () => robotsFor(url.origin, false), run.ignoreRobots);
|
|
2796
3675
|
if (!robots.allowed)
|
|
2797
3676
|
return empty("robots", "robots.txt disallows the page");
|
|
@@ -2799,7 +3678,7 @@ export async function observeLight(pageUrl, opts) {
|
|
|
2799
3678
|
run.log(robotsIgnoredWarning(pageUrl));
|
|
2800
3679
|
const policy = { resolver: run.resolver, allowPrivate, connectAddress: run.connectAddress };
|
|
2801
3680
|
const guard = { isBlocked: (u) => run.fetcher.hostIsBlocked(u, allowPrivate), policy, thirdParty: run.thirdParty };
|
|
2802
|
-
const outcomes = await settleLimit(run.viewports, VIEWPORT_CONCURRENCY, (viewport) => observeViewport(browser, pageUrl, viewport, guard, run.loadCapMs, undefined, true, (img) => opts.wanted(img, pageUrl)));
|
|
3681
|
+
const outcomes = await settleLimit(run.viewports, VIEWPORT_CONCURRENCY, (viewport) => observeViewport(browser, pageUrl, viewport, guard, run.loadCapMs, undefined, true, (img) => opts.wanted(img, pageUrl), undefined, opts.videos === true));
|
|
2803
3682
|
const observations = [];
|
|
2804
3683
|
for (const [index, outcome] of outcomes.entries()) {
|
|
2805
3684
|
const viewport = run.viewports[index];
|
|
@@ -2826,12 +3705,15 @@ export async function observeLight(pageUrl, opts) {
|
|
|
2826
3705
|
projectRoot: run.projectRoot,
|
|
2827
3706
|
};
|
|
2828
3707
|
const resolved = new Map();
|
|
3708
|
+
const resolvedVideos = new Map();
|
|
2829
3709
|
for (const obs of observations) {
|
|
2830
3710
|
const part = await resolveSources(ctx, [obs], (img) => opts.wanted(img, pageUrl));
|
|
2831
3711
|
for (const [img, source] of part)
|
|
2832
3712
|
resolved.set(img, source);
|
|
3713
|
+
for (const [video, source] of await resolveVideoSources(ctx, [obs]))
|
|
3714
|
+
resolvedVideos.set(video, source);
|
|
2833
3715
|
}
|
|
2834
|
-
return { status: "ok", observations, resolved };
|
|
3716
|
+
return { status: "ok", observations, resolved, resolvedVideos };
|
|
2835
3717
|
});
|
|
2836
3718
|
}
|
|
2837
3719
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
@@ -2869,7 +3751,20 @@ export function renderPage(result, aggressive, hintsFile) {
|
|
|
2869
3751
|
if (bg) {
|
|
2870
3752
|
lines.push(` CSS backgrounds (not in the lines above): ${bg.layers} layer(s), ${kb(bg.servedBytes)}/view served, ${kb(bg.estSaved)}/view est. saving, ${bg.undersized} undersized`);
|
|
2871
3753
|
}
|
|
2872
|
-
|
|
3754
|
+
// The Loaded but not shown group: the images the browser downloaded that no viewport shows in full, by bytes per view.
|
|
3755
|
+
if (result.hidden !== undefined) {
|
|
3756
|
+
lines.push(` ${hiddenSummaryLine(result.hidden) ?? ""}`);
|
|
3757
|
+
const hiddenRecords = (result.images ?? []).filter((r) => r.hidden !== undefined);
|
|
3758
|
+
const rows = hiddenRecords.sort((a, b) => (b.hidden?.perViewBytes ?? 0) - (a.hidden?.perViewBytes ?? 0)).slice(0, HIDDEN_ROWS);
|
|
3759
|
+
for (const record of rows) {
|
|
3760
|
+
const hidden = record.hidden;
|
|
3761
|
+
const where = record.locator.file !== undefined ? `${record.locator.file}:${record.locator.line ?? 0}` : record.locator.selector;
|
|
3762
|
+
lines.push(` ${JSON.stringify(safeText(where))} hidden at ${hidden.viewports.length} of ${hidden.of} ${kb(hidden.perViewBytes)}/view ${hidden.action}`);
|
|
3763
|
+
}
|
|
3764
|
+
if (hiddenRecords.length > rows.length)
|
|
3765
|
+
lines.push(` (+${hiddenRecords.length - rows.length} more: see imgtune hints)`);
|
|
3766
|
+
}
|
|
3767
|
+
const hinted = (result.images ?? []).filter((r) => hasFix(r) || r.hidden !== undefined).length;
|
|
2873
3768
|
if (hinted > 0 && hintsFile !== undefined)
|
|
2874
3769
|
lines.push(` ${hinted} images with fix hints: run \`imgtune hints ${safeText(hintsFile)}\``);
|
|
2875
3770
|
if (result.groups.length > 0) {
|
|
@@ -2880,11 +3775,28 @@ export function renderPage(result, aggressive, hintsFile) {
|
|
|
2880
3775
|
const loader = g.loader ? ` loader: ${safeText(g.loader)}` : "";
|
|
2881
3776
|
const sizes = g.sizes === undefined ? "no sizes" : safeText(g.sizes);
|
|
2882
3777
|
const suggested = g.suggestedSizes === undefined ? "-" : safeText(g.suggestedSizes);
|
|
2883
|
-
|
|
3778
|
+
const head = g.srcset !== undefined ? srcsetGroupText(g.srcset.widths) : `${sizes} -> ${suggested}`;
|
|
3779
|
+
lines.push(` ${head} saves ${kb(g.savedBytes)}/view${loader}${hint}${qualityHint}`);
|
|
2884
3780
|
}
|
|
2885
3781
|
}
|
|
3782
|
+
// The Video section: the page's <video> files, per view and per play, with each one's encode or the reason it has none.
|
|
3783
|
+
const videos = result.videos ?? [];
|
|
3784
|
+
const totals = videoTotalsLine(videos);
|
|
3785
|
+
if (totals !== undefined) {
|
|
3786
|
+
lines.push(` ${totals}`);
|
|
3787
|
+
for (const video of videos)
|
|
3788
|
+
lines.push(` ${videoRowOf(video, (text) => JSON.stringify(safeText(text)))}`);
|
|
3789
|
+
}
|
|
3790
|
+
// The Resources section: the page's non-image findings, ranked by saving (see resources/).
|
|
3791
|
+
lines.push(...resourceLinesOf(result.resources ?? [], (text) => safeText(text)));
|
|
2886
3792
|
return lines;
|
|
2887
3793
|
}
|
|
3794
|
+
/** The most loaded-but-not-shown images the page's text output lists (the rest are in the hints). */
|
|
3795
|
+
const HIDDEN_ROWS = 10;
|
|
3796
|
+
/** The top-groups line of a plain image group with no srcset: what to serve, in place of a sizes change. */
|
|
3797
|
+
export function srcsetGroupText(widths) {
|
|
3798
|
+
return `no srcset -> serve ${srcsetWidthLabel(widths)} + srcset/sizes`;
|
|
3799
|
+
}
|
|
2888
3800
|
/** Per-site line for hosts with two or more scanned pages. */
|
|
2889
3801
|
export function renderScan(results, aggressive, hintsFile, maxQualities) {
|
|
2890
3802
|
const lines = [];
|
|
@@ -2894,6 +3806,13 @@ export function renderScan(results, aggressive, hintsFile, maxQualities) {
|
|
|
2894
3806
|
const note = scorerNote(results);
|
|
2895
3807
|
if (note !== undefined)
|
|
2896
3808
|
lines.push(note, "");
|
|
3809
|
+
// The media phase's note (no ffmpeg, a failed encode run) is printed once, at the top, not once per page.
|
|
3810
|
+
const media = results.find((r) => r.mediaNote !== undefined)?.mediaNote;
|
|
3811
|
+
if (media !== undefined)
|
|
3812
|
+
lines.push(safeText(media), "");
|
|
3813
|
+
const vectorNote = results.find((r) => r.vectorNote !== undefined)?.vectorNote;
|
|
3814
|
+
if (vectorNote !== undefined)
|
|
3815
|
+
lines.push(safeText(vectorNote), "");
|
|
2897
3816
|
for (const result of results)
|
|
2898
3817
|
lines.push(...renderPage(result, aggressive, hintsFile?.(result)), "");
|
|
2899
3818
|
const byHost = new Map();
|
|
@@ -2974,6 +3893,12 @@ function markdownBody(results, aggressive, withHints, maxQualities) {
|
|
|
2974
3893
|
const note = scorerNote(results);
|
|
2975
3894
|
if (note !== undefined)
|
|
2976
3895
|
lines.push(note, "");
|
|
3896
|
+
const media = results.find((r) => r.mediaNote !== undefined)?.mediaNote;
|
|
3897
|
+
if (media !== undefined)
|
|
3898
|
+
lines.push(mdText(media), "");
|
|
3899
|
+
const vectorNote = results.find((r) => r.vectorNote !== undefined)?.vectorNote;
|
|
3900
|
+
if (vectorNote !== undefined)
|
|
3901
|
+
lines.push(mdText(vectorNote), "");
|
|
2977
3902
|
lines.push(mdRow(head, width), mdRow(align, width));
|
|
2978
3903
|
for (const result of results) {
|
|
2979
3904
|
const label = pageLabel(result.url);
|
|
@@ -2993,26 +3918,49 @@ function markdownBody(results, aggressive, withHints, maxQualities) {
|
|
|
2993
3918
|
continue;
|
|
2994
3919
|
// The LCP line goes above the page's table: the image that the first paint waits for comes first.
|
|
2995
3920
|
const lcp = lcpLinesOf(result, mdCode);
|
|
2996
|
-
|
|
3921
|
+
const videos = result.videos ?? [];
|
|
3922
|
+
if (lcp.length === 0 && result.groups.length === 0 && videos.length === 0)
|
|
2997
3923
|
continue;
|
|
2998
3924
|
lines.push("", `### ${pageLabel(result.url)}`, "");
|
|
2999
3925
|
for (const line of lcp)
|
|
3000
3926
|
lines.push(line, "");
|
|
3001
|
-
if (result.groups.length
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3927
|
+
if (result.groups.length > 0) {
|
|
3928
|
+
lines.push(mdRow(["current sizes", "suggested", "saves KB/view", "loader", "hint"], 5));
|
|
3929
|
+
lines.push(mdRow(["---", "---", "---:", "---", "---"], 5));
|
|
3930
|
+
for (const g of result.groups.slice(0, SUMMARY_GROUPS)) {
|
|
3931
|
+
const current = g.srcset !== undefined ? "no srcset" : g.sizes === undefined ? "no sizes" : mdCode(g.sizes);
|
|
3932
|
+
const suggested = g.srcset !== undefined ? mdText(`serve ${srcsetWidthLabel(g.srcset.widths)} + srcset/sizes`) : g.suggestedSizes === undefined ? "-" : mdCode(g.suggestedSizes);
|
|
3933
|
+
const loader = g.loader ? mdCode(g.loader) : "-";
|
|
3934
|
+
const hint = [g.hint, g.qualityHint].filter((h) => h !== undefined).map((h) => mdCode(h)).join("; ");
|
|
3935
|
+
lines.push(mdRow([current, suggested, `${kbNumber(g.savedBytes)}${g.estApprox === true ? APPROX_MARK : ""}`, loader, hint || "-"], 5));
|
|
3936
|
+
}
|
|
3937
|
+
}
|
|
3938
|
+
const totals = videoTotalsLine(videos);
|
|
3939
|
+
if (totals !== undefined) {
|
|
3940
|
+
lines.push("", totals, "");
|
|
3941
|
+
for (const video of videos)
|
|
3942
|
+
lines.push(`- ${mdCode(videoRowOf(video, (text) => JSON.stringify(safeText(text))))}`);
|
|
3943
|
+
}
|
|
3944
|
+
}
|
|
3945
|
+
// Resources (see resources/): the totals, then the largest findings. They are not in the image totals above.
|
|
3946
|
+
const resourceRows = results.filter((r) => r.status === "ok").flatMap((r) => r.resources ?? []);
|
|
3947
|
+
const resourceTotals = resourceTotalsLine(resourceRows);
|
|
3948
|
+
if (resourceTotals !== undefined) {
|
|
3949
|
+
lines.push("", mdText(resourceTotals));
|
|
3950
|
+
const largest = [...resourceRows].sort((a, b) => b.savedBytes - a.savedBytes).slice(0, RESOURCE_MARKDOWN_ROWS);
|
|
3951
|
+
for (const finding of largest) {
|
|
3952
|
+
const saved = finding.savedBytes > 0 ? ` saves ${kbNumber(finding.savedBytes)} KB${finding.estimate ? APPROX_MARK : ""}` : "";
|
|
3953
|
+
lines.push(`- ${mdCode(`${finding.kind}: ${safeText(finding.title)}`)}${saved}`);
|
|
3011
3954
|
}
|
|
3012
3955
|
}
|
|
3013
3956
|
const plan = planQualityCap(results, maxQualities);
|
|
3014
3957
|
if (plan !== undefined)
|
|
3015
3958
|
lines.push("", mdCode(qualitiesLine(plan)));
|
|
3959
|
+
const hiddenLine = hiddenPagesLine(results);
|
|
3960
|
+
const hiddenVideoLine = hiddenVideoPagesLine(results);
|
|
3961
|
+
const hiddenRows = [hiddenLine, hiddenVideoLine].flatMap((line) => (line !== undefined ? [mdCode(line)] : []));
|
|
3962
|
+
if (hiddenRows.length > 0)
|
|
3963
|
+
lines.push("", ...hiddenRows);
|
|
3016
3964
|
if (withHints) {
|
|
3017
3965
|
const hints = commentHints(results, undefined, maxQualities);
|
|
3018
3966
|
if (hints.lines.length > 0) {
|
|
@@ -3093,7 +4041,7 @@ export async function rootOf(root, cwd) {
|
|
|
3093
4041
|
return absolute;
|
|
3094
4042
|
}
|
|
3095
4043
|
/** `imgtune scan`: the arguments after `scan`. The JSON files are written to --out; --markdown and --summary-json too. */
|
|
3096
|
-
export async function scanCommand(argv, cwd, log) {
|
|
4044
|
+
export async function scanCommand(argv, cwd, log, signal) {
|
|
3097
4045
|
const args = parseScanArgs(argv);
|
|
3098
4046
|
const plan = scanPlan(args);
|
|
3099
4047
|
const viewports = DEFAULT_VIEWPORTS.filter((v) => !plan.viewports || plan.viewports.includes(v.name)).map((v) => ({ ...v }));
|
|
@@ -3120,6 +4068,7 @@ export async function scanCommand(argv, cwd, log) {
|
|
|
3120
4068
|
outDir: resolve(cwd, args.out),
|
|
3121
4069
|
cwd,
|
|
3122
4070
|
root,
|
|
4071
|
+
signal,
|
|
3123
4072
|
aggressive: args.aggressive,
|
|
3124
4073
|
quality: plan.quality,
|
|
3125
4074
|
viewports,
|
|
@@ -3130,12 +4079,21 @@ export async function scanCommand(argv, cwd, log) {
|
|
|
3130
4079
|
resume: args.resume === true,
|
|
3131
4080
|
maxPages: args.maxPages,
|
|
3132
4081
|
maxImages: args.maxImages,
|
|
4082
|
+
mediaBudget: args.mediaBudget,
|
|
4083
|
+
svgBudget: args.svgBudget,
|
|
4084
|
+
resourceTime: args.resourceTime,
|
|
4085
|
+
mediaTime: args.mediaTime,
|
|
4086
|
+
videoBar: args.videoBar,
|
|
4087
|
+
videoStillBar: args.videoStillBar,
|
|
4088
|
+
maxVideoWidths: args.maxVideoWidths,
|
|
3133
4089
|
pageConcurrency: plan.pageConcurrency,
|
|
3134
4090
|
idleTimeoutMs: plan.idleTimeoutMs,
|
|
3135
4091
|
sharedLoads: args.sharedLoads === true,
|
|
3136
4092
|
storageState,
|
|
3137
4093
|
gpu: args.gpu,
|
|
4094
|
+
gpuEncode: args.gpuEncode,
|
|
3138
4095
|
cache: args.noCache !== true,
|
|
4096
|
+
resources: args.resources !== false,
|
|
3139
4097
|
report,
|
|
3140
4098
|
thirdParty: args.thirdParty,
|
|
3141
4099
|
});
|