@percy/core 1.32.0-beta.4 → 1.32.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +8 -2
- package/dist/discovery.js +48 -8
- package/dist/network.js +49 -10
- package/package.json +9 -9
package/dist/browser.js
CHANGED
|
@@ -8,6 +8,7 @@ import WebSocket from 'ws';
|
|
|
8
8
|
import rimraf from 'rimraf';
|
|
9
9
|
import logger from '@percy/logger';
|
|
10
10
|
import install from './install.js';
|
|
11
|
+
import { MAX_CDP_PAYLOAD } from './network.js';
|
|
11
12
|
import Session from './session.js';
|
|
12
13
|
import Page from './page.js';
|
|
13
14
|
|
|
@@ -118,9 +119,14 @@ export class Browser extends EventEmitter {
|
|
|
118
119
|
for (let a of args) if (!this.args.includes(a)) this.args.push(a);
|
|
119
120
|
this.args.push(`--user-data-dir=${this.profile}`);
|
|
120
121
|
|
|
121
|
-
// spawn the browser process and connect a websocket to the devtools address
|
|
122
|
+
// spawn the browser process and connect a websocket to the devtools address.
|
|
123
|
+
// Bump maxPayload above the ws default (100 MB) — Network.getResponseBody
|
|
124
|
+
// returns binary bodies as base64 (+33%), so a 50 MB raw body becomes ~67 MB
|
|
125
|
+
// on the wire; framing pushes it close to the default. MAX_CDP_PAYLOAD leaves
|
|
126
|
+
// headroom for the PERCY_GZIP raw ceiling (see MAX_RAW_RESOURCE_SIZE_WITH_GZIP).
|
|
122
127
|
this.ws = new WebSocket(await this.spawn(timeout), {
|
|
123
|
-
perMessageDeflate: false
|
|
128
|
+
perMessageDeflate: false,
|
|
129
|
+
maxPayload: MAX_CDP_PAYLOAD
|
|
124
130
|
});
|
|
125
131
|
|
|
126
132
|
// wait until the websocket has connected
|
package/dist/discovery.js
CHANGED
|
@@ -5,6 +5,7 @@ import { normalizeURL, hostnameMatches, createResource, createRootResource, crea
|
|
|
5
5
|
import { ByteLRU, entrySize, DiskSpillStore, createSpillDir } from './cache/byte-lru.js';
|
|
6
6
|
import { sha256hash } from '@percy/client/utils';
|
|
7
7
|
import Pako from 'pako';
|
|
8
|
+
import { MAX_RESOURCE_SIZE, logAssetInstrumentation } from './network.js';
|
|
8
9
|
|
|
9
10
|
// Logs verbose debug logs detailing various snapshot options.
|
|
10
11
|
function debugSnapshotOptions(snapshot) {
|
|
@@ -227,14 +228,50 @@ function processSnapshotResources({
|
|
|
227
228
|
resources.push(createLogResource(logger.snapshotLogs(snapshot.meta.snapshot)));
|
|
228
229
|
logger.evictSnapshot(snapshot.meta.snapshot);
|
|
229
230
|
if (process.env.PERCY_GZIP) {
|
|
231
|
+
const kept = [];
|
|
230
232
|
for (let index = 0; index < resources.length; index++) {
|
|
231
|
-
const
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
233
|
+
const resource = resources[index];
|
|
234
|
+
try {
|
|
235
|
+
const alreadyZipped = isGzipped(resource.content);
|
|
236
|
+
/* istanbul ignore next: very hard to mock true */
|
|
237
|
+
if (!alreadyZipped) {
|
|
238
|
+
resource.content = Pako.gzip(resource.content);
|
|
239
|
+
resource.sha = sha256hash(resource.content);
|
|
240
|
+
log.debug(`- Gzipped resource: ${resource.url}`);
|
|
241
|
+
}
|
|
242
|
+
} catch (error) {
|
|
243
|
+
var _snapshot$meta;
|
|
244
|
+
// One bad resource must not fail the whole snapshot.
|
|
245
|
+
logAssetInstrumentation(log, 'asset_load_missing', 'network_error', {
|
|
246
|
+
url: resource.url,
|
|
247
|
+
snapshot: (_snapshot$meta = snapshot.meta) === null || _snapshot$meta === void 0 ? void 0 : _snapshot$meta.snapshot,
|
|
248
|
+
error: error.message
|
|
249
|
+
});
|
|
250
|
+
log.warn(`- Skipping resource: gzip failed for ${resource.url} - ${error.message}`);
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// resource.content is guaranteed by the try block above (either just
|
|
255
|
+
// assigned from Pako.gzip, or alreadyZipped was true).
|
|
256
|
+
const size = resource.content.length;
|
|
257
|
+
// Root (DOM HTML) and log resources are required for a valid snapshot;
|
|
258
|
+
// shipping an oversized one and letting the API surface a clear error
|
|
259
|
+
// is better than silently dropping it here.
|
|
260
|
+
if (size > MAX_RESOURCE_SIZE && !resource.root && !resource.log) {
|
|
261
|
+
var _snapshot$meta2;
|
|
262
|
+
logAssetInstrumentation(log, 'asset_not_uploaded', 'resource_too_large', {
|
|
263
|
+
url: resource.url,
|
|
264
|
+
size,
|
|
265
|
+
snapshot: (_snapshot$meta2 = snapshot.meta) === null || _snapshot$meta2 === void 0 ? void 0 : _snapshot$meta2.snapshot
|
|
266
|
+
});
|
|
267
|
+
log.debug(`- Skipping resource larger than allowed size after gzip (${size} bytes)`, {
|
|
268
|
+
url: resource.url
|
|
269
|
+
});
|
|
270
|
+
continue;
|
|
236
271
|
}
|
|
272
|
+
kept.push(resource);
|
|
237
273
|
}
|
|
274
|
+
resources = kept;
|
|
238
275
|
}
|
|
239
276
|
return {
|
|
240
277
|
...snapshot,
|
|
@@ -518,12 +555,15 @@ export function createDiscoveryQueue(percy) {
|
|
|
518
555
|
}).handle('start', async () => {
|
|
519
556
|
const configuredMaxCacheRamMB = percy.config.discovery.maxCacheRam;
|
|
520
557
|
let effectiveMaxCacheRamMB = configuredMaxCacheRamMB;
|
|
558
|
+
// Cache stores raw bodies; under PERCY_GZIP individual resources can
|
|
559
|
+
// exceed the default 25MB floor, so the floor must track the active cap.
|
|
560
|
+
const minCacheRamMB = process.env.PERCY_GZIP ? 50 : MAX_RESOURCE_SIZE_MB;
|
|
521
561
|
|
|
522
562
|
// User's config is not mutated; the post-clamp value lives on stats.
|
|
523
563
|
if (configuredMaxCacheRamMB != null) {
|
|
524
|
-
if (configuredMaxCacheRamMB <
|
|
525
|
-
percy.log.warn(`--max-cache-ram=${configuredMaxCacheRamMB}MB is below the ${
|
|
526
|
-
effectiveMaxCacheRamMB =
|
|
564
|
+
if (configuredMaxCacheRamMB < minCacheRamMB) {
|
|
565
|
+
percy.log.warn(`--max-cache-ram=${configuredMaxCacheRamMB}MB is below the ${minCacheRamMB}MB minimum ` + `(individual resources up to ${minCacheRamMB}MB would otherwise be dropped). ` + `Continuing with the minimum: ${minCacheRamMB}MB.`);
|
|
566
|
+
effectiveMaxCacheRamMB = minCacheRamMB;
|
|
527
567
|
} else if (configuredMaxCacheRamMB < MIN_REASONABLE_CAP_MB) {
|
|
528
568
|
percy.log.warn(`--max-cache-ram=${configuredMaxCacheRamMB}MB is very small; ` + 'most resources will not fit and hit rate will be near zero.');
|
|
529
569
|
}
|
package/dist/network.js
CHANGED
|
@@ -2,7 +2,23 @@ import { request as makeRequest } from '@percy/client/utils';
|
|
|
2
2
|
import logger from '@percy/logger';
|
|
3
3
|
import mime from 'mime-types';
|
|
4
4
|
import { AbortError, DefaultMap, createResource, hostnameMatches, normalizeURL, waitFor, decodeAndEncodeURLWithLogging, handleIncorrectFontMimeType, executeDomainValidation } from './utils.js';
|
|
5
|
-
const MAX_RESOURCE_SIZE = 25 * 1024 ** 2 * 0.63; // 25MB, 0.63 factor for accounting for base64 encoding
|
|
5
|
+
export const MAX_RESOURCE_SIZE = 25 * 1024 ** 2 * 0.63; // 25MB, 0.63 factor for accounting for base64 encoding
|
|
6
|
+
// CDP returns binary bodies via Network.getResponseBody as base64 in the JSON-RPC
|
|
7
|
+
// response. Base64 inflates by 4/3, so an N-byte body becomes ~1.33×N on the wire.
|
|
8
|
+
// The ws library defaults maxPayload to 100 MB; browser.js raises it to this value
|
|
9
|
+
// so base64(body) + framing stays well under the limit (see browser.js connect()).
|
|
10
|
+
export const MAX_CDP_PAYLOAD = 150 * 1024 ** 2;
|
|
11
|
+
// 50 MB is a conservative starting point for the PERCY_GZIP raw ceiling
|
|
12
|
+
// (50 × 4/3 ≈ 67 MB on the wire); the final value is pending a storage-cost
|
|
13
|
+
// review (PER-8648).
|
|
14
|
+
export const MAX_RAW_RESOURCE_SIZE_WITH_GZIP = 50 * 1024 ** 2;
|
|
15
|
+
// Direct-fetch fallback needs more time when bodies can be tens of MB.
|
|
16
|
+
const DIRECT_FETCH_TIMEOUT_WITH_GZIP = 30000;
|
|
17
|
+
export function shouldSkipForSize(size) {
|
|
18
|
+
if (!Number.isFinite(size)) return false;
|
|
19
|
+
if (process.env.PERCY_GZIP) return size > MAX_RAW_RESOURCE_SIZE_WITH_GZIP;
|
|
20
|
+
return size > MAX_RESOURCE_SIZE;
|
|
21
|
+
}
|
|
6
22
|
const ALLOWED_STATUSES = [200, 201, 301, 302, 304, 307, 308];
|
|
7
23
|
const ALLOWED_RESOURCES = ['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Other'];
|
|
8
24
|
const ABORTED_MESSAGE = 'Request was aborted by browser';
|
|
@@ -309,7 +325,7 @@ export class Network {
|
|
|
309
325
|
size: rawValue,
|
|
310
326
|
snapshot: meta.snapshot
|
|
311
327
|
});
|
|
312
|
-
this.log.debug('- Skipping resource larger than
|
|
328
|
+
this.log.debug('- Skipping resource larger than allowed size', meta);
|
|
313
329
|
|
|
314
330
|
// Disposition first, then forget the request — so we never leave Chrome's
|
|
315
331
|
// Fetch state paused while Percy thinks the request is already done.
|
|
@@ -582,7 +598,7 @@ export class Network {
|
|
|
582
598
|
}
|
|
583
599
|
|
|
584
600
|
// Logs asset instrumentation for failed/skipped asset loading
|
|
585
|
-
function logAssetInstrumentation(log, category, reason, details) {
|
|
601
|
+
export function logAssetInstrumentation(log, category, reason, details) {
|
|
586
602
|
const categoryMap = {
|
|
587
603
|
asset_load_5xx: '[ASSET_LOAD_5XX]',
|
|
588
604
|
asset_not_uploaded: '[ASSET_NOT_UPLOADED]',
|
|
@@ -629,7 +645,7 @@ function inspectContentLength(headers) {
|
|
|
629
645
|
let key = headers && Object.keys(headers).find(k => k.toLowerCase() === 'content-length');
|
|
630
646
|
let rawValue = key ? headers[key] : undefined;
|
|
631
647
|
let parsed = parseInt(rawValue, 10);
|
|
632
|
-
let tooLarge =
|
|
648
|
+
let tooLarge = shouldSkipForSize(parsed);
|
|
633
649
|
let malformed = rawValue !== undefined && rawValue !== null && String(rawValue).length > 0 && !Number.isFinite(parsed);
|
|
634
650
|
return {
|
|
635
651
|
tooLarge,
|
|
@@ -854,20 +870,23 @@ async function captureResourceDirectly(network, request, session) {
|
|
|
854
870
|
...network.meta,
|
|
855
871
|
url
|
|
856
872
|
};
|
|
873
|
+
|
|
874
|
+
// Under PERCY_GZIP the size ceiling can reach tens of MB; 5s is too short.
|
|
875
|
+
let timeoutMs = process.env.PERCY_GZIP ? DIRECT_FETCH_TIMEOUT_WITH_GZIP : DIRECT_FETCH_TIMEOUT;
|
|
857
876
|
try {
|
|
858
877
|
log.debug('- Requesting resource directly (responseReceived timeout fallback)', meta);
|
|
859
878
|
let {
|
|
860
879
|
body,
|
|
861
880
|
status,
|
|
862
881
|
headers: responseHeaders
|
|
863
|
-
} = await raceWithTimeout(makeDirectRequest(network, request, session),
|
|
864
|
-
if (body.length
|
|
882
|
+
} = await raceWithTimeout(makeDirectRequest(network, request, session), timeoutMs, `Direct fetch timed out after ${timeoutMs}ms`);
|
|
883
|
+
if (shouldSkipForSize(body.length)) {
|
|
865
884
|
logAssetInstrumentation(log, 'asset_not_uploaded', 'resource_too_large', {
|
|
866
885
|
url,
|
|
867
886
|
size: body.length,
|
|
868
887
|
snapshot: meta.snapshot
|
|
869
888
|
});
|
|
870
|
-
log.debug('- Skipping resource larger than
|
|
889
|
+
log.debug('- Skipping resource larger than allowed size', meta);
|
|
871
890
|
return;
|
|
872
891
|
}
|
|
873
892
|
let urlObj = new URL(url);
|
|
@@ -881,6 +900,12 @@ async function captureResourceDirectly(network, request, session) {
|
|
|
881
900
|
log.debug(`- Saving direct-fetched resource sha=${resource.sha} mimetype=${mimeType}`, meta);
|
|
882
901
|
network.intercept.saveResource(resource);
|
|
883
902
|
} catch (error) {
|
|
903
|
+
logAssetInstrumentation(log, 'asset_load_missing', 'network_error', {
|
|
904
|
+
url,
|
|
905
|
+
snapshot: meta.snapshot,
|
|
906
|
+
requestType: request.type,
|
|
907
|
+
error: error.message
|
|
908
|
+
});
|
|
884
909
|
log.debug(`Direct fetch failed for ${url} - ${error.message}`, meta);
|
|
885
910
|
}
|
|
886
911
|
}
|
|
@@ -923,7 +948,12 @@ async function saveResponseResource(network, request, session) {
|
|
|
923
948
|
log.debug(`- Capturing auto-validated domain: ${hostname}`, meta);
|
|
924
949
|
}
|
|
925
950
|
}
|
|
926
|
-
|
|
951
|
+
|
|
952
|
+
// Bound how long we'll wait for a single body — large bodies over a slow
|
|
953
|
+
// or dead CDP socket must not stall the discovery queue forever. Errors
|
|
954
|
+
// (timeout or underlying CDP failure) propagate to the outer try/catch
|
|
955
|
+
// which logs the failure via the existing "Encountered an error" path.
|
|
956
|
+
let body = shouldCapture && (await raceWithTimeout(response.buffer(), process.env.PERCY_GZIP ? 60000 : 30000, `response.buffer() timed out for ${url}`));
|
|
927
957
|
|
|
928
958
|
// Don't rename the below log line as it is used in getting network logs in api
|
|
929
959
|
/* istanbul ignore if: first check is a sanity check */
|
|
@@ -947,14 +977,14 @@ async function saveResponseResource(network, request, session) {
|
|
|
947
977
|
snapshot: meta.snapshot
|
|
948
978
|
});
|
|
949
979
|
return log.debug('- Skipping empty response', meta);
|
|
950
|
-
} else if (body.length
|
|
980
|
+
} else if (shouldSkipForSize(body.length)) {
|
|
951
981
|
logAssetInstrumentation(log, 'asset_not_uploaded', 'resource_too_large', {
|
|
952
982
|
url,
|
|
953
983
|
size: body.length,
|
|
954
984
|
snapshot: meta.snapshot
|
|
955
985
|
});
|
|
956
986
|
log.debug('- Missing headers for the requested resource.', meta);
|
|
957
|
-
return log.debug('- Skipping resource larger than
|
|
987
|
+
return log.debug('- Skipping resource larger than allowed size', meta);
|
|
958
988
|
} else if (!ALLOWED_STATUSES.includes(response.status)) {
|
|
959
989
|
/* istanbul ignore next: ternary branches tested separately */
|
|
960
990
|
const category = response.status >= 500 && response.status < 600 ? 'asset_load_5xx' : 'asset_not_uploaded';
|
|
@@ -1012,6 +1042,15 @@ async function saveResponseResource(network, request, session) {
|
|
|
1012
1042
|
// Don't rename the below log line as it is used in getting network logs in api
|
|
1013
1043
|
log.debug(`Encountered an error processing resource: ${url}`, meta);
|
|
1014
1044
|
log.debug(error, meta);
|
|
1045
|
+
// Surface to the structured asset instrumentation channel so support
|
|
1046
|
+
// tooling (analyse_build, analyze_cli_logs_tool) sees the failure
|
|
1047
|
+
// instead of just a debug-level line.
|
|
1048
|
+
logAssetInstrumentation(log, 'asset_load_missing', 'network_error', {
|
|
1049
|
+
url,
|
|
1050
|
+
snapshot: meta.snapshot,
|
|
1051
|
+
requestType: request === null || request === void 0 ? void 0 : request.type,
|
|
1052
|
+
error: error.message
|
|
1053
|
+
});
|
|
1015
1054
|
}
|
|
1016
1055
|
}
|
|
1017
1056
|
if (resource && !resource.root) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@percy/core",
|
|
3
|
-
"version": "1.32.0-beta.
|
|
3
|
+
"version": "1.32.0-beta.5",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -46,12 +46,12 @@
|
|
|
46
46
|
"dependencies": {
|
|
47
47
|
"@grpc/grpc-js": "^1.14.3",
|
|
48
48
|
"@grpc/proto-loader": "^0.8.0",
|
|
49
|
-
"@percy/client": "1.32.0-beta.
|
|
50
|
-
"@percy/config": "1.32.0-beta.
|
|
51
|
-
"@percy/dom": "1.32.0-beta.
|
|
52
|
-
"@percy/logger": "1.32.0-beta.
|
|
53
|
-
"@percy/monitoring": "1.32.0-beta.
|
|
54
|
-
"@percy/webdriver-utils": "1.32.0-beta.
|
|
49
|
+
"@percy/client": "1.32.0-beta.5",
|
|
50
|
+
"@percy/config": "1.32.0-beta.5",
|
|
51
|
+
"@percy/dom": "1.32.0-beta.5",
|
|
52
|
+
"@percy/logger": "1.32.0-beta.5",
|
|
53
|
+
"@percy/monitoring": "1.32.0-beta.5",
|
|
54
|
+
"@percy/webdriver-utils": "1.32.0-beta.5",
|
|
55
55
|
"busboy": "^1.6.0",
|
|
56
56
|
"content-disposition": "^0.5.4",
|
|
57
57
|
"cross-spawn": "^7.0.3",
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"yaml": "^2.4.1"
|
|
68
68
|
},
|
|
69
69
|
"optionalDependencies": {
|
|
70
|
-
"@percy/cli-doctor": "1.32.0-beta.
|
|
70
|
+
"@percy/cli-doctor": "1.32.0-beta.5"
|
|
71
71
|
},
|
|
72
|
-
"gitHead": "
|
|
72
|
+
"gitHead": "d93415f2cc83533c9c037579e440c6e2a5119920"
|
|
73
73
|
}
|