@percy/core 1.32.0-beta.4 → 1.32.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/browser.js CHANGED
@@ -8,6 +8,7 @@ import WebSocket from 'ws';
8
8
  import rimraf from 'rimraf';
9
9
  import logger from '@percy/logger';
10
10
  import install from './install.js';
11
+ import { MAX_CDP_PAYLOAD } from './network.js';
11
12
  import Session from './session.js';
12
13
  import Page from './page.js';
13
14
 
@@ -118,9 +119,14 @@ export class Browser extends EventEmitter {
118
119
  for (let a of args) if (!this.args.includes(a)) this.args.push(a);
119
120
  this.args.push(`--user-data-dir=${this.profile}`);
120
121
 
121
- // spawn the browser process and connect a websocket to the devtools address
122
+ // spawn the browser process and connect a websocket to the devtools address.
123
+ // Bump maxPayload above the ws default (100 MB) — Network.getResponseBody
124
+ // returns binary bodies as base64 (+33%), so a 50 MB raw body becomes ~67 MB
125
+ // on the wire; framing pushes it close to the default. MAX_CDP_PAYLOAD leaves
126
+ // headroom for the PERCY_GZIP raw ceiling (see MAX_RAW_RESOURCE_SIZE_WITH_GZIP).
122
127
  this.ws = new WebSocket(await this.spawn(timeout), {
123
- perMessageDeflate: false
128
+ perMessageDeflate: false,
129
+ maxPayload: MAX_CDP_PAYLOAD
124
130
  });
125
131
 
126
132
  // wait until the websocket has connected
package/dist/discovery.js CHANGED
@@ -5,6 +5,7 @@ import { normalizeURL, hostnameMatches, createResource, createRootResource, crea
5
5
  import { ByteLRU, entrySize, DiskSpillStore, createSpillDir } from './cache/byte-lru.js';
6
6
  import { sha256hash } from '@percy/client/utils';
7
7
  import Pako from 'pako';
8
+ import { MAX_RESOURCE_SIZE, logAssetInstrumentation } from './network.js';
8
9
 
9
10
  // Logs verbose debug logs detailing various snapshot options.
10
11
  function debugSnapshotOptions(snapshot) {
@@ -227,14 +228,50 @@ function processSnapshotResources({
227
228
  resources.push(createLogResource(logger.snapshotLogs(snapshot.meta.snapshot)));
228
229
  logger.evictSnapshot(snapshot.meta.snapshot);
229
230
  if (process.env.PERCY_GZIP) {
231
+ const kept = [];
230
232
  for (let index = 0; index < resources.length; index++) {
231
- const alreadyZipped = isGzipped(resources[index].content);
232
- /* istanbul ignore next: very hard to mock true */
233
- if (!alreadyZipped) {
234
- resources[index].content = Pako.gzip(resources[index].content);
235
- resources[index].sha = sha256hash(resources[index].content);
233
+ const resource = resources[index];
234
+ try {
235
+ const alreadyZipped = isGzipped(resource.content);
236
+ /* istanbul ignore next: very hard to mock true */
237
+ if (!alreadyZipped) {
238
+ resource.content = Pako.gzip(resource.content);
239
+ resource.sha = sha256hash(resource.content);
240
+ log.debug(`- Gzipped resource: ${resource.url}`);
241
+ }
242
+ } catch (error) {
243
+ var _snapshot$meta;
244
+ // One bad resource must not fail the whole snapshot.
245
+ logAssetInstrumentation(log, 'asset_load_missing', 'network_error', {
246
+ url: resource.url,
247
+ snapshot: (_snapshot$meta = snapshot.meta) === null || _snapshot$meta === void 0 ? void 0 : _snapshot$meta.snapshot,
248
+ error: error.message
249
+ });
250
+ log.warn(`- Skipping resource: gzip failed for ${resource.url} - ${error.message}`);
251
+ continue;
252
+ }
253
+
254
+ // resource.content is guaranteed by the try block above (either just
255
+ // assigned from Pako.gzip, or alreadyZipped was true).
256
+ const size = resource.content.length;
257
+ // Root (DOM HTML) and log resources are required for a valid snapshot;
258
+ // shipping an oversized one and letting the API surface a clear error
259
+ // is better than silently dropping it here.
260
+ if (size > MAX_RESOURCE_SIZE && !resource.root && !resource.log) {
261
+ var _snapshot$meta2;
262
+ logAssetInstrumentation(log, 'asset_not_uploaded', 'resource_too_large', {
263
+ url: resource.url,
264
+ size,
265
+ snapshot: (_snapshot$meta2 = snapshot.meta) === null || _snapshot$meta2 === void 0 ? void 0 : _snapshot$meta2.snapshot
266
+ });
267
+ log.debug(`- Skipping resource larger than allowed size after gzip (${size} bytes)`, {
268
+ url: resource.url
269
+ });
270
+ continue;
236
271
  }
272
+ kept.push(resource);
237
273
  }
274
+ resources = kept;
238
275
  }
239
276
  return {
240
277
  ...snapshot,
@@ -518,12 +555,15 @@ export function createDiscoveryQueue(percy) {
518
555
  }).handle('start', async () => {
519
556
  const configuredMaxCacheRamMB = percy.config.discovery.maxCacheRam;
520
557
  let effectiveMaxCacheRamMB = configuredMaxCacheRamMB;
558
+ // Cache stores raw bodies; under PERCY_GZIP individual resources can
559
+ // exceed the default 25MB floor, so the floor must track the active cap.
560
+ const minCacheRamMB = process.env.PERCY_GZIP ? 50 : MAX_RESOURCE_SIZE_MB;
521
561
 
522
562
  // User's config is not mutated; the post-clamp value lives on stats.
523
563
  if (configuredMaxCacheRamMB != null) {
524
- if (configuredMaxCacheRamMB < MAX_RESOURCE_SIZE_MB) {
525
- percy.log.warn(`--max-cache-ram=${configuredMaxCacheRamMB}MB is below the ${MAX_RESOURCE_SIZE_MB}MB minimum ` + '(individual resources up to 25MB would otherwise be dropped). ' + `Continuing with the minimum: ${MAX_RESOURCE_SIZE_MB}MB.`);
526
- effectiveMaxCacheRamMB = MAX_RESOURCE_SIZE_MB;
564
+ if (configuredMaxCacheRamMB < minCacheRamMB) {
565
+ percy.log.warn(`--max-cache-ram=${configuredMaxCacheRamMB}MB is below the ${minCacheRamMB}MB minimum ` + `(individual resources up to ${minCacheRamMB}MB would otherwise be dropped). ` + `Continuing with the minimum: ${minCacheRamMB}MB.`);
566
+ effectiveMaxCacheRamMB = minCacheRamMB;
527
567
  } else if (configuredMaxCacheRamMB < MIN_REASONABLE_CAP_MB) {
528
568
  percy.log.warn(`--max-cache-ram=${configuredMaxCacheRamMB}MB is very small; ` + 'most resources will not fit and hit rate will be near zero.');
529
569
  }
package/dist/network.js CHANGED
@@ -2,7 +2,23 @@ import { request as makeRequest } from '@percy/client/utils';
2
2
  import logger from '@percy/logger';
3
3
  import mime from 'mime-types';
4
4
  import { AbortError, DefaultMap, createResource, hostnameMatches, normalizeURL, waitFor, decodeAndEncodeURLWithLogging, handleIncorrectFontMimeType, executeDomainValidation } from './utils.js';
5
- const MAX_RESOURCE_SIZE = 25 * 1024 ** 2 * 0.63; // 25MB, 0.63 factor for accounting for base64 encoding
5
+ export const MAX_RESOURCE_SIZE = 25 * 1024 ** 2 * 0.63; // 25MB, 0.63 factor for accounting for base64 encoding
6
+ // CDP returns binary bodies via Network.getResponseBody as base64 in the JSON-RPC
7
+ // response. Base64 inflates by 4/3, so an N-byte body becomes ~1.33×N on the wire.
8
+ // The ws library defaults maxPayload to 100 MB; browser.js raises it to this value
9
+ // so base64(body) + framing stays well under the limit (see browser.js connect()).
10
+ export const MAX_CDP_PAYLOAD = 150 * 1024 ** 2;
11
+ // 50 MB is a conservative starting point for the PERCY_GZIP raw ceiling
12
+ // (50 × 4/3 ≈ 67 MB on the wire); the final value is pending a storage-cost
13
+ // review (PER-8648).
14
+ export const MAX_RAW_RESOURCE_SIZE_WITH_GZIP = 50 * 1024 ** 2;
15
+ // Direct-fetch fallback needs more time when bodies can be tens of MB.
16
+ const DIRECT_FETCH_TIMEOUT_WITH_GZIP = 30000;
17
+ export function shouldSkipForSize(size) {
18
+ if (!Number.isFinite(size)) return false;
19
+ if (process.env.PERCY_GZIP) return size > MAX_RAW_RESOURCE_SIZE_WITH_GZIP;
20
+ return size > MAX_RESOURCE_SIZE;
21
+ }
6
22
  const ALLOWED_STATUSES = [200, 201, 301, 302, 304, 307, 308];
7
23
  const ALLOWED_RESOURCES = ['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Other'];
8
24
  const ABORTED_MESSAGE = 'Request was aborted by browser';
@@ -309,7 +325,7 @@ export class Network {
309
325
  size: rawValue,
310
326
  snapshot: meta.snapshot
311
327
  });
312
- this.log.debug('- Skipping resource larger than 25MB', meta);
328
+ this.log.debug('- Skipping resource larger than allowed size', meta);
313
329
 
314
330
  // Disposition first, then forget the request — so we never leave Chrome's
315
331
  // Fetch state paused while Percy thinks the request is already done.
@@ -582,7 +598,7 @@ export class Network {
582
598
  }
583
599
 
584
600
  // Logs asset instrumentation for failed/skipped asset loading
585
- function logAssetInstrumentation(log, category, reason, details) {
601
+ export function logAssetInstrumentation(log, category, reason, details) {
586
602
  const categoryMap = {
587
603
  asset_load_5xx: '[ASSET_LOAD_5XX]',
588
604
  asset_not_uploaded: '[ASSET_NOT_UPLOADED]',
@@ -629,7 +645,7 @@ function inspectContentLength(headers) {
629
645
  let key = headers && Object.keys(headers).find(k => k.toLowerCase() === 'content-length');
630
646
  let rawValue = key ? headers[key] : undefined;
631
647
  let parsed = parseInt(rawValue, 10);
632
- let tooLarge = Number.isFinite(parsed) && parsed > MAX_RESOURCE_SIZE;
648
+ let tooLarge = shouldSkipForSize(parsed);
633
649
  let malformed = rawValue !== undefined && rawValue !== null && String(rawValue).length > 0 && !Number.isFinite(parsed);
634
650
  return {
635
651
  tooLarge,
@@ -854,20 +870,23 @@ async function captureResourceDirectly(network, request, session) {
854
870
  ...network.meta,
855
871
  url
856
872
  };
873
+
874
+ // Under PERCY_GZIP the size ceiling can reach tens of MB; 5s is too short.
875
+ let timeoutMs = process.env.PERCY_GZIP ? DIRECT_FETCH_TIMEOUT_WITH_GZIP : DIRECT_FETCH_TIMEOUT;
857
876
  try {
858
877
  log.debug('- Requesting resource directly (responseReceived timeout fallback)', meta);
859
878
  let {
860
879
  body,
861
880
  status,
862
881
  headers: responseHeaders
863
- } = await raceWithTimeout(makeDirectRequest(network, request, session), DIRECT_FETCH_TIMEOUT, `Direct fetch timed out after ${DIRECT_FETCH_TIMEOUT}ms`);
864
- if (body.length > MAX_RESOURCE_SIZE) {
882
+ } = await raceWithTimeout(makeDirectRequest(network, request, session), timeoutMs, `Direct fetch timed out after ${timeoutMs}ms`);
883
+ if (shouldSkipForSize(body.length)) {
865
884
  logAssetInstrumentation(log, 'asset_not_uploaded', 'resource_too_large', {
866
885
  url,
867
886
  size: body.length,
868
887
  snapshot: meta.snapshot
869
888
  });
870
- log.debug('- Skipping resource larger than 25MB', meta);
889
+ log.debug('- Skipping resource larger than allowed size', meta);
871
890
  return;
872
891
  }
873
892
  let urlObj = new URL(url);
@@ -881,6 +900,12 @@ async function captureResourceDirectly(network, request, session) {
881
900
  log.debug(`- Saving direct-fetched resource sha=${resource.sha} mimetype=${mimeType}`, meta);
882
901
  network.intercept.saveResource(resource);
883
902
  } catch (error) {
903
+ logAssetInstrumentation(log, 'asset_load_missing', 'network_error', {
904
+ url,
905
+ snapshot: meta.snapshot,
906
+ requestType: request.type,
907
+ error: error.message
908
+ });
884
909
  log.debug(`Direct fetch failed for ${url} - ${error.message}`, meta);
885
910
  }
886
911
  }
@@ -923,7 +948,12 @@ async function saveResponseResource(network, request, session) {
923
948
  log.debug(`- Capturing auto-validated domain: ${hostname}`, meta);
924
949
  }
925
950
  }
926
- let body = shouldCapture && (await response.buffer());
951
+
952
+ // Bound how long we'll wait for a single body — large bodies over a slow
953
+ // or dead CDP socket must not stall the discovery queue forever. Errors
954
+ // (timeout or underlying CDP failure) propagate to the outer try/catch
955
+ // which logs the failure via the existing "Encountered an error" path.
956
+ let body = shouldCapture && (await raceWithTimeout(response.buffer(), process.env.PERCY_GZIP ? 60000 : 30000, `response.buffer() timed out for ${url}`));
927
957
 
928
958
  // Don't rename the below log line as it is used in getting network logs in api
929
959
  /* istanbul ignore if: first check is a sanity check */
@@ -947,14 +977,14 @@ async function saveResponseResource(network, request, session) {
947
977
  snapshot: meta.snapshot
948
978
  });
949
979
  return log.debug('- Skipping empty response', meta);
950
- } else if (body.length > MAX_RESOURCE_SIZE) {
980
+ } else if (shouldSkipForSize(body.length)) {
951
981
  logAssetInstrumentation(log, 'asset_not_uploaded', 'resource_too_large', {
952
982
  url,
953
983
  size: body.length,
954
984
  snapshot: meta.snapshot
955
985
  });
956
986
  log.debug('- Missing headers for the requested resource.', meta);
957
- return log.debug('- Skipping resource larger than 25MB', meta);
987
+ return log.debug('- Skipping resource larger than allowed size', meta);
958
988
  } else if (!ALLOWED_STATUSES.includes(response.status)) {
959
989
  /* istanbul ignore next: ternary branches tested separately */
960
990
  const category = response.status >= 500 && response.status < 600 ? 'asset_load_5xx' : 'asset_not_uploaded';
@@ -1012,6 +1042,15 @@ async function saveResponseResource(network, request, session) {
1012
1042
  // Don't rename the below log line as it is used in getting network logs in api
1013
1043
  log.debug(`Encountered an error processing resource: ${url}`, meta);
1014
1044
  log.debug(error, meta);
1045
+ // Surface to the structured asset instrumentation channel so support
1046
+ // tooling (analyse_build, analyze_cli_logs_tool) sees the failure
1047
+ // instead of just a debug-level line.
1048
+ logAssetInstrumentation(log, 'asset_load_missing', 'network_error', {
1049
+ url,
1050
+ snapshot: meta.snapshot,
1051
+ requestType: request === null || request === void 0 ? void 0 : request.type,
1052
+ error: error.message
1053
+ });
1015
1054
  }
1016
1055
  }
1017
1056
  if (resource && !resource.root) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@percy/core",
3
- "version": "1.32.0-beta.4",
3
+ "version": "1.32.0-beta.5",
4
4
  "license": "MIT",
5
5
  "repository": {
6
6
  "type": "git",
@@ -46,12 +46,12 @@
46
46
  "dependencies": {
47
47
  "@grpc/grpc-js": "^1.14.3",
48
48
  "@grpc/proto-loader": "^0.8.0",
49
- "@percy/client": "1.32.0-beta.4",
50
- "@percy/config": "1.32.0-beta.4",
51
- "@percy/dom": "1.32.0-beta.4",
52
- "@percy/logger": "1.32.0-beta.4",
53
- "@percy/monitoring": "1.32.0-beta.4",
54
- "@percy/webdriver-utils": "1.32.0-beta.4",
49
+ "@percy/client": "1.32.0-beta.5",
50
+ "@percy/config": "1.32.0-beta.5",
51
+ "@percy/dom": "1.32.0-beta.5",
52
+ "@percy/logger": "1.32.0-beta.5",
53
+ "@percy/monitoring": "1.32.0-beta.5",
54
+ "@percy/webdriver-utils": "1.32.0-beta.5",
55
55
  "busboy": "^1.6.0",
56
56
  "content-disposition": "^0.5.4",
57
57
  "cross-spawn": "^7.0.3",
@@ -67,7 +67,7 @@
67
67
  "yaml": "^2.4.1"
68
68
  },
69
69
  "optionalDependencies": {
70
- "@percy/cli-doctor": "1.32.0-beta.4"
70
+ "@percy/cli-doctor": "1.32.0-beta.5"
71
71
  },
72
- "gitHead": "2e5b1654806b609f961890f1a49fbc4154da1de2"
72
+ "gitHead": "d93415f2cc83533c9c037579e440c6e2a5119920"
73
73
  }