@percy/core 1.10.4 → 1.12.0

package/dist/network.js

@@ -1,11 +1,16 @@
+import mime from 'mime-types';
 import logger from '@percy/logger';
-import { waitFor } from './utils.js';
-import { createRequestHandler, createRequestFinishedHandler, createRequestFailedHandler } from './discovery.js'; // The Interceptor class creates common handlers for dealing with intercepting asset requests
+import { request as makeRequest } from '@percy/client/utils';
+import { normalizeURL, hostnameMatches, createResource, waitFor } from './utils.js';
+const MAX_RESOURCE_SIZE = 15 * 1024 ** 2; // 15MB
+
+const ALLOWED_STATUSES = [200, 201, 301, 302, 304, 307, 308];
+const ALLOWED_RESOURCES = ['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Other']; // The Interceptor class creates common handlers for dealing with intercepting asset requests
 // for a given page using various devtools protocol events and commands.
 
 export class Network {
   static TIMEOUT = 30000;
-  log = logger('core:network');
+  log = logger('core:discovery');
   #pending = new Map();
   #requests = new Map();
   #intercepts = new Map();
@@ -18,14 +23,8 @@ export class Network {
     this.requestHeaders = options.requestHeaders ?? {};
     this.userAgent = options.userAgent ?? // by default, emulate a non-headless browser
     page.session.browser.version.userAgent.replace('Headless', '');
-    this.interceptEnabled = !!options.intercept;
+    this.intercept = options.intercept;
     this.meta = options.meta;
-
-    if (this.interceptEnabled) {
-      this.onRequest = createRequestHandler(this, options.intercept);
-      this.onRequestFinished = createRequestFinishedHandler(this, options.intercept);
-      this.onRequestFailed = createRequestFailedHandler(this, options.intercept);
-    }
   }
 
   watch(session) {
@@ -44,7 +43,7 @@ export class Network {
       headers: this.requestHeaders
     })];
 
-    if (this.interceptEnabled && session.isDocument) {
+    if (this.intercept && session.isDocument) {
       session.on('Fetch.requestPaused', this._handleRequestPaused.bind(this, session));
       session.on('Fetch.authRequired', this._handleAuthRequired.bind(this, session));
       commands.push(session.send('Fetch.enable', {
@@ -73,19 +72,22 @@ export class Network {
       timeout: Network.TIMEOUT,
       idle: timeout
     }).catch(error => {
-      // throw a better timeout error
       if (error.message.startsWith('Timeout')) {
-        let msg = 'Timed out waiting for network requests to idle.';
-
-        if (this.log.shouldLog('debug')) {
-          msg += `\n\n  ${['Active requests:', ...requests.map(r => r.url)].join('\n  - ')}\n`;
-        }
-
-        throw new Error(msg);
+        this._throwTimeoutError('Timed out waiting for network requests to idle.', filter);
       } else {
         throw error;
       }
     });
+  } // Throw a better network timeout error
+
+
+  _throwTimeoutError(msg, filter = () => true) {
+    if (this.log.shouldLog('debug')) {
+      let reqs = Array.from(this.#requests.values()).filter(filter).map(r => r.url);
+      msg += `\n\n  ${['Active requests:', ...reqs].join('\n  - ')}\n`;
+    }
+
+    throw new Error(msg);
   } // Called when a request should be removed from various trackers
 
 
@@ -163,7 +165,7 @@ export class Network {
 
     if (request.url.startsWith('data:')) return;
 
-    if (this.interceptEnabled) {
+    if (this.intercept) {
       let intercept = this.#intercepts.get(requestId);
       this.#pending.set(requestId, event);
 
@@ -186,8 +188,6 @@ export class Network {
   // or abort a request. One of the callbacks is required to be called and only one.
 
   _handleRequest = async (session, event) => {
-    var _this$onRequest;
-
     let {
       request,
       requestId,
@@ -208,37 +208,7 @@ export class Network {
     request.interceptId = interceptId;
     request.redirectChain = redirectChain;
     this.#requests.set(requestId, request);
-    await ((_this$onRequest = this.onRequest) === null || _this$onRequest === void 0 ? void 0 : _this$onRequest.call(this, { ...request,
-      // call to continue the request as-is
-      continue: () => session.send('Fetch.continueRequest', {
-        requestId: interceptId
-      }),
-      // call to respond with a specific status, content, and headers
-      respond: ({
-        status,
-        content,
-        headers
-      }) => session.send('Fetch.fulfillRequest', {
-        requestId: interceptId,
-        responseCode: status || 200,
-        body: Buffer.from(content).toString('base64'),
-        responseHeaders: Object.entries(headers || {}).map(([name, value]) => {
-          return {
-            name: name.toLowerCase(),
-            value: String(value)
-          };
-        })
-      }),
-      // call to fail or abort the request
-      abort: error => session.send('Fetch.failRequest', {
-        requestId: interceptId,
-        // istanbul note: this check used to be necessary and might be again in the future if we
-        // ever need to abort a request due to reasons other than failures
-        errorReason: error ? 'Failed' :
-        /* istanbul ignore next */
-        'Aborted'
-      })
-    }));
+    await sendResponseResource(this, request, session);
   }; // Called when a response has been received for a specific request. Associates the response with
   // the request data and adds a buffer method to fetch the response body when needed.
 
@@ -271,28 +241,174 @@ export class Network {
   // callback. The request should have an associated response and be finished with any redirects.
 
   _handleLoadingFinished = async event => {
-    var _this$onRequestFinish;
-
     let request = this.#requests.get(event.requestId);
     /* istanbul ignore if: race condition paranioa */
 
     if (!request) return;
-    await ((_this$onRequestFinish = this.onRequestFinished) === null || _this$onRequestFinish === void 0 ? void 0 : _this$onRequestFinish.call(this, request));
+    await saveResponseResource(this, request);
 
     this._forgetRequest(request);
   }; // Called when a request has failed loading and triggers the this.onrequestfailed callback.
 
-  _handleLoadingFailed = async event => {
-    var _this$onRequestFailed;
-
+  _handleLoadingFailed = event => {
     let request = this.#requests.get(event.requestId);
     /* istanbul ignore if: race condition paranioa */
 
-    if (!request) return;
-    request.error = event.errorText;
-    await ((_this$onRequestFailed = this.onRequestFailed) === null || _this$onRequestFailed === void 0 ? void 0 : _this$onRequestFailed.call(this, request));
+    if (!request) return; // do not log generic messages since the real error was likely logged elsewhere
+
+    if (event.errorText !== 'net::ERR_FAILED') {
+      let message = `Request failed for ${request.url}: ${event.errorText}`;
+      this.log.debug(message, { ...this.meta,
+        url: request.url
+      });
+    }
 
     this._forgetRequest(request);
   };
+} // Returns the normalized origin URL of a request
+
+function originURL(request) {
+  return normalizeURL((request.redirectChain[0] || request).url);
+} // Send a response for a given request, responding with cached resources when able
+
+
+async function sendResponseResource(network, request, session) {
+  let {
+    disallowedHostnames,
+    disableCache
+  } = network.intercept;
+  let log = network.log;
+  let url = originURL(request);
+  let meta = { ...network.meta,
+    url
+  };
+
+  try {
+    let resource = network.intercept.getResource(url);
+    network.log.debug(`Handling request: ${url}`, meta);
+
+    if (!(resource !== null && resource !== void 0 && resource.root) && hostnameMatches(disallowedHostnames, url)) {
+      log.debug('- Skipping disallowed hostname', meta);
+      await session.send('Fetch.failRequest', {
+        requestId: request.interceptId,
+        errorReason: 'Aborted'
+      });
+    } else if (resource && (resource.root || !disableCache)) {
+      log.debug(resource.root ? '- Serving root resource' : '- Resource cache hit', meta);
+      await session.send('Fetch.fulfillRequest', {
+        requestId: request.interceptId,
+        responseCode: resource.status || 200,
+        body: Buffer.from(resource.content).toString('base64'),
+        responseHeaders: Object.entries(resource.headers || {}).map(([k, v]) => ({
+          name: k.toLowerCase(),
+          value: String(v)
+        }))
+      });
+    } else {
+      await session.send('Fetch.continueRequest', {
+        requestId: request.interceptId
+      });
+    }
+  } catch (error) {
+    /* istanbul ignore next: too hard to test (create race condition) */
+    if (session.closing && error.message.includes('close')) return;
+    log.debug(`Encountered an error handling request: ${url}`, meta);
+    log.debug(error);
+    /* istanbul ignore next: catch race condition */
+
+    await session.send('Fetch.failRequest', {
+      requestId: request.interceptId,
+      errorReason: 'Failed'
+    }).catch(e => log.debug(e, meta));
+  }
+} // Make a new request with Node based on a network request
+
+
+function makeDirectRequest(network, request) {
+  var _network$authorizatio;
+
+  let headers = { ...request.headers
+  };
+
+  if ((_network$authorizatio = network.authorization) !== null && _network$authorizatio !== void 0 && _network$authorizatio.username) {
+    // include basic authorization username and password
+    let {
+      username,
+      password
+    } = network.authorization;
+    let token = Buffer.from([username, password || ''].join(':')).toString('base64');
+    headers.Authorization = `Basic ${token}`;
+  }
+
+  return makeRequest(request.url, {
+    buffer: true,
+    headers
+  });
+} // Save a resource from a request, skipping it if specific paramters are not met
+
+
+async function saveResponseResource(network, request) {
+  let {
+    disableCache,
+    allowedHostnames,
+    enableJavaScript
+  } = network.intercept;
+  let log = network.log;
+  let url = originURL(request);
+  let response = request.response;
+  let meta = { ...network.meta,
+    url
+  };
+  let resource = network.intercept.getResource(url);
+
+  if (!resource || !resource.root && disableCache) {
+    try {
+      log.debug(`Processing resource: ${url}`, meta);
+      let shouldCapture = response && hostnameMatches(allowedHostnames, url);
+      let body = shouldCapture && (await response.buffer());
+      /* istanbul ignore if: first check is a sanity check */
+
+      if (!response) {
+        return log.debug('- Skipping no response', meta);
+      } else if (!shouldCapture) {
+        return log.debug('- Skipping remote resource', meta);
+      } else if (!body.length) {
+        return log.debug('- Skipping empty response', meta);
+      } else if (body.length > MAX_RESOURCE_SIZE) {
+        return log.debug('- Skipping resource larger than 15MB', meta);
+      } else if (!ALLOWED_STATUSES.includes(response.status)) {
+        return log.debug(`- Skipping disallowed status [${response.status}]`, meta);
+      } else if (!enableJavaScript && !ALLOWED_RESOURCES.includes(request.type)) {
+        return log.debug(`- Skipping disallowed resource type [${request.type}]`, meta);
+      }
+
+      let mimeType = // ensure the mimetype is correct for text/plain responses
+      response.mimeType === 'text/plain' && mime.lookup(response.url) || response.mimeType; // font responses from the browser may not be properly encoded, so request them directly
+
+      if (mimeType !== null && mimeType !== void 0 && mimeType.includes('font')) {
+        log.debug('- Requesting asset directly');
+        body = await makeDirectRequest(network, request);
+      }
+
+      resource = createResource(url, body, mimeType, {
+        status: response.status,
+        // 'Network.responseReceived' returns headers split by newlines, however
+        // `Fetch.fulfillRequest` (used for cached responses) will hang with newlines.
+        headers: Object.entries(response.headers).reduce((norm, [key, value]) => Object.assign(norm, {
+          [key]: value.split('\n')
+        }), {})
+      });
+      log.debug(`- sha: ${resource.sha}`, meta);
+      log.debug(`- mimetype: ${resource.mimetype}`, meta);
+    } catch (error) {
+      log.debug(`Encountered an error processing resource: ${url}`, meta);
+      log.debug(error);
+    }
+  }
+
+  if (resource) {
+    network.intercept.saveResource(resource);
+  }
 }
+
 export default Network;

package/dist/page.js

@@ -86,12 +86,18 @@ export class Page {
         return handlers.every(handler => handler.finished);
       }, Page.TIMEOUT)]);
     } catch (error) {
-      // remove handlers and modify the error message
-      for (let handler of handlers) handler.off();
+      // remove any unused handlers
+      for (let handler of handlers) handler.off(); // assign context to unknown errors
 
-      throw Object.assign(error, {
-        message: `Navigation failed: ${error.message}`
-      });
+
+      if (!error.message.startsWith('Timeout')) {
+        throw Object.assign(error, {
+          message: `Navigation failed: ${error.message}`
+        });
+      } // throw a network error to show active requests
+
+
+      this.network._throwTimeoutError(`Navigation failed: Timed out waiting for the page ${waitUntil} event`);
     }
 
     this.log.debug('Page navigated', this.meta);
@@ -130,19 +136,23 @@ export class Page {
     });
 
     for (let script of scripts) await this.eval(script);
-  } // Take a snapshot after waiting for any timeout, waiting for any selector, executing any scripts,
-  // and waiting for the network idle
+  } // Takes a snapshot after waiting for any timeout, waiting for any selector, executing any
+  // scripts, and waiting for the network idle. Returns all other provided snapshot options along
+  // with the captured URL and DOM snapshot.
 
 
   async snapshot({
-    name,
     waitForTimeout,
     waitForSelector,
     execute,
-    meta,
-    ...options
+    ...snapshot
   }) {
-    this.log.debug(`Taking snapshot: ${name}`, this.meta); // wait for any specified timeout
+    let {
+      name,
+      width,
+      enableJavaScript
+    } = snapshot;
+    this.log.debug(`Taking snapshot: ${name}${width ? ` @${width}px` : ''}`, this.meta); // wait for any specified timeout
 
     if (waitForTimeout) {
       this.log.debug(`Wait for ${waitForTimeout}ms timeout`, this.meta);
@@ -178,11 +188,16 @@ export class Page {
     this.log.debug('Serialize DOM', this.meta);
     /* istanbul ignore next: no instrumenting injected code */
 
-    return await this.eval((_, options) => ({
+    let capture = await this.eval((_, options) => ({
       /* eslint-disable-next-line no-undef */
-      dom: PercyDOM.serialize(options),
+      domSnapshot: PercyDOM.serialize(options),
       url: document.URL
-    }), options);
+    }), {
+      enableJavaScript
+    });
+    return { ...snapshot,
+      ...capture
+    };
   } // Initialize newly attached pages and iframes with page options