@percy/core 1.10.3 → 1.11.0

package/dist/discovery.js

@@ -1,140 +1,302 @@
-import mime from 'mime-types';
 import logger from '@percy/logger';
-import { request as makeRequest } from '@percy/client/utils';
-import { normalizeURL, hostnameMatches, createResource } from './utils.js';
-const MAX_RESOURCE_SIZE = 15 * 1024 ** 2; // 15MB
-
-const ALLOWED_STATUSES = [200, 201, 301, 302, 304, 307, 308];
-const ALLOWED_RESOURCES = ['Document', 'Stylesheet', 'Image', 'Media', 'Font', 'Other'];
-export function createRequestHandler(network, {
-  disableCache,
-  disallowedHostnames,
-  getResource
-}) {
-  let log = logger('core:discovery');
-  return async request => {
-    let url = request.url;
-    let meta = { ...network.meta,
-      url
-    };
+import Queue from './queue.js';
+import { normalizeURL, hostnameMatches, createRootResource, createPercyCSSResource, createLogResource, yieldAll } from './utils.js'; // Logs verbose debug logs detailing various snapshot options.
 
-    try {
-      log.debug(`Handling request: ${url}`, meta);
-      let resource = getResource(url);
-
-      if (resource !== null && resource !== void 0 && resource.root) {
-        log.debug('- Serving root resource', meta);
-        await request.respond(resource);
-      } else if (hostnameMatches(disallowedHostnames, url)) {
-        log.debug('- Skipping disallowed hostname', meta);
-        await request.abort(true);
-      } else if (resource && !disableCache) {
-        log.debug('- Resource cache hit', meta);
-        await request.respond(resource);
-      } else {
-        await request.continue();
-      }
-    } catch (error) {
-      log.debug(`Encountered an error handling request: ${url}`, meta);
-      log.debug(error);
-      /* istanbul ignore next: race condition */
+function debugSnapshotOptions(snapshot) {
+  let log = logger('core:snapshot'); // log snapshot info
 
-      await request.abort(error).catch(e => log.debug(e, meta));
+  log.debug('---------', snapshot.meta);
+  log.debug(`Received snapshot: ${snapshot.name}`, snapshot.meta); // will log debug info for an object property if its value is defined
+
+  let debugProp = (obj, prop, format = String) => {
+    let val = prop.split('.').reduce((o, k) => o === null || o === void 0 ? void 0 : o[k], obj);
+
+    if (val != null) {
+      // join formatted array values with a space
+      val = [].concat(val).map(format).join(', ');
+      log.debug(`- ${prop}: ${val}`, snapshot.meta);
     }
   };
-}
-export function createRequestFinishedHandler(network, {
-  enableJavaScript,
-  allowedHostnames,
-  disableCache,
-  getResource,
-  saveResource
+
+  debugProp(snapshot, 'url');
+  debugProp(snapshot, 'scope');
+  debugProp(snapshot, 'widths', v => `${v}px`);
+  debugProp(snapshot, 'minHeight', v => `${v}px`);
+  debugProp(snapshot, 'enableJavaScript');
+  debugProp(snapshot, 'deviceScaleFactor');
+  debugProp(snapshot, 'waitForTimeout');
+  debugProp(snapshot, 'waitForSelector');
+  debugProp(snapshot, 'execute.afterNavigation');
+  debugProp(snapshot, 'execute.beforeResize');
+  debugProp(snapshot, 'execute.afterResize');
+  debugProp(snapshot, 'execute.beforeSnapshot');
+  debugProp(snapshot, 'discovery.allowedHostnames');
+  debugProp(snapshot, 'discovery.disallowedHostnames');
+  debugProp(snapshot, 'discovery.requestHeaders', JSON.stringify);
+  debugProp(snapshot, 'discovery.authorization', JSON.stringify);
+  debugProp(snapshot, 'discovery.disableCache');
+  debugProp(snapshot, 'discovery.userAgent');
+  debugProp(snapshot, 'clientInfo');
+  debugProp(snapshot, 'environmentInfo');
+  debugProp(snapshot, 'domSnapshot', Boolean);
+
+  for (let added of snapshot.additionalSnapshots || []) {
+    log.debug(`Additional snapshot: ${added.name}`, snapshot.meta);
+    debugProp(added, 'waitForTimeout');
+    debugProp(added, 'waitForSelector');
+    debugProp(added, 'execute');
+  }
+} // Wait for a page's asset discovery network to idle
+
+
+function waitForDiscoveryNetworkIdle(page, options) {
+  let {
+    allowedHostnames,
+    networkIdleTimeout
+  } = options;
+
+  let filter = r => hostnameMatches(allowedHostnames, r.url);
+
+  return page.network.idle(filter, networkIdleTimeout);
+} // Calls the provided callback with additional resources
+
+
+function processSnapshotResources({
+  domSnapshot,
+  resources,
+  ...snapshot
 }) {
-  let log = logger('core:discovery');
-  return async request => {
-    let origin = request.redirectChain[0] || request;
-    let url = normalizeURL(origin.url);
-    let meta = { ...network.meta,
-      url
+  var _resources;
+
+  resources = [...(((_resources = resources) === null || _resources === void 0 ? void 0 : _resources.values()) ?? [])]; // find or create a root resource if one does not exist
+
+  let root = resources.find(r => r.content === domSnapshot);
+
+  if (!root) {
+    root = createRootResource(snapshot.url, domSnapshot);
+    resources.unshift(root);
+  } // inject Percy CSS
+
+
+  if (snapshot.percyCSS) {
+    let css = createPercyCSSResource(root.url, snapshot.percyCSS);
+    resources.push(css); // replace root contents and associated properties
+
+    Object.assign(root, createRootResource(root.url, root.content.replace(/(<\/body>)(?!.*\1)/is, `<link data-percy-specific-css rel="stylesheet" href="${css.pathname}"/>` + '$&')));
+  } // include associated snapshot logs matched by meta information
+
+
+  resources.push(createLogResource(logger.query(log => {
+    var _log$meta$snapshot;
+
+    return ((_log$meta$snapshot = log.meta.snapshot) === null || _log$meta$snapshot === void 0 ? void 0 : _log$meta$snapshot.name) === snapshot.meta.snapshot.name;
+  })));
+  return { ...snapshot,
+    resources
+  };
+} // Triggers the capture of resource requests for a page by iterating over snapshot widths to resize
+// the page and calling any provided execute options.
+
+
+async function* captureSnapshotResources(page, snapshot, options) {
+  let {
+    discovery,
+    additionalSnapshots = [],
+    ...baseSnapshot
+  } = snapshot;
+  let {
+    capture,
+    captureWidths,
+    deviceScaleFactor,
+    mobile
+  } = options; // used to take snapshots and remove any discovered root resource
+
+  let takeSnapshot = async (options, width) => {
+    if (captureWidths) options = { ...options,
+      width
     };
+    let captured = await page.snapshot(options);
+    captured.resources.delete(normalizeURL(captured.url));
+    capture(processSnapshotResources(captured));
+    return captured;
+  }; // used to resize the using capture options
 
-    try {
-      var _resource;
-
-      let resource = getResource(url); // process and cache the response and resource
-
-      if (!((_resource = resource) !== null && _resource !== void 0 && _resource.root) && (!resource || disableCache)) {
-        let headers = request.headers;
-        let response = request.response;
-        let capture = response && hostnameMatches(allowedHostnames, url);
-        let body = capture && (await response.buffer());
-        log.debug(`Processing resource: ${url}`, meta);
-        /* istanbul ignore next: sanity check */
-
-        if (!response) {
-          return log.debug('- Skipping no response', meta);
-        } else if (!capture) {
-          return log.debug('- Skipping remote resource', meta);
-        } else if (!body.length) {
-          return log.debug('- Skipping empty response', meta);
-        } else if (body.length > MAX_RESOURCE_SIZE) {
-          return log.debug('- Skipping resource larger than 15MB', meta);
-        } else if (!ALLOWED_STATUSES.includes(response.status)) {
-          return log.debug(`- Skipping disallowed status [${response.status}]`, meta);
-        } else if (!enableJavaScript && !ALLOWED_RESOURCES.includes(request.type)) {
-          return log.debug(`- Skipping disallowed resource type [${request.type}]`, meta);
-        } // Try to get the proper mimetype if the server or asset discovery browser is sending `text/plain`
-
-
-        let mimeType = response.mimeType === 'text/plain' && mime.lookup(response.url) || response.mimeType; // font responses from the browser may not be properly encoded, so request them directly
-
-        if (mimeType !== null && mimeType !== void 0 && mimeType.includes('font')) {
-          var _network$authorizatio;
-
-          log.debug('- Requesting asset directly');
-
-          if (!headers.Authorization && (_network$authorizatio = network.authorization) !== null && _network$authorizatio !== void 0 && _network$authorizatio.username) {
-            let token = Buffer.from([network.authorization.username, network.authorization.password || ''].join(':')).toString('base64');
-            headers.Authorization = `Basic ${token}`;
-          }
-
-          body = await makeRequest(response.url, {
-            buffer: true,
-            headers
-          });
-        }
-
-        resource = createResource(url, body, mimeType, {
-          status: response.status,
-          // 'Network.responseReceived' returns headers split by newlines, however
-          // `Fetch.fulfillRequest` (used for cached responses) will hang with newlines.
-          headers: Object.entries(response.headers).reduce((norm, [key, value]) => Object.assign(norm, {
-            [key]: value.split('\n')
-          }), {})
-        });
-        log.debug(`- sha: ${resource.sha}`, meta);
-        log.debug(`- mimetype: ${resource.mimetype}`, meta);
+
+  let resizePage = width => page.resize({
+    height: snapshot.minHeight,
+    deviceScaleFactor,
+    mobile,
+    width
+  }); // navigate to the url
+
+
+  yield resizePage(snapshot.widths[0]);
+  yield page.goto(snapshot.url);
+
+  if (snapshot.execute) {
+    // when any execute options are provided, inject snapshot options
+
+    /* istanbul ignore next: cannot detect coverage of injected code */
+    yield page.eval((_, s) => window.__PERCY__.snapshot = s, snapshot);
+    yield page.evaluate(snapshot.execute.afterNavigation);
+  } // iterate over additional snapshots for proper DOM capturing
+
+
+  for (let additionalSnapshot of [baseSnapshot, ...additionalSnapshots]) {
+    let isBaseSnapshot = additionalSnapshot === baseSnapshot;
+    let snap = { ...baseSnapshot,
+      ...additionalSnapshot
+    };
+    let {
+      widths,
+      execute
+    } = snap;
+    let [width] = widths; // iterate over widths to trigger reqeusts and capture other widths
+
+    if (isBaseSnapshot || captureWidths) {
+      for (let i = 0; i < widths.length - 1; i++) {
+        if (captureWidths) yield takeSnapshot(snap, width);
+        yield page.evaluate(execute === null || execute === void 0 ? void 0 : execute.beforeResize);
+        yield waitForDiscoveryNetworkIdle(page, discovery);
+        yield resizePage(width = widths[i + 1]);
+        yield page.evaluate(execute === null || execute === void 0 ? void 0 : execute.afterResize);
       }
+    }
+
+    if (capture && !snapshot.domSnapshot) {
+      // capture this snapshot and update the base snapshot after capture
+      let captured = yield takeSnapshot(snap, width);
+      if (isBaseSnapshot) baseSnapshot = captured; // resize back to the initial width when capturing additional snapshot widths
 
-      saveResource(resource);
-    } catch (error) {
-      log.debug(`Encountered an error processing resource: ${url}`, meta);
-      log.debug(error);
+      if (captureWidths && additionalSnapshots.length) {
+        let l = additionalSnapshots.indexOf(additionalSnapshot) + 1;
+        if (l < additionalSnapshots.length) yield resizePage(snapshot.widths[0]);
+      }
     }
-  };
-}
-export function createRequestFailedHandler(network) {
-  let log = logger('core:discovery');
-  return ({
-    url,
-    error
-  }) => {
-    // do not log generic failures since the real error was most likely
-    // already logged from elsewhere
-    if (error !== 'net::ERR_FAILED') {
-      log.debug(`Request failed for ${url}: ${error}`, { ...network.meta,
-        url
+  } // recursively trigger resource requests for any alternate device pixel ratio
+
+
+  if (deviceScaleFactor !== discovery.devicePixelRatio) {
+    yield waitForDiscoveryNetworkIdle(page, discovery);
+    yield* captureSnapshotResources(page, snapshot, {
+      deviceScaleFactor: discovery.devicePixelRatio,
+      mobile: true
+    });
+  } // wait for final network idle when not capturing DOM
+
+
+  if (capture && snapshot.domSnapshot) {
+    yield waitForDiscoveryNetworkIdle(page, discovery);
+    capture(processSnapshotResources(snapshot));
+  }
+} // Pushes all provided snapshots to a discovery queue with the provided callback, yielding to each
+// one concurrently. When skipping asset discovery, the callback is called immediately for each
+// snapshot, also processing snapshot resources when not dry-running.
+
+
+export async function* discoverSnapshotResources(queue, options, callback) {
+  let {
+    snapshots,
+    skipDiscovery,
+    dryRun
+  } = options;
+  yield* yieldAll(snapshots.reduce((all, snapshot) => {
+    debugSnapshotOptions(snapshot);
+
+    if (skipDiscovery) {
+      let {
+        additionalSnapshots,
+        ...baseSnapshot
+      } = snapshot;
+      additionalSnapshots = dryRun && additionalSnapshots || [];
+
+      for (let snap of [baseSnapshot, ...additionalSnapshots]) {
+        callback(dryRun ? snap : processSnapshotResources(snap));
+      }
+    } else {
+      all.push(queue.push(snapshot, callback));
+    }
+
+    return all;
+  }, []));
+} // Used to cache resources across core instances
+
+const RESOURCE_CACHE_KEY = Symbol('resource-cache'); // Creates an asset discovery queue that uses the percy browser instance to create a page for each
+// snapshot which is used to intercept and capture snapshot resource requests.
+
+export function createDiscoveryQueue(percy) {
+  let {
+    concurrency
+  } = percy.config.discovery;
+  let queue = new Queue();
+  let cache;
+  return queue.set({
+    concurrency
+  }) // on start, launch the browser and run the queue
+  .handle('start', async () => {
+    cache = percy[RESOURCE_CACHE_KEY] = new Map();
+    await percy.browser.launch();
+    queue.run();
+  }) // on end, close the browser
+  .handle('end', async () => {
+    await percy.browser.close();
+  }) // snapshots are unique by name; when deferred also by widths
+  .handle('find', ({
+    name,
+    widths
+  }, snapshot) => snapshot.name === name && (!percy.deferUploads || !widths || widths.join() === snapshot.widths.join())) // initialize the root resource for DOM snapshots
+  .handle('push', snapshot => {
+    let {
+      url,
+      domSnapshot
+    } = snapshot;
+    let root = domSnapshot && createRootResource(url, domSnapshot);
+    let resources = new Map(root ? [[root.url, root]] : []);
+    return { ...snapshot,
+      resources
+    };
+  }) // discovery resources for snapshots and call the callback for each discovered snapshot
+  .handle('task', async function* (snapshot, callback) {
+    percy.log.debug(`Discovering resources: ${snapshot.name}`, snapshot.meta); // create a new browser page
+
+    let page = yield percy.browser.page({
+      enableJavaScript: snapshot.enableJavaScript ?? !snapshot.domSnapshot,
+      networkIdleTimeout: snapshot.discovery.networkIdleTimeout,
+      requestHeaders: snapshot.discovery.requestHeaders,
+      authorization: snapshot.discovery.authorization,
+      userAgent: snapshot.discovery.userAgent,
+      meta: snapshot.meta,
+      // enable network inteception
+      intercept: {
+        enableJavaScript: snapshot.enableJavaScript,
+        disableCache: snapshot.discovery.disableCache,
+        allowedHostnames: snapshot.discovery.allowedHostnames,
+        disallowedHostnames: snapshot.discovery.disallowedHostnames,
+        getResource: u => snapshot.resources.get(u) || cache.get(u),
+        saveResource: r => snapshot.resources.set(r.url, r) && cache.set(r.url, r)
+      }
+    });
+
+    try {
+      yield* captureSnapshotResources(page, snapshot, {
+        captureWidths: !snapshot.domSnapshot && percy.deferUploads,
+        capture: callback
       });
+    } finally {
+      // always close the page when done
+      await page.close();
     }
-  };
+  }).handle('error', ({
+    name,
+    meta
+  }, error) => {
+    if (error.name === 'AbortError' && queue.readyState < 3) {
+      // only error about aborted snapshots when not closed
+      percy.log.error('Received a duplicate snapshot, ' + `the previous snapshot was aborted: ${name}`, meta);
+    } else {
+      // log all other encountered errors
+      percy.log.error(`Encountered an error taking snapshot: ${name}`, meta);
+      percy.log.error(error, meta);
+    }
+  });
 }