browsertime 26.3.2 → 27.0.0

package/lib/core/engine/collector.js

@@ -142,37 +142,7 @@ export class Collector {
    */
   async perIteration(allData, index) {
     for (let data of allData) {
-      const alias = data.alias;
-      let url = data.url || this.urlFromCli;
-
-      if (data.browserScripts && data.browserScripts.pageinfo) {
-        const actualURL = data.browserScripts.pageinfo.url;
-        // If the URL that we wanted to test do not match the actual URL used
-        // which can happen in scripting when using alias and URLs has session etc in it
-        if (url !== actualURL) {
-          this.urlAndActualUrl[actualURL] = url;
-        }
-      }
-      if (alias && !this.aliasAndUrl[alias]) {
-        this.aliasAndUrl[alias] = url;
-      } else if (alias && this.aliasAndUrl[alias]) {
-        url = this.aliasAndUrl[alias];
-      }
-
-      const results = this.allResults[url] ?? getNewResult(url, this.options);
-      results.info.description = allData.description;
-      results.info.title = allData.title;
-      results.info.browser.userAgent = getProperty(
-        allData[0],
-        'browserScripts.browser.userAgent',
-        ''
-      );
-
-      if (allData.screenshots && allData.screenshots.length > 0) {
-        results.files.screenshot.push(allData.screenshots);
-      }
-
-      const statistics = this.allStats[url] ?? new Statistics();
+      const { url, results, statistics } = this._resolveURL(data, allData);
 
       // The user can add extra data in a script or post script
       if (data.extras) {
@@ -200,35 +170,8 @@ export class Collector {
       results.errors.push(data.error || []);
       // From each iteration, collect the result we want
       results.timestamps.push(data.timestamp);
-      if (data.browserScripts) {
-        results.browserScripts.push(data.browserScripts);
-        // Add all browserscripts to the stats
-        const equals = (a1, a2) => JSON.stringify(a1) === JSON.stringify(a2);
-        statistics.addDeep(data.browserScripts, (keyPath, value) => {
-          if (equals(keyPath.slice(-2), ['userTimings', 'marks'])) {
-            // eslint-disable-next-line unicorn/no-array-reduce
-            return value.reduce((result, mark) => {
-              result[mark.name] = mark.startTime;
-              return result;
-            }, {});
-          } else if (equals(keyPath.slice(-2), ['userTimings', 'measures'])) {
-            // eslint-disable-next-line unicorn/no-array-reduce
-            return value.reduce((result, mark) => {
-              result[mark.name] = mark.duration;
-              return result;
-            }, {});
-          } else if (equals(keyPath.slice(-1), ['resourceTimings'])) {
-            return {};
-          } else if (equals(keyPath.slice(-2), ['timings', 'serverTimings'])) {
-            // eslint-disable-next-line unicorn/no-array-reduce
-            return value.reduce((result, timing) => {
-              result[timing.name] = timing.duration;
-              return result;
-            }, {});
-          }
-          return value;
-        });
-      }
+
+      this._collectBrowserScripts(data, statistics, results);
 
       if (data.alias) {
         results.info.alias = data.alias;
@@ -242,116 +185,11 @@ export class Collector {
         });
       }
 
-      if (this.options.iterations > 1) {
-        const ttfb = getProperty(
-          data,
-          'browserScripts.timings.pageTimings.backEndTime'
-        );
-        const domContentLoaded = getProperty(
-          data,
-          'browserScripts.timings.pageTimings.domContentLoadedTime'
-        );
-        const firstPaint = getProperty(
-          data,
-          'browserScripts.timings.firstPaint'
-        );
-        const fcp = getProperty(
-          data,
-          "browserScripts.timings.paintTiming['first-contentful-paint']"
-        );
-        const pageLoadTime = getProperty(
-          data,
-          'browserScripts.timings.pageTimings.pageLoadTime'
-        );
-        const lcp = getProperty(
-          data,
-          'browserScripts.timings.largestContentfulPaint.renderTime'
-        );
-        const cls = getProperty(
-          data,
-          'browserScripts.pageinfo.cumulativeLayoutShift'
-        );
-        const tbt = getProperty(data, 'cpu.longTasks.totalBlockingTime');
-        const mem = getProperty(data, 'memory');
-
-        log.info(
-          `${url} ${ttfb ? formatMetric('TTFB', ttfb, false, true) + ' ' : ''}${
-            domContentLoaded
-              ? formatMetric(
-                  'DOMContentLoaded',
-                  domContentLoaded,
-                  false,
-                  true
-                ) + ' '
-              : ''
-          }${
-            firstPaint
-              ? formatMetric('firstPaint', firstPaint, false, true) + ' '
-              : ''
-          }${fcp ? formatMetric('FCP', fcp, false, true) + ' ' : ''}${
-            lcp ? formatMetric('LCP', lcp, false, true) + ' ' : ''
-          }${
-            pageLoadTime
-              ? formatMetric('Load', pageLoadTime, false, true) + ' '
-              : ''
-          }${tbt ? formatMetric('TBT', tbt, false, true) + ' ' : ''}${
-            cls ? 'CLS:' + cls.toFixed(4) : ''
-          }${
-            mem
-              ? formatMetric(
-                  'Memory',
-                  Math.round(mem / 1024 / 1024),
-                  false,
-                  false
-                ) + 'mb '
-              : ''
-          }`
-        );
-      }
+      this._logIterationMetrics(data, url);
 
       if (data.visualMetrics) {
         if (this.options.iterations > 1) {
-          log.info(
-            `VisualMetrics: ${formatMetric(
-              'FirstVisualChange',
-              data.visualMetrics.FirstVisualChange,
-              false,
-              true
-            )} ${formatMetric(
-              'SpeedIndex',
-              data.visualMetrics.SpeedIndex,
-              false,
-              true
-            )}${
-              data.visualMetrics.PerceptualSpeedIndex
-                ? formatMetric(
-                    ' PerceptualSpeedIndex',
-                    data.visualMetrics.PerceptualSpeedIndex,
-                    false,
-                    true
-                  )
-                : ''
-            }${
-              data.visualMetrics.ContentfulSpeedIndex
-                ? formatMetric(
-                    ' ContentfulSpeedIndex',
-                    data.visualMetrics.ContentfulSpeedIndex,
-                    false,
-                    true
-                  )
-                : ''
-            } ${formatMetric(
-              'VisualComplete85',
-              data.visualMetrics.VisualComplete85,
-              false,
-              true
-            )} ${formatMetric(
-              'LastVisualChange',
-              data.visualMetrics.LastVisualChange,
-              false,
-              true
-            )}`
-          );
+          this._logVisualMetrics(data);
         }
         for (let key of Object.keys(data.visualMetrics)) {
           // Skip VisualProgress/ContentfulProgress etc
@@ -363,195 +201,380 @@ export class Collector {
         }
         results.visualMetrics.push(data.visualMetrics);
       }
+
+      this._collectPerformanceData(data, statistics, results);
+      this._computeDeltaToTTFB(data, results, statistics);
+
+      // Store all extra JSON metrics
       // Put all work that involves storing metrics to disk in a
       // array and the promise them all later on
       const extraWork = [];
-
-      if (
-        data.browserScripts &&
-        data.browserScripts.pageinfo &&
-        data.browserScripts.pageinfo.longTask
-      ) {
-        for (let longTask of data.browserScripts.pageinfo.longTask) {
-          statistics.addDeep({
-            cpu: {
-              longTasks: {
-                durations: longTask.duration
-              }
-            }
-          });
-        }
-      }
-
-      // Add CPU data (if we have it)
-      if (data.cpu) {
-        results.cpu.push(data.cpu);
-        // We skip adding stats per URL .., we should do it per domain instead in sitespeed.io
-        if (data.cpu.categories) {
-          statistics.addDeep({
-            cpu: {
-              categories: data.cpu.categories,
-              events: data.cpu.events
-            }
-          });
-        }
-        if (data.cpu.longTasks) {
-          statistics.addDeep({
-            cpu: {
-              longTasks: data.cpu.longTasks
-            }
-          });
-        }
+      for (let filename of Object.keys(data.extraJson)) {
+        extraWork.push(
+          this.storageManager.writeJson(
+            path.join(pathToFolder(url, this.options), filename),
+            data.extraJson[filename],
+            true
+          )
+        );
       }
 
-      if (data.googleWebVitals) {
-        results.googleWebVitals.push(data.googleWebVitals);
-        statistics.addDeep({
-          googleWebVitals: data.googleWebVitals
-        });
-      }
+      this._collectFilePaths(url, index, results);
 
-      if (data.renderBlocking) {
-        results.renderBlocking.push(data.renderBlocking);
+      await Promise.all(extraWork);
+      this.allStats[url] = statistics;
+      this.allResults[url] = results;
+    }
+  }
 
-        statistics.addDeep({
-          renderBlocking: {
-            recalculateStyle: data.renderBlocking.recalculateStyle
-          }
-        });
-      }
+  _resolveURL(data, allData) {
+    const alias = data.alias;
+    let url = data.url || this.urlFromCli;
 
-      // Add power data (if we have it)
-      if (data.power) {
-        results.android.power.push(data.power);
-        statistics.addDeep({
-          android: { power: data.power }
-        });
+    if (data.browserScripts && data.browserScripts.pageinfo) {
+      const actualURL = data.browserScripts.pageinfo.url;
+      // If the URL that we wanted to test do not match the actual URL used
+      // which can happen in scripting when using alias and URLs has session etc in it
+      if (url !== actualURL) {
+        this.urlAndActualUrl[actualURL] = url;
       }
+    }
+    if (alias && !this.aliasAndUrl[alias]) {
+      this.aliasAndUrl[alias] = url;
+    } else if (alias && this.aliasAndUrl[alias]) {
+      url = this.aliasAndUrl[alias];
+    }
 
-      // Add Firefox perfStats if available
-      if (this.options.firefox && this.options.firefox.perfStats) {
-        if (!results.geckoPerfStats) {
-          results.geckoPerfStats = [];
-        }
-        results.geckoPerfStats.push(data.perfStats);
-      }
+    const results = this.allResults[url] ?? getNewResult(url, this.options);
+    results.info.description = allData.description;
+    results.info.title = allData.title;
+    results.info.browser.userAgent = getProperty(
+      allData[0],
+      'browserScripts.browser.userAgent',
+      ''
+    );
+
+    if (allData.screenshots && allData.screenshots.length > 0) {
+      results.files.screenshot.push(allData.screenshots);
+    }
 
-      // Add Firefox cpu power
-      if (data.powerConsumption) {
-        if (!results.powerConsumption) {
-          results.powerConsumption = [];
-        }
-        results.powerConsumption.push(data.powerConsumption);
-        statistics.addDeep({
-          powerConsumption: data.powerConsumption
-        });
-      }
+    const statistics = this.allStats[url] ?? new Statistics();
 
-      // Add total memory
-      if (this.options.firefox && this.options.firefox.memoryReport) {
-        statistics.addDeep({
-          memory: data.memory
-        });
+    return { url, results, statistics };
+  }
 
-        if (!results.memory) {
-          results.memory = [];
+  _collectBrowserScripts(data, statistics, results) {
+    if (data.browserScripts) {
+      results.browserScripts.push(data.browserScripts);
+      // Add all browserscripts to the stats
+      const equals = (a1, a2) => JSON.stringify(a1) === JSON.stringify(a2);
+      statistics.addDeep(data.browserScripts, (keyPath, value) => {
+        if (equals(keyPath.slice(-2), ['userTimings', 'marks'])) {
+          // eslint-disable-next-line unicorn/no-array-reduce
+          return value.reduce((result, mark) => {
+            result[mark.name] = mark.startTime;
+            return result;
+          }, {});
+        } else if (equals(keyPath.slice(-2), ['userTimings', 'measures'])) {
+          // eslint-disable-next-line unicorn/no-array-reduce
+          return value.reduce((result, mark) => {
+            result[mark.name] = mark.duration;
+            return result;
+          }, {});
+        } else if (equals(keyPath.slice(-1), ['resourceTimings'])) {
+          return {};
+        } else if (equals(keyPath.slice(-2), ['timings', 'serverTimings'])) {
+          // eslint-disable-next-line unicorn/no-array-reduce
+          return value.reduce((result, timing) => {
+            result[timing.name] = timing.duration;
+            return result;
+          }, {});
         }
-        results.memory.push(data.memory);
-      }
+        return value;
+      });
+    }
+  }
 
-      // Add delta to TTFB
-      const deltaToTTFB = {};
+  _logIterationMetrics(data, url) {
+    if (this.options.iterations > 1) {
+      const ttfb = getProperty(
+        data,
+        'browserScripts.timings.pageTimings.backEndTime'
+      );
+      const domContentLoaded = getProperty(
+        data,
+        'browserScripts.timings.pageTimings.domContentLoadedTime'
+      );
+      const firstPaint = getProperty(data, 'browserScripts.timings.firstPaint');
       const fcp = getProperty(
         data,
-        'browserScripts.timings.paintTiming.first-contentful-paint'
+        "browserScripts.timings.paintTiming['first-contentful-paint']"
+      );
+      const pageLoadTime = getProperty(
+        data,
+        'browserScripts.timings.pageTimings.pageLoadTime'
       );
-      const ttfb = getProperty(data, 'browserScripts.timings.ttfb');
       const lcp = getProperty(
         data,
         'browserScripts.timings.largestContentfulPaint.renderTime'
       );
-      const firstVisualChange = getProperty(
+      const cls = getProperty(
         data,
-        'visualMetrics.FirstVisualChange'
+        'browserScripts.pageinfo.cumulativeLayoutShift'
       );
-      const lastVisualChange = getProperty(
-        data,
-        'visualMetrics.LastVisualChange'
+      const tbt = getProperty(data, 'cpu.longTasks.totalBlockingTime');
+      const mem = getProperty(data, 'memory');
+
+      log.info(
+        `${url} ${ttfb ? formatMetric('TTFB', ttfb, false, true) + ' ' : ''}${
+          domContentLoaded
+            ? formatMetric('DOMContentLoaded', domContentLoaded, false, true) +
+              ' '
+            : ''
+        }${
+          firstPaint
+            ? formatMetric('firstPaint', firstPaint, false, true) + ' '
+            : ''
+        }${fcp ? formatMetric('FCP', fcp, false, true) + ' ' : ''}${
+          lcp ? formatMetric('LCP', lcp, false, true) + ' ' : ''
+        }${
+          pageLoadTime
+            ? formatMetric('Load', pageLoadTime, false, true) + ' '
+            : ''
+        }${tbt ? formatMetric('TBT', tbt, false, true) + ' ' : ''}${
+          cls ? 'CLS:' + cls.toFixed(4) : ''
+        }${
+          mem
+            ? formatMetric(
+                'Memory',
+                Math.round(mem / 1024 / 1024),
+                false,
+                false
+              ) + 'mb '
+            : ''
+        }`
       );
+    }
+  }
 
-      if (fcp) {
-        deltaToTTFB['firstContentfulPaint'] = fcp - ttfb;
-      }
-      if (lcp) {
-        deltaToTTFB['largestContentfulPaint'] = lcp - ttfb;
-      }
-      if (firstVisualChange) {
-        deltaToTTFB['firstVisualChange'] = firstVisualChange - ttfb;
-      }
-      if (lastVisualChange) {
-        deltaToTTFB['lastVisualChange'] = lastVisualChange - ttfb;
-      }
-      results.deltaToTTFB.push(deltaToTTFB);
-      statistics.addDeep({ deltaToTFFB: deltaToTTFB });
+  _logVisualMetrics(data) {
+    log.info(
+      `VisualMetrics: ${formatMetric(
+        'FirstVisualChange',
+        data.visualMetrics.FirstVisualChange,
+        false,
+        true
+      )} ${formatMetric(
+        'SpeedIndex',
+        data.visualMetrics.SpeedIndex,
+        false,
+        true
+      )}${
+        data.visualMetrics.PerceptualSpeedIndex
+          ? formatMetric(
+              ' PerceptualSpeedIndex',
+              data.visualMetrics.PerceptualSpeedIndex,
+              false,
+              true
+            )
+          : ''
+      }${
+        data.visualMetrics.ContentfulSpeedIndex
+          ? formatMetric(
+              ' ContentfulSpeedIndex',
+              data.visualMetrics.ContentfulSpeedIndex,
+              false,
+              true
+            )
+          : ''
+      } ${formatMetric(
+        'VisualComplete85',
+        data.visualMetrics.VisualComplete85,
+        false,
+        true
+      )} ${formatMetric(
+        'LastVisualChange',
+        data.visualMetrics.LastVisualChange,
+        false,
+        true
+      )}`
+    );
+  }
 
-      // Store all extra JSON metrics
-      for (let filename of Object.keys(data.extraJson)) {
-        extraWork.push(
-          this.storageManager.writeJson(
-            path.join(pathToFolder(url, this.options), filename),
-            data.extraJson[filename],
-            true
-          )
-        );
+  _collectPerformanceData(data, statistics, results) {
+    if (
+      data.browserScripts &&
+      data.browserScripts.pageinfo &&
+      data.browserScripts.pageinfo.longTask
+    ) {
+      for (let longTask of data.browserScripts.pageinfo.longTask) {
+        statistics.addDeep({
+          cpu: {
+            longTasks: {
+              durations: longTask.duration
+            }
+          }
+        });
       }
+    }
 
-      if (this.options.video) {
-        results.files.video.push(
-          `${pathToFolder(url, this.options)}video/${index}.mp4`
-        );
-      }
-      if (this.options.chrome && this.options.chrome.timeline) {
-        results.files.timeline.push(
-          `${pathToFolder(url, this.options)}trace-${index}.json.gz`
-        );
+    // Add CPU data (if we have it)
+    if (data.cpu) {
+      results.cpu.push(data.cpu);
+      // We skip adding stats per URL .., we should do it per domain instead in sitespeed.io
+      if (data.cpu.categories) {
+        statistics.addDeep({
+          cpu: {
+            categories: data.cpu.categories,
+            events: data.cpu.events
+          }
+        });
       }
-      if (this.options.chrome && this.options.chrome.collectConsoleLog) {
-        results.files.consoleLog.push(
-          `${pathToFolder(url, this.options)}console-${index}.json.gz`
-        );
+      if (data.cpu.longTasks) {
+        statistics.addDeep({
+          cpu: {
+            longTasks: data.cpu.longTasks
+          }
+        });
       }
+    }
 
-      if (this.options.chrome && this.options.chrome.collectNetLog) {
-        results.files.netLog.push(
-          `${pathToFolder(url, this.options)}chromeNetlog-${index}.json.gz`
-        );
-      }
+    if (data.googleWebVitals) {
+      results.googleWebVitals.push(data.googleWebVitals);
+      statistics.addDeep({
+        googleWebVitals: data.googleWebVitals
+      });
+    }
 
-      if (this.options.chrome && this.options.chrome.collectPerfLog) {
-        results.files.perfLog.push(
-          `${pathToFolder(url, this.options)}chromePerflog-${index}.json.gz`
-        );
+    if (data.renderBlocking) {
+      results.renderBlocking.push(data.renderBlocking);
+
+      statistics.addDeep({
+        renderBlocking: {
+          recalculateStyle: data.renderBlocking.recalculateStyle
+        }
+      });
+    }
+
+    // Add power data (if we have it)
+    if (data.power) {
+      results.android.power.push(data.power);
+      statistics.addDeep({
+        android: { power: data.power }
+      });
+    }
+
+    // Add Firefox perfStats if available
+    if (this.options.firefox && this.options.firefox.perfStats) {
+      if (!results.geckoPerfStats) {
+        results.geckoPerfStats = [];
       }
+      results.geckoPerfStats.push(data.perfStats);
+    }
 
-      if (this.options.firefox && this.options.firefox.geckoProfiler) {
-        const name = this.options.enableProfileRun
-          ? `geckoProfile-${index}-extra.json.gz`
-          : `geckoProfile-${index}.json.gz`;
-        results.files.geckoProfiles.push(
-          `${pathToFolder(url, this.options)}${name}`
-        );
+    // Add Firefox cpu power
+    if (data.powerConsumption) {
+      if (!results.powerConsumption) {
+        results.powerConsumption = [];
       }
+      results.powerConsumption.push(data.powerConsumption);
+      statistics.addDeep({
+        powerConsumption: data.powerConsumption
+      });
+    }
 
-      if (this.options.firefox && this.options.firefox.memoryReport) {
-        results.files.memoryReports.push(
-          `${pathToFolder(url, this.options)}memory-report-${index}.json.gz`
-        );
+    // Add total memory
+    if (this.options.firefox && this.options.firefox.memoryReport) {
+      statistics.addDeep({
+        memory: data.memory
+      });
+
+      if (!results.memory) {
+        results.memory = [];
       }
+      results.memory.push(data.memory);
+    }
+  }
 
-      await Promise.all(extraWork);
-      this.allStats[url] = statistics;
-      this.allResults[url] = results;
+  _computeDeltaToTTFB(data, results, statistics) {
+    // Add delta to TTFB
+    const deltaToTTFB = {};
+    const fcp = getProperty(
+      data,
+      'browserScripts.timings.paintTiming.first-contentful-paint'
+    );
+    const ttfb = getProperty(data, 'browserScripts.timings.ttfb');
+    const lcp = getProperty(
+      data,
+      'browserScripts.timings.largestContentfulPaint.renderTime'
+    );
+    const firstVisualChange = getProperty(
+      data,
+      'visualMetrics.FirstVisualChange'
+    );
+    const lastVisualChange = getProperty(
+      data,
+      'visualMetrics.LastVisualChange'
+    );
+
+    if (fcp) {
+      deltaToTTFB['firstContentfulPaint'] = fcp - ttfb;
+    }
+    if (lcp) {
+      deltaToTTFB['largestContentfulPaint'] = lcp - ttfb;
+    }
+    if (firstVisualChange) {
+      deltaToTTFB['firstVisualChange'] = firstVisualChange - ttfb;
+    }
+    if (lastVisualChange) {
+      deltaToTTFB['lastVisualChange'] = lastVisualChange - ttfb;
+    }
+    results.deltaToTTFB.push(deltaToTTFB);
+    statistics.addDeep({ deltaToTFFB: deltaToTTFB });
+  }
+
+  _collectFilePaths(url, index, results) {
+    if (this.options.video) {
+      results.files.video.push(
+        `${pathToFolder(url, this.options)}video/${index}.mp4`
+      );
+    }
+    if (this.options.chrome && this.options.chrome.timeline) {
+      results.files.timeline.push(
+        `${pathToFolder(url, this.options)}trace-${index}.json.gz`
+      );
+    }
+    if (this.options.chrome && this.options.chrome.collectConsoleLog) {
+      results.files.consoleLog.push(
+        `${pathToFolder(url, this.options)}console-${index}.json.gz`
+      );
+    }
+
+    if (this.options.chrome && this.options.chrome.collectNetLog) {
+      results.files.netLog.push(
+        `${pathToFolder(url, this.options)}chromeNetlog-${index}.json.gz`
+      );
+    }
+
+    if (this.options.chrome && this.options.chrome.collectPerfLog) {
+      results.files.perfLog.push(
+        `${pathToFolder(url, this.options)}chromePerflog-${index}.json.gz`
+      );
+    }
+
+    if (this.options.firefox && this.options.firefox.geckoProfiler) {
+      const name = this.options.enableProfileRun
+        ? `geckoProfile-${index}-extra.json.gz`
+        : `geckoProfile-${index}.json.gz`;
+      results.files.geckoProfiles.push(
+        `${pathToFolder(url, this.options)}${name}`
+      );
+    }
+
+    if (this.options.firefox && this.options.firefox.memoryReport) {
+      results.files.memoryReports.push(
+        `${pathToFolder(url, this.options)}memory-report-${index}.json.gz`
+      );
     }
   }
 }