browsertime 17.8.0 → 17.9.0

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Browsertime changelog (we do [semantic versioning](https://semver.org))
 
+## 17.9.0 - 2022-04-19
+### Added
+* New command to start geckoprofiling when scripting using Firefox. Thank you [KS](https://github.com/92kns) for PR [#1934](https://github.com/sitespeedio/browsertime/pull/1934). This is probably only something you need if you work at Mozilla. There's also a follow PR [#1940](https://github.com/sitespeedio/browsertime/pull/1940) that fixes some of the issues I've been wanted to fix with the Geckoprofiler code.
+
+* Use `--enableProfileRun` to make one extra run where metrics aren't collected but Geckoprofiler or Chromes timeline log is turned on. This useful if you are worrying about the trace causing overhead, but you still wan trace from the test/journey. It's a little rough at the moment so it will a follow up in the future [#1943](https://github.com/sitespeedio/browsertime/pull/1943).
+
+## 17.8.1 - 2022-04-12
+### Fixed
+* Upgraded to Firefox HAR 0.0.10 [#1939](https://github.com/sitespeedio/browsertime/pull/1939). 
 ## 17.8.0 - 2022-04-11
 ### Fixed
 * Upgraded to Firefox HAR 0.0.8 [#1937](https://github.com/sitespeedio/browsertime/pull/1937). 

package/bin/browsertime.js

@@ -122,6 +122,28 @@ async function run(urls, options) {
           storageManager.writeJson(harName + '.har', result.har, useGzip)
         );
       }
+
+      if (options.enableProfileRun) {
+        log.info('Make one extra run to collect trace information');
+        options.iterations = 1;
+        if (options.browser === 'firefox') {
+          options.firefox.geckoProfiler = true;
+        } else if (options.browser === 'chrome') {
+          options.chrome.timeline = true;
+          options.cpu = true;
+          options.chrome.enableTraceScreenshots = true;
+          options.chrome.traceCategory = [
+            'disabled-by-default-v8.cpu_profiler'
+          ];
+        }
+        options.video = false;
+        options.visualMetrics = false;
+        const traceEngine = new Engine(options);
+        await traceEngine.start();
+        await traceEngine.runMultiple(urls, scriptsByCategory);
+        await traceEngine.stop();
+      }
+
       await Promise.all(saveOperations);
 
       const resultDirectory = relative(process.cwd(), storageManager.directory);

package/lib/chrome/webdriver/chromium.js

@@ -331,7 +331,11 @@ export class Chromium {
 
     if (this.collectTracingEvents) {
       const trace = parse(this.events, result.url);
-      result.extraJson[`trace-${index}.json`] = trace;
+      const name = this.options.enableProfileRun
+        ? `trace-${index}-extra-run.json`
+        : `trace-${index}.json`;
+      result.extraJson[name] = trace;
+
       const cpu = await parseCPUTrace(trace, result.url);
       result.cpu = cpu;
 

package/lib/core/engine/collector.js

@@ -503,8 +503,11 @@ export class Collector {
       }
 
       if (this.options.firefox && this.options.firefox.geckoProfiler) {
+        const name = this.options.enableProfileRun
+          ? `geckoProfile-${index}-extra.json.gz`
+          : `geckoProfile-${index}.json.gz`;
         results.files.geckoProfiles.push(
-          `${pathToFolder(url, this.options)}geckoProfile-${index}.json.gz`
+          `${pathToFolder(url, this.options)}${name}`
         );
       }
 

package/lib/core/engine/command/geckoProfiler.js

@@ -0,0 +1,38 @@
+import intel from 'intel';
+const log = intel.getLogger('browsertime.command.geckoprofiler');
+export class GeckoProfiler {
+  constructor(GeckoProfiler, browser, index, options) {
+    this.GeckoProfiler = GeckoProfiler;
+    this.browser = browser;
+    this.index = index;
+    this.options = options;
+  }
+
+  async start() {
+    if (this.options.browser === 'firefox') {
+      if (this.options.firefox.geckoProfilerRecordingType === 'custom') {
+        return this.GeckoProfiler.start();
+      } else {
+        log.info(
+          'You need to set geckoProfilerRecordingType to custom to turn on the profiler in scripting'
+        );
+      }
+    } else {
+      throw new Error('Geckoprofiler only works in Firefox');
+    }
+  }
+
+  async stop() {
+    if (this.options.browser === 'firefox') {
+      if (this.options.firefox.geckoProfilerRecordingType === 'custom') {
+        const url = await this.browser
+          .getDriver()
+          .executeScript('return document.documentURI;');
+
+        return this.GeckoProfiler.stop(this.index, url);
+      }
+    } else {
+      throw new Error('Geckoprofiler only works in Firefox');
+    }
+  }
+}

package/lib/core/engine/index.js

@@ -381,7 +381,9 @@ export class Engine {
       }
     }
 
-    logResultLogLine(totalResult);
+    if (!options.enableProfileRun) {
+      logResultLogLine(totalResult);
+    }
 
     if (failures.length > 0) {
       // If we have a result

package/lib/core/engine/iteration.js

@@ -41,6 +41,8 @@ import { getNumberOfRunningProcesses } from '../../support/processes.js';
 import { isAndroidConfigured, Android } from '../../android/index.js';
 import { Scroll } from './command/scroll.js';
 import { Navigation } from './command/navigation.js';
+import { GeckoProfiler } from '../../firefox/geckoProfiler.js';
+import { GeckoProfiler as GeckoProfilerCommand } from './command/geckoProfiler.js';
 const log = intel.getLogger('browsertime');
 const delay = ms => new Promise(res => setTimeout(res, ms));
 
@@ -147,6 +149,17 @@ export class Iteration {
         options
       );
 
+      const browserProfiler = new GeckoProfiler(
+        browser,
+        this.storageManager,
+        this.options
+      );
+      const profiler = new GeckoProfilerCommand(
+        browserProfiler,
+        browser,
+        index,
+        this.options
+      );
       const cdp = new ChromeDevelopmentToolsProtocol(
         engineDelegate,
         options.browser
@@ -154,6 +167,7 @@ export class Iteration {
       const android = new Android(options);
       const debug = new Debug(browser, options);
       const commands = {
+        profiler: profiler,
         click: new Click(browser, this.pageCompleteCheck),
         scroll: new Scroll(browser, options),
         addText: new AddText(browser),

package/lib/firefox/geckoProfiler.js

@@ -1,10 +1,7 @@
 import { join } from 'node:path';
 import get from 'lodash.get';
 import intel from 'intel';
-import {
-  android_sampling_interval,
-  desktop_sampling_interval
-} from './settings/geckoProfilerDefaults.js';
+import { geckoProfilerDefaults } from './settings/geckoProfilerDefaults.js';
 import { isAndroidConfigured, Android } from '../android/index.js';
 import { pathToFolder } from '../support/pathToFolder.js';
 import { BrowserError } from '../support/errors.js';
@@ -33,10 +30,10 @@ async function timeout(promise, ms, errorMessage) {
   ]);
 }
 export class GeckoProfiler {
-  constructor(runner, storageManager, firefoxConfig, options) {
+  constructor(runner, storageManager, options) {
     this.runner = runner;
     this.storageManager = storageManager;
-    this.firefoxConfig = firefoxConfig;
+    this.firefoxConfig = options.firefox;
     this.options = options;
   }
 
@@ -44,14 +41,26 @@ export class GeckoProfiler {
     const runner = this.runner;
     const firefoxConfig = this.firefoxConfig;
     const options = this.options;
-    const chosenFeatures =
-      firefoxConfig.geckoProfilerParams.features.split(',');
+    const chosenFeatures = get(
+      firefoxConfig,
+      'geckoProfilerParams.features',
+      geckoProfilerDefaults.features
+    ).split(',');
+
     const featureString = '["' + chosenFeatures.join('","') + '"]';
 
-    const chosenThreads = firefoxConfig.geckoProfilerParams.threads.split(',');
+    const chosenThreads = get(
+      firefoxConfig,
+      'geckoProfilerParams.threads',
+      geckoProfilerDefaults.threads
+    ).split(',');
     const threadString = '["' + chosenThreads.join('","') + '"]';
 
-    let interval = firefoxConfig.geckoProfilerParams.interval;
+    let interval = get(
+      firefoxConfig,
+      'geckoProfilerParams.interval',
+      geckoProfilerDefaults.interval
+    );
 
     // Set platform specific sampling intervals if not explicitly specified.
     if (
@@ -59,11 +68,15 @@ export class GeckoProfiler {
       !firefoxConfig.geckoProfilerParams.interval
     ) {
       interval = isAndroidConfigured(options)
-        ? android_sampling_interval
-        : desktop_sampling_interval;
+        ? geckoProfilerDefaults.androidSamplingInterval
+        : geckoProfilerDefaults.desktopSamplingInterval;
     }
 
-    const bufferSize = firefoxConfig.geckoProfilerParams.bufferSize;
+    const bufferSize = get(
+      firefoxConfig,
+      'geckoProfilerParams.bufferSize',
+      geckoProfilerDefaults.bufferSize
+    );
 
     // Firefox 69 and above has a slightly different API for StartProfiler
     // See nsIProfiler::StartProfiler in tools/profiler/gecko/nsIProfiler.idl
@@ -121,6 +134,7 @@ export class GeckoProfiler {
     const runner = this.runner;
     const storageManager = this.storageManager;
     const options = this.options;
+
     let profileDir = await storageManager.createSubDataDir(
       join(pathToFolder(url, options))
     );
@@ -171,12 +185,11 @@ export class GeckoProfiler {
 
       // GZIP the profile and remove the old file
       log.info('Gzip file the profile.');
+      const name = this.options.enableProfileRun
+        ? `geckoProfile-${index}-extra.json.gz`
+        : `geckoProfile-${index}.json.gz`;
       await timeout(
-        storageManager.gzip(
-          destinationFilename,
-          join(profileDir, `geckoProfile-${index}.json.gz`),
-          true
-        ),
+        storageManager.gzip(destinationFilename, join(profileDir, name), true),
         300_000, // 5 minutes
         'Could not gzip the profile.'
       );

package/lib/firefox/settings/geckoProfilerDefaults.js

@@ -1,5 +1,7 @@
-export const features = 'js,stackwalk,leaf';
-export const threads = 'GeckoMain,Compositor,Renderer';
-export const desktop_sampling_interval = 1;
-export const android_sampling_interval = 4;
-export const bufferSize = 13_107_200; // 100MB;
+export const geckoProfilerDefaults = {
+  features: 'js,stackwalk,leaf',
+  threads: 'GeckoMain,Compositor,Renderer',
+  desktopSamplingInterval: 1,
+  androidSamplingInterval: 4,
+  bufferSize: 13_107_200 // 100MB
+};

package/lib/firefox/webdriver/firefox.js

@@ -101,11 +101,13 @@ export class Firefox {
       await this.android.resetPowerUsage();
     }
 
-    if (this.firefoxConfig.geckoProfiler) {
+    if (
+      this.firefoxConfig.geckoProfiler &&
+      this.firefoxConfig.geckoProfilerRecordingType !== 'custom'
+    ) {
       this.geckoProfiler = new GeckoProfiler(
         runner,
         this.storageManager,
-        this.firefoxConfig,
         this.options
       );
 
@@ -133,7 +135,10 @@ export class Firefox {
       );
     }
 
-    if (this.firefoxConfig.geckoProfiler) {
+    if (
+      this.firefoxConfig.geckoProfiler &&
+      this.firefoxConfig.geckoProfilerRecordingType !== 'custom'
+    ) {
       await this.geckoProfiler.stop(index, url);
     }
 

package/lib/support/cli.js

@@ -14,13 +14,7 @@ import {
   xvfbDisplay
 } from '../video/defaults.js';
 import { screenshotDefaults } from '../screenshot/defaults.js';
-import {
-  features,
-  threads as _threads,
-  desktop_sampling_interval,
-  android_sampling_interval,
-  bufferSize
-} from '../firefox/settings/geckoProfilerDefaults.js';
+import { geckoProfilerDefaults } from '../firefox/settings/geckoProfilerDefaults.js';
 
 const configPath = findUpSync(['.browsertime.json']);
 
@@ -472,26 +466,33 @@ export function parseCommandLine() {
       type: 'boolean',
       group: 'firefox'
     })
+    .option('firefox.geckoProfilerRecordingType', {
+      describe: 'Expose the start/stop commands for the gecko profiler',
+      default: 'pageload',
+      choices: ['pageload', 'custom'],
+      type: 'string',
+      group: 'firefox'
+    })
     .option('firefox.geckoProfilerParams.features', {
       describe: 'Enabled features during gecko profiling',
-      default: features,
+      default: geckoProfilerDefaults.features,
       type: 'string',
       group: 'firefox'
     })
     .option('firefox.geckoProfilerParams.threads', {
       describe: 'Threads to profile.',
-      default: _threads,
+      default: geckoProfilerDefaults.threads,
       type: 'string',
       group: 'firefox'
     })
     .option('firefox.geckoProfilerParams.interval', {
-      describe: `Sampling interval in ms.  Defaults to ${desktop_sampling_interval} on desktop, and ${android_sampling_interval} on android.`,
+      describe: `Sampling interval in ms.  Defaults to ${geckoProfilerDefaults.desktopSamplingInterval} on desktop, and ${geckoProfilerDefaults.androidSamplingInterval} on android.`,
       type: 'number',
       group: 'firefox'
     })
     .option('firefox.geckoProfilerParams.bufferSize', {
       describe: 'Buffer size in elements. Default is ~90MB.',
-      default: bufferSize,
+      default: geckoProfilerDefaults.bufferSize,
       type: 'number',
       group: 'firefox'
     })
@@ -578,6 +579,11 @@ export function parseCommandLine() {
         'URL to a running Selenium server (e.g. to run a browser on another machine).',
       group: 'selenium'
     })
+    .option('enableProfileRun', {
+      type: 'boolean',
+      describe:
+        'Make one extra run that collects the profiling trace log (no other metrics is collected). For Chrome it will collect the timeline trace, for Firefox it will get the Geckoprofiler trace. This means you do not need to get the trace for all runs and can skip the overhead it produces.'
+    })
     .option('video', {
       type: 'boolean',
       describe:

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "browsertime",
   "description": "Get performance metrics from your web page using Browsertime.",
-  "version": "17.8.0",
+  "version": "17.9.0",
   "bin": "./bin/browsertime.js",
   "type": "module",
   "dependencies": {
@@ -12,12 +12,12 @@
     "@sitespeed.io/geckodriver": "0.33.0",
     "@sitespeed.io/throttle": "5.0.0",
     "@sitespeed.io/tracium": "0.3.3",
-    "ff-test-bidi-har-export": "0.0.8",
+    "ff-test-bidi-har-export": "0.0.10",
     "btoa": "1.2.1",
     "chrome-har": "0.13.1",
-    "chrome-remote-interface": "0.32.1",
+    "chrome-remote-interface": "0.32.2",
     "dayjs": "1.11.7",
-    "execa": "7.0.0",
+    "execa": "7.1.1",
     "fast-stats": "0.0.6",
     "find-up": "6.3.0",
     "get-port": "6.1.2",