@jupyterlab/galata 4.3.0-alpha.9 → 4.3.0-beta.0

package/README.md

@@ -26,7 +26,7 @@ jlpm playwright install
 
 Create a Playwright configuration file `playwright.config.js` containing:
 
-```
+```js
 module.exports = require('@jupyterlab/galata/lib/playwright-config');
 ```
 
@@ -35,8 +35,7 @@ module.exports = require('@jupyterlab/galata/lib/playwright-config');
 Create `ui-tests/foo.spec.ts` to define your test.
 
 ```typescript
-import { test } from '@jupyterlab/galata';
-import { expect } from '@playwright/test';
+import { expect, test } from '@jupyterlab/galata';
 
 test.describe('Notebook Tests', () => {
   test('Create New Notebook', async ({ page, tmpPath }) => {
@@ -66,8 +65,8 @@ Create `jupyter_server_test_config.py` with the following content.
 from tempfile import mkdtemp
 
 c.ServerApp.port = 8888
+c.ServerApp.port_retries = 0
 c.ServerApp.open_browser = False
-c.LabApp.dev_mode = True
 
 c.ServerApp.root_dir = mkdtemp(prefix='galata-test-')
 c.ServerApp.token = ""
@@ -137,6 +136,13 @@ PWDEBUG=1 jlpm playwright test
 
 Here are the new test fixture introduced by Galata on top of [Playwright fixtures](https://playwright.dev/docs/api/class-fixtures).
 
+### baseURL
+
+- type: < string >
+
+Application base URL without `/lab`. It defaults to environment variable `TARGET_URL` or `http://localhost:8888` if nothing
+is defined.
+
 ### appPath
 
 - type: < string >
@@ -379,7 +385,7 @@ test.describe.serial('Table of Contents', () => {
 
 ## Benchmark
 
-Benchmark of JupyterLab is done automatically using Playwright. The actions measured are:
+Benchmark of JupyterLab is done using Playwright. The actions measured are:
 
 - Opening a file
 - Switching from the file to a simple text file
@@ -388,6 +394,13 @@ Benchmark of JupyterLab is done automatically using Playwright. The actions meas
 
 Two files are tested: a notebook with many code cells and another with many markdown cells.
 
+The test is run on the CI by comparing the result in the commit at which a PR branch started and the PR branch head on
+the same CI job to ensure using the same hardware.  
+The benchmark job is triggered on:
+
+- Approved PR review
+- PR review that contains the sentence `please run benchmark`
+
 The tests are located in the subfolder [test/benchmark](./test/benchmark). And they can be
 executed with the following command:
 
@@ -402,7 +415,7 @@ A special report will be generated in the folder `benchmark-results` that will c
 - `lab-benchmark.png`: A comparison of execution time distribution
 - `lab-benchmark.vl.json`: The [_Vega-Lite_](https://vega.github.io/vega-lite) description used to produce the PNG file.
 
-The reference, tagged _expected_, is stored in [`lab-benchmark-expected.json`](lab-benchmark-expected.json). It can be
+The reference, tagged _expected_, is stored in `lab-benchmark-expected.json`. It can be
 updated using the `-u` option of Playwright; i.e. `jlpm run test:benchmark -u`.
 
 ### Benchmark parameters
@@ -442,6 +455,41 @@ By default, both projects will be executed when running `jlpm run test`. But you
 
 Galata can be configured by using [command line arguments](https://playwright.dev/docs/cli) or using [`playwright.config.js` file](https://playwright.dev/docs/test-configuration). Full list of config options can be accessed using `jlpm playwright test --help`.
 
+### Custom benchmark report
+
+By default, Galata will generate a text report in the form of `markdown` table and a [_Vega-Lite_](https://vega.github.io/vega-lite) graph of execution time distribution. Users can customize these reports in two ways:
+
+- Using `playwright.config.js` file: in `reporter` section, users can supply two functions `vegaLiteConfigFactory` and `textReportFactory` to the reporter's constructor options. These functions will be used to create [Vega-Lite configuration](https://vega.github.io/vega-lite/docs/) (`vegaLiteConfigFactory`) or to create a text report (`textReportFactory`) from test records.
+
+```javascript
+  // An example of `playwright.config.js` with customized builder
+  reporter: [
+    ...,
+    [
+      '@jupyterlab/galata/lib/benchmarkReporter',
+      { outputFile: 'lab-benchmark.json',
+        vegaLiteConfigFactory: (
+          allData: Array<IReportRecord>, // All test records
+          comparison?: 'snapshot' | 'project'// Logic of test comparisons:'snapshot' or 'project' - default 'snapshot'.
+        ) => {
+          // Return a Vega-Lite graph configuration object
+          return {};
+        }
+        textReportFactory: (
+          allData: Array<IReportRecord>, // All test records
+          comparison?: 'snapshot' | 'project'// Logic of test comparisons:'snapshot' or 'project' - default 'snapshot'.
+        ) => {
+          // Return a promise of with the tuple [report content, file extension]
+          return Promise.resolve(['My report content', 'md']);
+        }
+      }
+    ],
+    ...
+  ]
+```
+
+- The second way to customize the reports is to override the default text report factory (`defaultTextReportFactory`) and Vega-Lite graph config factory (`defaultVegaLiteConfigFactory`) of `BenchmarkReporter` class in a sub-class and then use it as a reporter in `playwright.config.js` file.
+
 ## Reference Image Captures
 
 Reference image are saved next to test files in `<test-file-name>-snapshots` folders. If a reference screenshots does not exist, it will be generated at the first execution

package/lib/benchmarkReporter.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" />
+import { JSONObject } from '@lumino/coreutils';
 import { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter';
 /**
  * Benchmark namespace
@@ -89,11 +90,51 @@ export declare namespace benchmark {
      * @returns The attachment
      */
     function addAttachment<IRecord>(data: IRecord): IAttachment;
+    /**
+     * Change between two distributions
+     */
+    interface IDistributionChange {
+        /**
+         * Mean value
+         */
+        mean: number;
+        /**
+         * Spread around the mean value
+         */
+        confidenceInterval: number;
+    }
+    /**
+     * Compute the performance change based on a number of old and new measurements.
+     *
+     * Based on the work by Tomas Kalibera and Richard Jones. See their paper
+     * "Quantifying Performance Changes with Effect Size Confidence Intervals", section 6.2,
+     * formula "Quantifying Performance Change".
+     *
+     * However, it simplifies it to only assume one level of benchmarks, not multiple levels.
+     * If you do have multiple levels, simply use the mean of the lower levels as your data,
+     * like they do in the paper.
+     *
+     * Note: The measurements must have the same length. As fallback, you could use the minimum
+     * size of the two measurement sets.
+     *
+     * @param oldMeasures The old measurements
+     * @param newMeasures The new measurements
+     * @param confidenceInterval The confidence interval for the results.
+     * @param minLength Fall back to the minimum length of the two arrays
+     */
+    function distributionChange(oldMeasures: number[], newMeasures: number[], confidenceInterval?: number, minLength?: boolean): IDistributionChange;
+    /**
+     * Format a performance changes like `between 20.1% slower and 30.3% faster`.
+     *
+     * @param distribution The distribution change
+     * @returns The formatted distribution change
+     */
+    function formatChange(distribution: IDistributionChange): string;
 }
 /**
  * Report record interface
  */
-interface IReportRecord extends benchmark.IRecord {
+export interface IReportRecord extends benchmark.IRecord {
     /**
      * Test suite reference
      */
@@ -144,14 +185,19 @@ interface IReport {
 declare class BenchmarkReporter implements Reporter {
     /**
      * @param options
-     *   outputFile: Name of the output file (default to env BENCHMARK_OUTPUTFILE)
-     *   comparison: Logic of test comparisons: 'snapshot' or 'project'
-     *    - 'snapshot': (default) This will compare the 'actual' result with the 'expected' one
-     *    - 'project': This will compare the different project
+     *   - outputFile: Name of the output file (default to env BENCHMARK_OUTPUTFILE)
+     *   - comparison: Logic of test comparisons: 'snapshot' or 'project'
+     *      * 'snapshot': (default) This will compare the 'actual' result with the 'expected' one
+     *      * 'project': This will compare the different project
+     *   - vegaLiteConfigFactory: Function to create VegaLite configuration from test records; see https://vega.github.io/vega-lite/docs/.
+     *   - textReportFactory: Function to create  text report from test records, this function
+     *   should return the content and extension of report file.
      */
     constructor(options?: {
         outputFile?: string;
         comparison?: 'snapshot' | 'project';
+        vegaLiteConfigFactory?: (allData: Array<IReportRecord>, comparison?: 'snapshot' | 'project') => JSONObject;
+        textReportFactory?: (allData: Array<IReportRecord>, comparison?: 'snapshot' | 'project') => Promise<[string, string]>;
     });
     /**
      * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s.
@@ -176,6 +222,31 @@ declare class BenchmarkReporter implements Reporter {
      * - `'interrupted'` - Interrupted by the user.
      */
     onEnd(result: FullResult): Promise<void>;
+    /**
+     * Default text report factory of `BenchmarkReporter`, this method will
+     * be used by to generate markdown report. Users can customize the builder
+     * by supplying another builder to constructor's option or override this
+     * method on a sub-class.
+     *
+     * @param allData all test records.
+     * @param comparison logic of test comparisons:
+     * 'snapshot' or 'project'; default 'snapshot'.
+     * @return A list of two strings, the first one
+     * is the content of report, the second one is the extension of report file.
+     */
+    protected defaultTextReportFactory(allData: Array<IReportRecord>, comparison?: 'snapshot' | 'project'): Promise<[string, string]>;
+    /**
+     * Default Vega Lite config factory of `BenchmarkReporter`, this method will
+     * be used by to generate VegaLite configuration. Users can customize
+     * the builder by supplying another builder to constructor's option or
+     * override this method on a sub-class.
+     *
+     * @param allData all test records.
+     * @param comparison logic of test comparisons:
+     * 'snapshot' or 'project'; default 'snapshot'.
+     * @return VegaLite configuration
+     */
+    protected defaultVegaLiteConfigFactory(allData: Array<IReportRecord>, comparison?: 'snapshot' | 'project'): Record<string, any>;
     protected getMetadata(browser?: string): Promise<any>;
     protected buildReport(): Promise<IReport>;
     protected config: FullConfig;
@@ -184,5 +255,7 @@ declare class BenchmarkReporter implements Reporter {
     private _outputFile;
     private _reference;
     private _report;
+    private _buildVegaLiteGraph;
+    private _buildTextReport;
 }
 export default BenchmarkReporter;

package/lib/benchmarkReporter.js

@@ -35,6 +35,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.benchmark = void 0;
 const test_1 = require("@playwright/test");
+const base_1 = require("@stdlib/stats/base");
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const systeminformation_1 = __importDefault(require("systeminformation"));
@@ -91,6 +92,92 @@ var benchmark;
         };
     }
     benchmark.addAttachment = addAttachment;
+    /**
+     * Quantifies the performance changes between two measures systems. Assumes we gathered
+     * n independent measurement from each, and calculated their means and variance.
+     *
+     * Based on the work by Tomas Kalibera and Richard Jones. See their paper
+     * "Quantifying Performance Changes with Effect Size Confidence Intervals", section 6.2,
+     * formula "Quantifying Performance Change".
+     *
+     * However, it simplifies it to only assume one level of benchmarks, not multiple levels.
+     * If you do have multiple levels, simply use the mean of the lower levels as your data,
+     * like they do in the paper.
+     *
+     * @param oldDistribution The old distribution description
+     * @param newDistribution The new distribution description
+     * @param n The number of samples from each system (must be equal)
+     * @param confidenceInterval The confidence interval for the results.
+     *  The default is a 95% confidence interval (95% of the time the true mean will be
+     *  between the resulting mean +- the resulting CI)
+     */
+    function performanceChange(oldDistribution, newDistribution, n, confidenceInterval = 0.95) {
+        const { mean: yO, variance: sO } = oldDistribution;
+        const { mean: yN, variance: sN } = newDistribution;
+        const dof = n - 1;
+        const t = base_1.dists.t.quantile(1 - (1 - confidenceInterval) / 2, dof);
+        const oldFactor = sq(yO) - (sq(t) * sO) / n;
+        const newFactor = sq(yN) - (sq(t) * sN) / n;
+        const meanNum = yO * yN;
+        const ciNum = Math.sqrt(sq(yO * yN) - newFactor * oldFactor);
+        return {
+            mean: meanNum / oldFactor,
+            confidenceInterval: ciNum / oldFactor
+        };
+    }
+    /**
+     * Compute the performance change based on a number of old and new measurements.
+     *
+     * Based on the work by Tomas Kalibera and Richard Jones. See their paper
+     * "Quantifying Performance Changes with Effect Size Confidence Intervals", section 6.2,
+     * formula "Quantifying Performance Change".
+     *
+     * However, it simplifies it to only assume one level of benchmarks, not multiple levels.
+     * If you do have multiple levels, simply use the mean of the lower levels as your data,
+     * like they do in the paper.
+     *
+     * Note: The measurements must have the same length. As fallback, you could use the minimum
+     * size of the two measurement sets.
+     *
+     * @param oldMeasures The old measurements
+     * @param newMeasures The new measurements
+     * @param confidenceInterval The confidence interval for the results.
+     * @param minLength Fall back to the minimum length of the two arrays
+     */
+    function distributionChange(oldMeasures, newMeasures, confidenceInterval = 0.95, minLength = false) {
+        const n = oldMeasures.length;
+        if (!minLength && n !== newMeasures.length) {
+            throw new Error('Data have different length');
+        }
+        return performanceChange({ mean: mean(...oldMeasures), variance: variance(...oldMeasures) }, { mean: mean(...newMeasures), variance: variance(...newMeasures) }, minLength ? Math.min(n, newMeasures.length) : n, confidenceInterval);
+    }
+    benchmark.distributionChange = distributionChange;
+    /**
+     * Format a performance changes like `between 20.1% slower and 30.3% faster`.
+     *
+     * @param distribution The distribution change
+     * @returns The formatted distribution change
+     */
+    function formatChange(distribution) {
+        const { mean, confidenceInterval } = distribution;
+        return `between ${formatPercent(mean + confidenceInterval)} and ${formatPercent(mean - confidenceInterval)}`;
+    }
+    benchmark.formatChange = formatChange;
+    function formatPercent(percent) {
+        if (percent < 1) {
+            return `${((1 - percent) * 100).toFixed(1)}% faster`;
+        }
+        return `${((percent - 1) * 100).toFixed(1)}% slower`;
+    }
+    function sq(x) {
+        return Math.pow(x, 2);
+    }
+    function mean(...x) {
+        return base_1.meanpw(x.length, x, 1);
+    }
+    function variance(...x) {
+        return base_1.variancepn(x.length, 1, x, 1);
+    }
 })(benchmark = exports.benchmark || (exports.benchmark = {}));
 /**
  * Custom Playwright reporter for benchmark tests
@@ -98,16 +185,21 @@ var benchmark;
 class BenchmarkReporter {
     /**
      * @param options
-     *   outputFile: Name of the output file (default to env BENCHMARK_OUTPUTFILE)
-     *   comparison: Logic of test comparisons: 'snapshot' or 'project'
-     *    - 'snapshot': (default) This will compare the 'actual' result with the 'expected' one
-     *    - 'project': This will compare the different project
+     *   - outputFile: Name of the output file (default to env BENCHMARK_OUTPUTFILE)
+     *   - comparison: Logic of test comparisons: 'snapshot' or 'project'
+     *      * 'snapshot': (default) This will compare the 'actual' result with the 'expected' one
+     *      * 'project': This will compare the different project
+     *   - vegaLiteConfigFactory: Function to create VegaLite configuration from test records; see https://vega.github.io/vega-lite/docs/.
+     *   - textReportFactory: Function to create  text report from test records, this function
+     *   should return the content and extension of report file.
      */
     constructor(options = {}) {
-        var _a, _b, _c, _d;
+        var _a, _b, _c, _d, _e, _f;
         this._outputFile = (_b = (_a = options.outputFile) !== null && _a !== void 0 ? _a : process.env['BENCHMARK_OUTPUTFILE']) !== null && _b !== void 0 ? _b : benchmark.DEFAULT_OUTPUT;
         this._comparison = (_c = options.comparison) !== null && _c !== void 0 ? _c : 'snapshot';
         this._reference = (_d = process.env['BENCHMARK_REFERENCE']) !== null && _d !== void 0 ? _d : benchmark.DEFAULT_REFERENCE;
+        this._buildVegaLiteGraph = (_e = options.vegaLiteConfigFactory) !== null && _e !== void 0 ? _e : this.defaultVegaLiteConfigFactory;
+        this._buildTextReport = (_f = options.textReportFactory) !== null && _f !== void 0 ? _f : this.defaultTextReportFactory;
     }
     /**
      * Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s.
@@ -183,63 +275,14 @@ class BenchmarkReporter {
                 }
                 allData.push(...expectations.values);
             }
-            // Create graph file
+            // - Create report
+            const [reportContentString, reportExtension] = await this._buildTextReport(allData);
+            const reportFile = path_1.default.resolve(outputDir, `${baseName}.${reportExtension}`);
+            fs_1.default.writeFileSync(reportFile, reportContentString, 'utf-8');
+            // Generate graph file and image
             const graphConfigFile = path_1.default.resolve(outputDir, `${baseName}.vl.json`);
-            const config = benchmarkVLTpl_1.default([...new Set(allData.map(d => d.test))], this._comparison == 'snapshot' ? 'reference' : 'project', [...new Set(allData.map(d => d.file))]);
-            config.data.values = allData;
+            const config = this._buildVegaLiteGraph(allData, this._comparison);
             fs_1.default.writeFileSync(graphConfigFile, JSON.stringify(config), 'utf-8');
-            // Compute statistics
-            // - Groupby (test, browser, reference, file)
-            const groups = new Map();
-            allData.forEach(d => {
-                var _a;
-                if (!groups.has(d.test)) {
-                    groups.set(d.test, new Map());
-                }
-                const testGroup = groups.get(d.test);
-                if (!testGroup.has(d.browser)) {
-                    testGroup.set(d.browser, new Map());
-                }
-                const browserGroup = testGroup.get(d.browser);
-                if (!browserGroup.has(d.reference)) {
-                    browserGroup.set(d.reference, new Map());
-                }
-                const fileGroup = browserGroup.get(d.reference);
-                if (!fileGroup.has(d.file)) {
-                    fileGroup.set(d.file, new Array());
-                }
-                (_a = fileGroup.get(d.file)) === null || _a === void 0 ? void 0 : _a.push(d.time);
-            });
-            // - Create report
-            const reportContent = new Array('## Benchmark report', '', 'The execution time (in milliseconds) are grouped by test file, test type and browser.', 'For each case, the following values are computed: _min_ <- [_1st quartile_ - _median_ - _3rd quartile_] -> _max_.', '', '<details><summary>Results table</summary>', '');
-            let header = '| Test file |';
-            let nFiles = 0;
-            for (const [file] of groups.values().next().value.values().next().value.values().next()
-                .value) {
-                header += ` ${file} |`;
-                nFiles++;
-            }
-            reportContent.push(header);
-            reportContent.push(new Array(nFiles + 2).fill('|').join(' --- '));
-            const filler = new Array(nFiles).fill('|').join(' ');
-            for (const [test, testGroup] of groups) {
-                reportContent.push(`| **${test}** | ` + filler);
-                for (const [browser, browserGroup] of testGroup) {
-                    reportContent.push(`| \`${browser}\` | ` + filler);
-                    for (const [reference, fileGroup] of browserGroup) {
-                        let line = `| ${reference} |`;
-                        for (const [_, dataGroup] of fileGroup) {
-                            const [q1, median, q3] = vs.quartiles(dataGroup);
-                            line += ` ${Math.min(...dataGroup).toFixed()} <- [${q1.toFixed()} - ${median.toFixed()} - ${q3.toFixed()}] -> ${Math.max(...dataGroup).toFixed()} |`;
-                        }
-                        reportContent.push(line);
-                    }
-                }
-            }
-            reportContent.push('', '</details>', '');
-            const reportFile = path_1.default.resolve(outputDir, `${baseName}.md`);
-            fs_1.default.writeFileSync(reportFile, reportContent.join('\n'), 'utf-8');
-            // Generate image
             const vegaSpec = vl.compile(config).spec;
             const view = new vega.View(vega.parse(vegaSpec), {
                 renderer: 'canvas'
@@ -263,6 +306,134 @@ class BenchmarkReporter {
             console.log(reportString);
         }
     }
+    /**
+     * Default text report factory of `BenchmarkReporter`, this method will
+     * be used by to generate markdown report. Users can customize the builder
+     * by supplying another builder to constructor's option or override this
+     * method on a sub-class.
+     *
+     * @param allData all test records.
+     * @param comparison logic of test comparisons:
+     * 'snapshot' or 'project'; default 'snapshot'.
+     * @return A list of two strings, the first one
+     * is the content of report, the second one is the extension of report file.
+     */
+    async defaultTextReportFactory(allData, comparison = 'snapshot') {
+        var _a;
+        // Compute statistics
+        // - Groupby (test, browser, reference | project, file)
+        const reportExtension = 'md';
+        const groups = new Map();
+        allData.forEach(d => {
+            var _a;
+            if (!groups.has(d.test)) {
+                groups.set(d.test, new Map());
+            }
+            const testGroup = groups.get(d.test);
+            if (!testGroup.has(d.browser)) {
+                testGroup.set(d.browser, new Map());
+            }
+            const browserGroup = testGroup.get(d.browser);
+            const lastLevel = comparison === 'snapshot' ? d.reference : d.project;
+            if (!browserGroup.has(lastLevel)) {
+                browserGroup.set(lastLevel, new Map());
+            }
+            const fileGroup = browserGroup.get(lastLevel);
+            if (!fileGroup.has(d.file)) {
+                fileGroup.set(d.file, new Array());
+            }
+            (_a = fileGroup.get(d.file)) === null || _a === void 0 ? void 0 : _a.push(d.time);
+        });
+        // If the reference | project lists has two items, the intervals will be compared.
+        if (!groups.values().next().value) {
+            return ['## Benchmark report\n\nNot enough data', reportExtension];
+        }
+        const compare = ((_a = groups.values().next().value) === null || _a === void 0 ? void 0 : _a.values().next().value).size === 2;
+        // - Create report
+        const reportContent = new Array('## Benchmark report', '', 'The execution time (in milliseconds) are grouped by test file, test type and browser.', 'For each case, the following values are computed: _min_ <- [_1st quartile_ - _median_ - _3rd quartile_] -> _max_.');
+        if (compare) {
+            reportContent.push('', 'The mean relative comparison is computed with 95% confidence.');
+        }
+        reportContent.push('', '<details><summary>Results table</summary>', '');
+        let header = '| Test file |';
+        let nFiles = 0;
+        for (const [file] of groups.values().next().value.values().next().value.values().next()
+            .value) {
+            header += ` ${file} |`;
+            nFiles++;
+        }
+        reportContent.push(header);
+        reportContent.push(new Array(nFiles + 2).fill('|').join(' --- '));
+        const filler = new Array(nFiles).fill('|').join(' ');
+        let changeReference = benchmark.DEFAULT_EXPECTED_REFERENCE;
+        for (const [test, testGroup] of groups) {
+            reportContent.push(`| **${test}** | ` + filler);
+            for (const [browser, browserGroup] of testGroup) {
+                reportContent.push(`| \`${browser}\` | ` + filler);
+                const actual = new Map();
+                const expected = new Map();
+                for (const [reference, fileGroup] of browserGroup) {
+                    let line = `| ${reference} |`;
+                    for (const [filename, dataGroup] of fileGroup) {
+                        const [q1, median, q3] = vs.quartiles(dataGroup);
+                        if (compare) {
+                            if (reference === benchmark.DEFAULT_REFERENCE ||
+                                !actual.has(filename)) {
+                                actual.set(filename, dataGroup);
+                            }
+                            else {
+                                changeReference = reference;
+                                expected.set(filename, dataGroup);
+                            }
+                        }
+                        line += ` ${Math.min(...dataGroup).toFixed()} <- [${q1.toFixed()} - ${median.toFixed()} - ${q3.toFixed()}] -> ${Math.max(...dataGroup).toFixed()} |`;
+                    }
+                    reportContent.push(line);
+                }
+                if (compare) {
+                    let line = `| Mean relative change |`;
+                    for (const [filename, oldDistribution] of expected) {
+                        const newDistribution = actual.get(filename);
+                        try {
+                            const delta = benchmark.distributionChange(oldDistribution, newDistribution, 0.95, true);
+                            let unmatchWarning = '';
+                            if (oldDistribution.length != newDistribution.length) {
+                                unmatchWarning = `[:warning:](# "Reference size ${oldDistribution.length} != Actual size ${newDistribution.length}") `;
+                            }
+                            line += ` ${unmatchWarning}${((delta.mean - 1) * 100).toFixed(1)}% ± ${(delta.confidenceInterval * 100).toFixed(1)}% |`;
+                        }
+                        catch (error) {
+                            console.error(`Reference has length ${oldDistribution.length} and new has ${newDistribution.length}.`);
+                            line += ` ${error} |`;
+                        }
+                    }
+                    reportContent.push(line);
+                }
+            }
+        }
+        if (compare) {
+            reportContent.push('', `Changes are computed with _${changeReference}_ as reference.`);
+        }
+        reportContent.push('', '</details>', '');
+        const reportContentString = reportContent.join('\n');
+        return [reportContentString, reportExtension];
+    }
+    /**
+     * Default Vega Lite config factory of `BenchmarkReporter`, this method will
+     * be used by to generate VegaLite configuration. Users can customize
+     * the builder by supplying another builder to constructor's option or
+     * override this method on a sub-class.
+     *
+     * @param allData all test records.
+     * @param comparison logic of test comparisons:
+     * 'snapshot' or 'project'; default 'snapshot'.
+     * @return VegaLite configuration
+     */
+    defaultVegaLiteConfigFactory(allData, comparison = 'snapshot') {
+        const config = benchmarkVLTpl_1.default([...new Set(allData.map(d => d.test))], comparison == 'snapshot' ? 'reference' : 'project', [...new Set(allData.map(d => d.file))]);
+        config.data.values = allData;
+        return config;
+    }
     async getMetadata(browser) {
         const cpu = await systeminformation_1.default.cpu();
         // Keep only non-variable value

package/lib/benchmarkReporter.js.map

@@ -1 +1 @@
-{"version":3,"file":"benchmarkReporter.js","sourceRoot":"","sources":["../src/benchmarkReporter.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6D;AAU7D,4CAAoB;AACpB,gDAAwB;AACxB,0EAAmC;AACnC,2CAA6B;AAC7B,8CAAgC;AAChC,oDAAsC;AACtC,sEAAoD;AAEpD;;GAEG;AACH,IAAiB,SAAS,CAkGzB;AAlGD,WAAiB,SAAS;;IACxB;;OAEG;IACU,wBAAc,GAAG,mBAAmB,CAAC;IAClD;;OAEG;IACU,iCAAuB,GAAG,kBAAkB,CAAC;IAC1D;;OAEG;IACU,gCAAsB,GAAG,EAAE,CAAC;IACzC;;OAEG;IACU,wBAAc,GAAG,gBAAgB,CAAC;IAC/C;;OAEG;IACU,oCAA0B,GAAG,UAAU,CAAC;IACrD;;OAEG;IACU,2BAAiB,GAAG,QAAQ,CAAC;IAE1C;;OAEG;IACU,kBAAQ,GACnB,QAAQ,OAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,mCAAI,GAAG,EAAE,EAAE,CAAC;QAC5D,UAAA,sBAAsB,CAAC;IAsDzB;;;;;OAKG;IACH,SAAgB,aAAa,CAAU,IAAa;QAClD,OAAO;YACL,IAAI,EAAE,UAAA,uBAAuB;YAC7B,WAAW,EAAE,kBAAkB;YAC/B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SACxC,CAAC;IACJ,CAAC;IANe,uBAAa,gBAM5B,CAAA;AACH,CAAC,EAlGgB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAkGzB;AAmDD;;GAEG;AACH,MAAM,iBAAiB;IACrB;;;;;;OAMG;IACH,YACE,UAAwE,EAAE;;QAE1E,IAAI,CAAC,WAAW,eACd,OAAO,CAAC,UAAU,mCAClB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,mCACnC,SAAS,CAAC,cAAc,CAAC;QAC3B,IAAI,CAAC,WAAW,SAAG,OAAO,CAAC,UAAU,mCAAI,UAAU,CAAC;QACpD,IAAI,CAAC,UAAU,SACb,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,mCAAI,SAAS,CAAC,iBAAiB,CAAC;IACtE,CAAC;IAED;;;;OAIG;IACH,OAAO,CAAC,MAAkB,EAAE,KAAY;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,KAAK,EAAiB,CAAC;QAC1C,sCAAsC;QACtC,IAAI,IAAI,CAAC,WAAW,EAAE;YACpB,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAC5B,cAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAC9B,SAAS,CAAC,cAAc,CACzB,CAAC;YACF,IAAI,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5B,YAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;aACxD;SACF;IACH,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAc,EAAE,MAAkB;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE;YAC9B,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,GAAG,MAAM,CAAC,WAAW;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,uBAAuB,CAAC;iBACzD,GAAG,CAAC,GAAG,CAAC,EAAE;;gBACT,MAAM,IAAI,GAAI,IAAI,CAAC,KAAK,aACtB,GAAG,CAAC,IAAI,0CAAE,QAAQ,qCAAM,IAAI,CACA,CAAC;gBAC/B,uCAAY,IAAI,KAAE,SAAS,EAAE,IAAI,CAAC,UAAU,IAAG;YACjD,CAAC,CAAC,CACL,CAAC;SACH;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,KAAK,CAAC,MAAkB;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1D,IAAI,IAAI,CAAC,WAAW,EAAE;YACpB,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAC5B,cAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAC9B,SAAS,CAAC,cAAc,CACzB,CAAC;YACF,MAAM,QAAQ,GAAG,cAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAC1D,YAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,QAAQ,OAAO,CAAC,EAC3C,YAAY,EACZ,OAAO,CACR,CAAC;YAEF,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAEnC,IAAI,IAAI,CAAC,WAAW,KAAK,UAAU,EAAE;gBACnC,qFAAqF;gBACrF,MAAM,gBAAgB,GAAG,cAAI,CAAC,OAAO,CACnC,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,GAAG,QAAQ,gBAAgB,CAC5B,CAAC;gBACF,MAAM,eAAe,GAAG,YAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;gBACxD,IAAI,YAAqB,CAAC;gBAC1B,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,KAAK,KAAK,EAAE;oBAC7D,YAAY,GAAG;wBACb,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;4BAC5B,uCAAY,CAAC,KAAE,SAAS,EAAE,SAAS,CAAC,0BAA0B,IAAG;wBACnE,CAAC,CAAC;wBACF,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC;oBAEF,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,KAAK,MAAM,EAAE;wBAC1C,YAAE,CAAC,aAAa,CACd,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,EAC1C,OAAO,CACR,CAAC;qBACH;iBACF;qBAAM;oBACL,MAAM,QAAQ,GAAG,YAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;oBAC5D,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;iBACrC;gBAED,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;aACtC;YAED,oBAAoB;YACpB,MAAM,eAAe,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,QAAQ,UAAU,CAAC,CAAC;YACvE,MAAM,MAAM,GAAG,wBAAoB,CACjC,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EACtC,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,EACxD,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CACvC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;YAC7B,YAAE,CAAC,aAAa,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;YAEnE,qBAAqB;YACrB,6CAA6C;YAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,EAGnB,CAAC;YAEJ,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;;gBAClB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE;oBACvB,MAAM,CAAC,GAAG,CACR,CAAC,CAAC,IAAI,EACN,IAAI,GAAG,EAA8C,CACtD,CAAC;iBACH;gBAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC;gBAEtC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE;oBAC7B,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,GAAG,EAAiC,CAAC,CAAC;iBACpE;gBAED,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAE,CAAC;gBAE/C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE;oBAClC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,GAAG,EAAoB,CAAC,CAAC;iBAC5D;gBAED,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAE,CAAC;gBAEjD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE;oBAC1B,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,EAAU,CAAC,CAAC;iBAC5C;gBAED,MAAA,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,0CAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;YACtC,CAAC,CAAC,CAAC;YAEH,kBAAkB;YAClB,MAAM,aAAa,GAAG,IAAI,KAAK,CAC7B,qBAAqB,EACrB,EAAE,EACF,uFAAuF,EACvF,mHAAmH,EACnH,EAAE,EACF,2CAA2C,EAC3C,EAAE,CACH,CAAC;YAEF,IAAI,MAAM,GAAG,eAAe,CAAC;YAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,CACT,IAAI,CACL,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;iBACpE,KAAK,EAAE;gBACR,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC;gBACvB,MAAM,EAAE,CAAC;aACV;YACD,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,aAAa,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAClE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAErD,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,EAAE;gBACtC,aAAa,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC;gBAChD,KAAK,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,IAAI,SAAS,EAAE;oBAC/C,aAAa,CAAC,IAAI,CAAC,OAAO,OAAO,OAAO,GAAG,MAAM,CAAC,CAAC;oBACnD,KAAK,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,YAAY,EAAE;wBACjD,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC;wBAC9B,KAAK,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,SAAS,EAAE;4BACtC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;4BACjD,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,CAClB,GAAG,SAAS,CACb,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,OAAO,EAAE,QAAQ,IAAI,CAAC,GAAG,CACrF,GAAG,SAAS,CACb,CAAC,OAAO,EAAE,IAAI,CAAC;yBACjB;wBAED,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;qBAC1B;iBACF;aACF;YACD,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;YAEzC,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAC;YAC7D,YAAE,CAAC,aAAa,CAAC,UAAU,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAEhE,iBAAiB;YACjB,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,MAAa,CAAC,CAAC,IAAI,CAAC;YAEhD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;gBAC/C,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,MAAM,GAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAA0B,CAAC;YACjE,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,QAAQ,MAAM,CAAC,CAAC;YAC7D,MAAM,UAAU,GAAG,YAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEnD,oCAAoC;YACpC,IAAI,QAA8B,CAAC;YACnC,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;gBACxC,QAAQ,GAAG,OAAO,CAAC;YACrB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;gBAC7B,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAE5B,MAAM,WAAW,CAAC;SACnB;aAAM;YACL,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;SAC3B;IACH,CAAC;IAES,KAAK,CAAC,WAAW,CAAC,OAAgB;QAC1C,MAAM,GAAG,GAAG,MAAM,2BAAE,CAAC,GAAG,EAAE,CAAC;QAC3B,+BAA+B;QAC/B,MAAM,WAAW,GAAG,CAAC,MAAM,2BAAE,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3C,6BAA6B;QAC7B,6DAA6D;QAC7D,MAAM,KAAgC,MAAM,2BAAE,CAAC,MAAM,EAAE,EAAjD,EAAE,QAAQ,EAAE,IAAI,OAAiC,EAA5B,MAAM,cAA3B,oBAA6B,CAAoB,CAAC;QAExD,MAAM,QAAQ,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QACnD,MAAM,eAAe,GAA+B,EAAE,CAAC;QAEvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC9B,IAAI;gBACF,QAAQ,OAAO,EAAE;oBACf,KAAK,UAAU;wBACb,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,eAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC/D,MAAM;oBACR,KAAK,SAAS;wBACZ,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,cAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC9D,MAAM;oBACR,KAAK,QAAQ;wBACX,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,aAAM,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC7D,MAAM;iBACT;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,6BAA6B;aAC9B;SACF;QAED,OAAO;YACL,QAAQ,EAAE,eAAe;YACzB,SAAS,EAAE;gBACT,oBAAoB,EAAE,IAAI,CAAC,WAAW;gBACtC,mBAAmB,EAAE,IAAI,CAAC,UAAU;aACrC;YACD,iBAAiB,EAAE;gBACjB,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE;gBAC3B,MAAM,EAAE,MAAM;aACf;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,WAAW;QACzB,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;SACnC,CAAC;IACJ,CAAC;CAQF;AAED,kBAAe,iBAAiB,CAAC"}
+{"version":3,"file":"benchmarkReporter.js","sourceRoot":"","sources":["../src/benchmarkReporter.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2CAA6D;AAS7D,6CAA+D;AAE/D,4CAAoB;AACpB,gDAAwB;AACxB,0EAAmC;AACnC,2CAA6B;AAC7B,8CAAgC;AAChC,oDAAsC;AACtC,sEAAoD;AAEpD;;GAEG;AACH,IAAiB,SAAS,CAyOzB;AAzOD,WAAiB,SAAS;;IACxB;;OAEG;IACU,wBAAc,GAAG,mBAAmB,CAAC;IAClD;;OAEG;IACU,iCAAuB,GAAG,kBAAkB,CAAC;IAC1D;;OAEG;IACU,gCAAsB,GAAG,EAAE,CAAC;IACzC;;OAEG;IACU,wBAAc,GAAG,gBAAgB,CAAC;IAC/C;;OAEG;IACU,oCAA0B,GAAG,UAAU,CAAC;IACrD;;OAEG;IACU,2BAAiB,GAAG,QAAQ,CAAC;IAE1C;;OAEG;IACU,kBAAQ,GACnB,QAAQ,OAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,mCAAI,GAAG,EAAE,EAAE,CAAC;QAC5D,UAAA,sBAAsB,CAAC;IAsDzB;;;;;OAKG;IACH,SAAgB,aAAa,CAAU,IAAa;QAClD,OAAO;YACL,IAAI,EAAE,UAAA,uBAAuB;YAC7B,WAAW,EAAE,kBAAkB;YAC/B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SACxC,CAAC;IACJ,CAAC;IANe,uBAAa,gBAM5B,CAAA;IA6BD;;;;;;;;;;;;;;;;;;OAkBG;IACH,SAAS,iBAAiB,CACxB,eAA8B,EAC9B,eAA8B,EAC9B,CAAS,EACT,qBAA6B,IAAI;QAEjC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,eAAe,CAAC;QACnD,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,eAAe,CAAC;QACnD,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,MAAM,CAAC,GAAG,YAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,SAAS,GAAG,SAAS,CAAC,CAAC;QAC7D,OAAO;YACL,IAAI,EAAE,OAAO,GAAG,SAAS;YACzB,kBAAkB,EAAE,KAAK,GAAG,SAAS;SACtC,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,SAAgB,kBAAkB,CAChC,WAAqB,EACrB,WAAqB,EACrB,qBAA6B,IAAI,EACjC,SAAS,GAAG,KAAK;QAEjB,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,WAAW,CAAC,MAAM,EAAE;YAC1C,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;SAC/C;QACD,OAAO,iBAAiB,CACtB,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,WAAW,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,WAAW,CAAC,EAAE,EAClE,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,WAAW,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,WAAW,CAAC,EAAE,EAClE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAC/C,kBAAkB,CACnB,CAAC;IACJ,CAAC;IAhBe,4BAAkB,qBAgBjC,CAAA;IAED;;;;;OAKG;IACH,SAAgB,YAAY,CAAC,YAAiC;QAC5D,MAAM,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,YAAY,CAAC;QAClD,OAAO,WAAW,aAAa,CAC7B,IAAI,GAAG,kBAAkB,CAC1B,QAAQ,aAAa,CAAC,IAAI,GAAG,kBAAkB,CAAC,EAAE,CAAC;IACtD,CAAC;IALe,sBAAY,eAK3B,CAAA;IAED,SAAS,aAAa,CAAC,OAAe;QACpC,IAAI,OAAO,GAAG,CAAC,EAAE;YACf,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;SACtD;QACD,OAAO,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC;IACvD,CAAC;IAED,SAAS,EAAE,CAAC,CAAS;QACnB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,SAAS,IAAI,CAAC,GAAG,CAAW;QAC1B,OAAO,aAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,SAAS,QAAQ,CAAC,GAAG,CAAW;QAC9B,OAAO,iBAAU,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;AACH,CAAC,EAzOgB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAyOzB;AAmDD;;GAEG;AACH,MAAM,iBAAiB;IACrB;;;;;;;;;OASG;IACH,YACE,UAWI,EAAE;;QAEN,IAAI,CAAC,WAAW,eACd,OAAO,CAAC,UAAU,mCAClB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,mCACnC,SAAS,CAAC,cAAc,CAAC;QAE3B,IAAI,CAAC,WAAW,SAAG,OAAO,CAAC,UAAU,mCAAI,UAAU,CAAC;QAEpD,IAAI,CAAC,UAAU,SACb,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,mCAAI,SAAS,CAAC,iBAAiB,CAAC;QAEpE,IAAI,CAAC,mBAAmB,SACtB,OAAO,CAAC,qBAAqB,mCAAI,IAAI,CAAC,4BAA4B,CAAC;QAErE,IAAI,CAAC,gBAAgB,SACnB,OAAO,CAAC,iBAAiB,mCAAI,IAAI,CAAC,wBAAwB,CAAC;IAC/D,CAAC;IAED;;;;OAIG;IACH,OAAO,CAAC,MAAkB,EAAE,KAAY;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,KAAK,EAAiB,CAAC;QAC1C,sCAAsC;QACtC,IAAI,IAAI,CAAC,WAAW,EAAE;YACpB,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAC5B,cAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAC9B,SAAS,CAAC,cAAc,CACzB,CAAC;YACF,IAAI,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5B,YAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;aACxD;SACF;IACH,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,IAAc,EAAE,MAAkB;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE;YAC9B,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,GAAG,MAAM,CAAC,WAAW;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,uBAAuB,CAAC;iBACzD,GAAG,CAAC,GAAG,CAAC,EAAE;;gBACT,MAAM,IAAI,GAAI,IAAI,CAAC,KAAK,aACtB,GAAG,CAAC,IAAI,0CAAE,QAAQ,qCAAM,IAAI,CACA,CAAC;gBAC/B,uCAAY,IAAI,KAAE,SAAS,EAAE,IAAI,CAAC,UAAU,IAAG;YACjD,CAAC,CAAC,CACL,CAAC;SACH;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,KAAK,CAAC,MAAkB;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;QAC1D,IAAI,IAAI,CAAC,WAAW,EAAE;YACpB,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAC5B,cAAI,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAC9B,SAAS,CAAC,cAAc,CACzB,CAAC;YACF,MAAM,QAAQ,GAAG,cAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAC1D,YAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,QAAQ,OAAO,CAAC,EAC3C,YAAY,EACZ,OAAO,CACR,CAAC;YAEF,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAEnC,IAAI,IAAI,CAAC,WAAW,KAAK,UAAU,EAAE;gBACnC,qFAAqF;gBACrF,MAAM,gBAAgB,GAAG,cAAI,CAAC,OAAO,CACnC,IAAI,CAAC,MAAM,CAAC,OAAO,EACnB,GAAG,QAAQ,gBAAgB,CAC5B,CAAC;gBACF,MAAM,eAAe,GAAG,YAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;gBACxD,IAAI,YAAqB,CAAC;gBAC1B,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,KAAK,KAAK,EAAE;oBAC7D,YAAY,GAAG;wBACb,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;4BAC5B,uCAAY,CAAC,KAAE,SAAS,EAAE,SAAS,CAAC,0BAA0B,IAAG;wBACnE,CAAC,CAAC;wBACF,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC;oBAEF,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,KAAK,MAAM,EAAE;wBAC1C,YAAE,CAAC,aAAa,CACd,gBAAgB,EAChB,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,EAC1C,OAAO,CACR,CAAC;qBACH;iBACF;qBAAM;oBACL,MAAM,QAAQ,GAAG,YAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;oBAC5D,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;iBACrC;gBAED,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;aACtC;YAED,kBAAkB;YAClB,MAAM,CACJ,mBAAmB,EACnB,eAAe,CAChB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACzC,MAAM,UAAU,GAAG,cAAI,CAAC,OAAO,CAC7B,SAAS,EACT,GAAG,QAAQ,IAAI,eAAe,EAAE,CACjC,CAAC;YACF,YAAE,CAAC,aAAa,CAAC,UAAU,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;YAE3D,gCAAgC;YAChC,MAAM,eAAe,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,QAAQ,UAAU,CAAC,CAAC;YACvE,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YACnE,YAAE,CAAC,aAAa,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;YACnE,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,MAAa,CAAC,CAAC,IAAI,CAAC;YAEhD,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE;gBAC/C,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,MAAM,GAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAA0B,CAAC;YACjE,MAAM,SAAS,GAAG,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,QAAQ,MAAM,CAAC,CAAC;YAC7D,MAAM,UAAU,GAAG,YAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEnD,oCAAoC;YACpC,IAAI,QAA8B,CAAC;YACnC,MAAM,WAAW,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;gBACxC,QAAQ,GAAG,OAAO,CAAC;YACrB,CAAC,CAAC,CAAC;YACH,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;gBAC7B,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAE5B,MAAM,WAAW,CAAC;SACnB;aAAM;YACL,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;SAC3B;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACO,KAAK,CAAC,wBAAwB,CACtC,OAA6B,EAC7B,aAAqC,UAAU;;QAE/C,qBAAqB;QACrB,uDAAuD;QACvD,MAAM,eAAe,GAAG,IAAI,CAAC;QAE7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAGnB,CAAC;QAEJ,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;;YAClB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE;gBACvB,MAAM,CAAC,GAAG,CACR,CAAC,CAAC,IAAI,EACN,IAAI,GAAG,EAA8C,CACtD,CAAC;aACH;YAED,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC;YAEtC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE;gBAC7B,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,GAAG,EAAiC,CAAC,CAAC;aACpE;YAED,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAE,CAAC;YAE/C,MAAM,SAAS,GAAG,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAEtE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBAChC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,GAAG,EAAoB,CAAC,CAAC;aAC1D;YAED,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;YAE/C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE;gBAC1B,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,EAAU,CAAC,CAAC;aAC5C;YAED,MAAA,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,0CAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE;QACtC,CAAC,CAAC,CAAC;QAEH,kFAAkF;QAClF,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE;YACjC,OAAO,CAAC,wCAAwC,EAAE,eAAe,CAAC,CAAC;SACpE;QAED,MAAM,OAAO,GACX,CAAC,MAAA,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,0CAAE,MAAM,GAAG,IAAI,GAAG,KAG7C,CAAA,CAAC,IAAI,KAAK,CAAC,CAAC;QAEhB,kBAAkB;QAClB,MAAM,aAAa,GAAG,IAAI,KAAK,CAC7B,qBAAqB,EACrB,EAAE,EACF,uFAAuF,EACvF,mHAAmH,CACpH,CAAC;QAEF,IAAI,OAAO,EAAE;YACX,aAAa,CAAC,IAAI,CAChB,EAAE,EACF,+DAA+D,CAChE,CAAC;SACH;QAED,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,2CAA2C,EAAE,EAAE,CAAC,CAAC;QAExE,IAAI,MAAM,GAAG,eAAe,CAAC;QAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,CACT,IAAI,CACL,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;aACpE,KAAK,EAAE;YACR,MAAM,IAAI,IAAI,IAAI,IAAI,CAAC;YACvB,MAAM,EAAE,CAAC;SACV;QACD,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,aAAa,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAErD,IAAI,eAAe,GAAG,SAAS,CAAC,0BAA0B,CAAC;QAE3D,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,EAAE;YACtC,aAAa,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC;YAChD,KAAK,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,IAAI,SAAS,EAAE;gBAC/C,aAAa,CAAC,IAAI,CAAC,OAAO,OAAO,OAAO,GAAG,MAAM,CAAC,CAAC;gBACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;gBAC3C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;gBAC7C,KAAK,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,YAAY,EAAE;oBACjD,IAAI,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC;oBAC9B,KAAK,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,SAAS,EAAE;wBAC7C,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;wBAEjD,IAAI,OAAO,EAAE;4BACX,IACE,SAAS,KAAK,SAAS,CAAC,iBAAiB;gCACzC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EACrB;gCACA,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;6BACjC;iCAAM;gCACL,eAAe,GAAG,SAAS,CAAC;gCAC5B,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;6BACnC;yBACF;wBAED,IAAI,IAAI,IAAI,IAAI,CAAC,GAAG,CAClB,GAAG,SAAS,CACb,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,OAAO,EAAE,QAAQ,IAAI,CAAC,GAAG,CACrF,GAAG,SAAS,CACb,CAAC,OAAO,EAAE,IAAI,CAAC;qBACjB;oBAED,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC1B;gBAED,IAAI,OAAO,EAAE;oBACX,IAAI,IAAI,GAAG,0BAA0B,CAAC;oBACtC,KAAK,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,QAAQ,EAAE;wBAClD,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;wBAC9C,IAAI;4BACF,MAAM,KAAK,GAAG,SAAS,CAAC,kBAAkB,CACxC,eAAe,EACf,eAAe,EACf,IAAI,EACJ,IAAI,CACL,CAAC;4BAEF,IAAI,cAAc,GAAG,EAAE,CAAC;4BACxB,IAAI,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,MAAM,EAAE;gCACpD,cAAc,GAAG,iCAAiC,eAAe,CAAC,MAAM,mBAAmB,eAAe,CAAC,MAAM,KAAK,CAAC;6BACxH;4BAED,IAAI,IAAI,IAAI,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAC3D,CAAC,CACF,OAAO,CAAC,KAAK,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;yBAC1D;wBAAC,OAAO,KAAK,EAAE;4BACd,OAAO,CAAC,KAAK,CACX,wBAAwB,eAAe,CAAC,MAAM,gBAAgB,eAAe,CAAC,MAAM,GAAG,CACxF,CAAC;4BACF,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC;yBACvB;qBACF;oBAED,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBAC1B;aACF;SACF;QACD,IAAI,OAAO,EAAE;YACX,aAAa,CAAC,IAAI,CAChB,EAAE,EACF,8BAA8B,eAAe,iBAAiB,CAC/D,CAAC;SACH;QACD,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,mBAAmB,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,OAAO,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;IAChD,CAAC;IAED;;;;;;;;;;OAUG;IACO,4BAA4B,CACpC,OAA6B,EAC7B,aAAqC,UAAU;QAE/C,MAAM,MAAM,GAAG,wBAAoB,CACjC,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EACtC,UAAU,IAAI,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,EAClD,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CACvC,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IAES,KAAK,CAAC,WAAW,CAAC,OAAgB;QAC1C,MAAM,GAAG,GAAG,MAAM,2BAAE,CAAC,GAAG,EAAE,CAAC;QAC3B,+BAA+B;QAC/B,MAAM,WAAW,GAAG,CAAC,MAAM,2BAAE,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC;QAC3C,6BAA6B;QAC7B,6DAA6D;QAC7D,MAAM,KAAgC,MAAM,2BAAE,CAAC,MAAM,EAAE,EAAjD,EAAE,QAAQ,EAAE,IAAI,OAAiC,EAA5B,MAAM,cAA3B,oBAA6B,CAAoB,CAAC;QAExD,MAAM,QAAQ,GAAG,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QACnD,MAAM,eAAe,GAA+B,EAAE,CAAC;QAEvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC9B,IAAI;gBACF,QAAQ,OAAO,EAAE;oBACf,KAAK,UAAU;wBACb,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,eAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC/D,MAAM;oBACR,KAAK,SAAS;wBACZ,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,cAAO,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC9D,MAAM;oBACR,KAAK,QAAQ;wBACX,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,aAAM,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC7D,MAAM;iBACT;aACF;YAAC,OAAO,CAAC,EAAE;gBACV,6BAA6B;aAC9B;SACF;QAED,OAAO;YACL,QAAQ,EAAE,eAAe;YACzB,SAAS,EAAE;gBACT,oBAAoB,EAAE,IAAI,CAAC,WAAW;gBACtC,mBAAmB,EAAE,IAAI,CAAC,UAAU;aACrC;YACD,iBAAiB,EAAE;gBACjB,GAAG,EAAE,GAAG;gBACR,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE;gBAC3B,MAAM,EAAE,MAAM;aACf;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,WAAW;QACzB,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,QAAQ,EAAE,MAAM,IAAI,CAAC,WAAW,EAAE;SACnC,CAAC;IACJ,CAAC;CAeF;AAED,kBAAe,iBAAiB,CAAC"}

package/lib/benchmarkVLTpl.js

@@ -26,13 +26,11 @@ function configPerFile(tests, comparison) {
                 transform: [{ filter: `datum.test === '${t}'` }],
                 facet: {
                     column: { field: 'browser', type: 'nominal' }
-                    // row: { field: 'test', type: 'nominal' }
                 },
                 spec: {
                     mark: { type: 'boxplot', extent: 'min-max' },
                     encoding: {
                         y: { field: comparison, type: 'nominal' },
-                        // color: { field: 'file', type: 'nominal', legend: null },
                         x: {
                             field: 'time',
                             title: 'Time (ms)',

package/lib/benchmarkVLTpl.js.map

@@ -1 +1 @@
-{"version":3,"file":"benchmarkVLTpl.js","sourceRoot":"","sources":["../src/benchmarkVLTpl.ts"],"names":[],"mappings":";AAAA,0BAA0B;;AAE1B;;GAEG;AACH,MAAM,cAAc,GAAG;IACrB,OAAO,EAAE,iDAAiD;IAC1D,WAAW,EAAE,gCAAgC;IAC7C,KAAK,EAAE,4BAA4B;IACnC,IAAI,EAAE,EAAyB;IAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE;CACnC,CAAC;AAEF;;;;;;GAMG;AACH,SAAS,aAAa,CACpB,KAAe,EACf,UAAkB;IAElB,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACrB,OAAO;gBACL,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,CAAC,GAAG,EAAE,CAAC;gBAChD,KAAK,EAAE;oBACL,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;oBAC7C,0CAA0C;iBAC3C;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE;oBAC5C,QAAQ,EAAE;wBACR,CAAC,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE;wBACzC,2DAA2D;wBAC3D,CAAC,EAAE;4BACD,KAAK,EAAE,MAAM;4BACb,KAAK,EAAE,WAAW;4BAClB,IAAI,EAAE,cAAc;4BACpB,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;yBACvB;qBACF;iBACF;aACF,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,oBAAoB,CAC3B,KAAe,EACf,UAAkB,EAClB,SAAoB;IAEpB,MAAM,KAAK,GAAG,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,EAAE,CAAC;IAE9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,uCACK,cAAc,GACd,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,EACnC;KACH;SAAM;QACL,uCACK,cAAc,KACjB,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBACrB,uBACE,KAAK,EAAE,CAAC,EACR,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,CAAC,GAAG,EAAE,CAAC,IAC7C,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,EACnC;YACJ,CAAC,CAAC,IACF;KACH;AACH,CAAC;AAED,kBAAe,oBAAoB,CAAC"}
+{"version":3,"file":"benchmarkVLTpl.js","sourceRoot":"","sources":["../src/benchmarkVLTpl.ts"],"names":[],"mappings":";AAAA,0BAA0B;;AAE1B;;GAEG;AACH,MAAM,cAAc,GAAG;IACrB,OAAO,EAAE,iDAAiD;IAC1D,WAAW,EAAE,gCAAgC;IAC7C,KAAK,EAAE,4BAA4B;IACnC,IAAI,EAAE,EAAyB;IAC/B,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE;CACnC,CAAC;AAEF;;;;;;GAMG;AACH,SAAS,aAAa,CACpB,KAAe,EACf,UAAkB;IAElB,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACrB,OAAO;gBACL,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,CAAC,GAAG,EAAE,CAAC;gBAChD,KAAK,EAAE;oBACL,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;iBAC9C;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE;oBAC5C,QAAQ,EAAE;wBACR,CAAC,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE;wBACzC,CAAC,EAAE;4BACD,KAAK,EAAE,MAAM;4BACb,KAAK,EAAE,WAAW;4BAClB,IAAI,EAAE,cAAc;4BACpB,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;yBACvB;qBACF;iBACF;aACF,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,oBAAoB,CAC3B,KAAe,EACf,UAAkB,EAClB,SAAoB;IAEpB,MAAM,KAAK,GAAG,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,EAAE,CAAC;IAE9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,uCACK,cAAc,GACd,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,EACnC;KACH;SAAM;QACL,uCACK,cAAc,KACjB,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBACrB,uBACE,KAAK,EAAE,CAAC,EACR,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,mBAAmB,CAAC,GAAG,EAAE,CAAC,IAC7C,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,EACnC;YACJ,CAAC,CAAC,IACF;KACH;AACH,CAAC;AAED,kBAAe,oBAAoB,CAAC"}