@vercel/devlow-bench 0.1.3 → 0.2.4

package/README.md

@@ -86,12 +86,12 @@ describe(
 import { measureTime, reportMeasurement } from "devlow-bench";
 
 // Measure a time
-measureTime("name of the timing", {
+await measureTime("name of the timing", {
   /* optional options */
 });
 
 // Report some other measurement
-reportMeasurement("name of the measurement", value, unit, {
+await reportMeasurement("name of the measurement", value, unit, {
   /* optional options */
 });
 ```
@@ -118,6 +118,10 @@ await session.softNavigationByClick("metric name", ".selector-to-click");
 await session.close();
 ```
 
+Run with `BROWSER_OUTPUT=1` to show the output of the browser.
+
+Run with `HEADLESS=false` to show the actual browser window.
+
 ## Shell operations
 
 The `devlow-bench` package provides a few helper functions to run operations in the shell.
@@ -151,6 +155,8 @@ shell.output
 await shell.kill();
 ```
 
+Run with `SHELL_OUTPUT=1` to show the output of the shell commands.
+
 ## File operations
 
 The `devlow-bench` package provides a few helper functions to run operations on the file system.

package/dist/browser.d.ts

@@ -1,7 +1,7 @@
 import type { Page } from "playwright-chromium";
 interface BrowserSession {
     close(): Promise<void>;
-    hardNavigation(url: string, metricName?: string): Promise<Page>;
+    hardNavigation(metricName: string, url: string): Promise<Page>;
     softNavigationByClick(metricName: string, selector: string): Promise<void>;
     reload(metricName: string): Promise<void>;
 }

package/dist/browser.js

@@ -1,5 +1,6 @@
 import { chromium } from "playwright-chromium";
 import { measureTime, reportMeasurement } from "./index.js";
+const browserOutput = !!process.env.BROWSER_OUTPUT;
 async function withRequestMetrics(metricName, page, fn) {
     const activePromises = [];
     const sizeByExtension = new Map();
@@ -25,50 +26,118 @@ async function withRequestMetrics(metricName, page, fn) {
             }
         })());
     };
+    let errorCount = 0;
+    let warningCount = 0;
+    let logCount = 0;
+    const consoleHandler = (message) => {
+        const type = message.type();
+        if (type === "error") {
+            errorCount++;
+        }
+        else if (type === "warning") {
+            warningCount++;
+        }
+        else {
+            logCount++;
+        }
+        if (browserOutput) {
+            activePromises.push((async () => {
+                const args = [];
+                try {
+                    const text = message.text();
+                    for (const arg of message.args()) {
+                        args.push(await arg.jsonValue());
+                    }
+                    console.log(`[${type}] ${text}`, ...args);
+                }
+                catch {
+                    // Ignore
+                }
+            })());
+        }
+    };
+    let uncaughtCount = 0;
+    const exceptionHandler = (error) => {
+        uncaughtCount++;
+        if (browserOutput) {
+            console.error(`[UNCAUGHT]`, error);
+        }
+    };
     try {
         page.on("response", responseHandler);
+        page.on("console", consoleHandler);
+        page.on("pageerror", exceptionHandler);
         await fn();
         await Promise.all(activePromises);
         let totalDownload = 0;
         for (const [extension, size] of sizeByExtension.entries()) {
-            reportMeasurement(`${metricName}/responseSizes/${extension}`, size, "bytes");
+            await reportMeasurement(`${metricName}/responseSizes/${extension}`, size, "bytes");
             totalDownload += size;
         }
-        reportMeasurement(`${metricName}/responseSizes`, totalDownload, "bytes");
+        await reportMeasurement(`${metricName}/responseSizes`, totalDownload, "bytes");
         let totalRequests = 0;
         for (const [extension, count] of requestsByExtension.entries()) {
-            reportMeasurement(`${metricName}/requests/${extension}`, count, "requests");
+            await reportMeasurement(`${metricName}/requests/${extension}`, count, "requests");
             totalRequests += count;
         }
-        reportMeasurement(`${metricName}/requests`, totalRequests, "requests");
+        await reportMeasurement(`${metricName}/requests`, totalRequests, "requests");
+        await reportMeasurement(`${metricName}/console/logs`, logCount, "messages");
+        await reportMeasurement(`${metricName}/console/warnings`, warningCount, "messages");
+        await reportMeasurement(`${metricName}/console/errors`, errorCount, "messages");
+        await reportMeasurement(`${metricName}/console/uncaught`, uncaughtCount, "messages");
+        await reportMeasurement(`${metricName}/console`, logCount + warningCount + errorCount + uncaughtCount, "messages");
     }
     finally {
         page.off("response", responseHandler);
     }
 }
-function networkIdle(page) {
-    return new Promise((resolve) => {
+function networkIdle(page, delay = 300, rejectTimeout = 180000) {
+    return new Promise((resolve, reject) => {
         const cleanup = () => {
             page.off("request", requestHandler);
             page.off("requestfailed", requestFinishedHandler);
             page.off("requestfinished", requestFinishedHandler);
+            clearTimeout(fullTimeout);
+            if (timeout) {
+                clearTimeout(timeout);
+            }
         };
         let activeRequests = 0;
         let timeout = null;
-        const requestHandler = (request) => {
+        const requests = new Set();
+        const fullTimeout = setTimeout(() => {
+            cleanup();
+            reject(new Error(`Timeout while waiting for network idle. These requests are still pending: ${Array.from(requests).join(", ")}}`));
+        }, rejectTimeout);
+        const requestFilter = async (request) => {
+            return (await request.headers().accept) !== "text/event-stream";
+        };
+        const requestHandler = async (request) => {
+            requests.add(request.url());
             activeRequests++;
             if (timeout) {
                 clearTimeout(timeout);
                 timeout = null;
             }
+            // Avoid tracking some requests, but we only know this after awaiting
+            // so we need to do this weird stunt to ensure that
+            if (!(await requestFilter(request))) {
+                await requestFinishedInternal(request);
+            }
         };
-        const requestFinishedHandler = (request) => {
+        const requestFinishedHandler = async (request) => {
+            if (await requestFilter(request)) {
+                requestFinishedInternal(request);
+            }
+        };
+        const requestFinishedInternal = async (request) => {
+            requests.delete(request.url());
             activeRequests--;
             if (activeRequests === 0) {
                 timeout = setTimeout(() => {
                     cleanup();
                     resolve();
-                }, 300);
+                }, delay);
             }
         };
         page.on("request", requestHandler);
@@ -93,6 +162,7 @@ class BrowserSessionImpl {
         const page = (this.page = this.page ?? (await this.context.newPage()));
         await withRequestMetrics(metricName, page, async () => {
             measureTime(`${metricName}/start`);
+            const idle = networkIdle(page, 3000);
             await page.goto(url, {
                 waitUntil: "commit",
             });
@@ -107,8 +177,9 @@ class BrowserSessionImpl {
             measureTime(`${metricName}/load`, {
                 relativeTo: `${metricName}/start`,
             });
-            await page.waitForLoadState("networkidle");
+            await idle;
             measureTime(`${metricName}`, {
+                offset: 3000,
                 relativeTo: `${metricName}/start`,
             });
         });
@@ -122,7 +193,7 @@ class BrowserSessionImpl {
         await withRequestMetrics(metricName, page, async () => {
             measureTime(`${metricName}/start`);
             const firstResponse = new Promise((resolve) => page.once("response", () => resolve()));
-            const idle = networkIdle(page);
+            const idle = networkIdle(page, 3000);
             await page.click(selector);
             await firstResponse;
             measureTime(`${metricName}/firstResponse`, {
@@ -130,6 +201,7 @@ class BrowserSessionImpl {
             });
             await idle;
             measureTime(`${metricName}`, {
+                offset: 3000,
                 relativeTo: `${metricName}/start`,
             });
         });
@@ -141,6 +213,7 @@ class BrowserSessionImpl {
         }
         await withRequestMetrics(metricName, page, async () => {
             measureTime(`${metricName}/start`);
+            const idle = networkIdle(page, 3000);
             await page.reload({
                 waitUntil: "commit",
             });
@@ -155,8 +228,9 @@ class BrowserSessionImpl {
             measureTime(`${metricName}/load`, {
                 relativeTo: `${metricName}/start`,
             });
-            await page.waitForLoadState("networkidle");
+            await idle;
             measureTime(`${metricName}`, {
+                offset: 3000,
                 relativeTo: `${metricName}/start`,
             });
         });

package/dist/describe.d.ts

@@ -10,9 +10,10 @@ export declare function measureTime(name: string, options?: {
     relativeTo?: string | typeof PREVIOUS;
     scenario?: string;
     props?: Record<string, string | number | null>;
-}): void;
+    offset?: number;
+}): Promise<void>;
 export declare function reportMeasurement(name: string, value: number, unit: string, options?: {
     relativeTo?: string | typeof PREVIOUS;
     scenario?: string;
     props?: Record<string, string | number | null>;
-}): void;
+}): Promise<void>;

package/dist/describe.js

@@ -56,14 +56,32 @@ export function withCurrent(current, fn) {
     });
 }
 export const PREVIOUS = Symbol("previous measurement with that unit");
-export function measureTime(name, options = {}) {
-    const end = Date.now();
-    reportMeasurement(name, end, "ms", { relativeTo: PREVIOUS, ...options });
+export async function measureTime(name, options = {}) {
+    const end = Date.now() - (options.offset || 0);
+    await reportMeasurement(name, end, "ms", {
+        relativeTo: PREVIOUS,
+        ...options,
+    });
 }
-export function reportMeasurement(name, value, unit, options = {}) {
+export async function reportMeasurement(name, value, unit, options = {}) {
     if (!currentScenario) {
         throw new Error("reportMeasurement() must be called inside of describe()");
     }
+    if (typeof name !== "string") {
+        throw new Error("reportMeasurement() must be called with a name that is a string");
+    }
+    if (typeof value !== "number") {
+        throw new Error("reportMeasurement() must be called with a value that is a number");
+    }
+    if (isNaN(value)) {
+        throw new Error("reportMeasurement() must be called with a value that is not NaN");
+    }
+    if (!isFinite(value)) {
+        throw new Error("reportMeasurement() must be called with a value that is finite");
+    }
+    if (typeof unit !== "string") {
+        throw new Error("reportMeasurement() must be called with a unit that is a string");
+    }
     let { relativeTo, scenario, props } = options;
     if (relativeTo === PREVIOUS) {
         relativeTo = "previous";
@@ -83,12 +101,12 @@ export function reportMeasurement(name, value, unit, options = {}) {
         if (!prev) {
             throw new Error(`No measurement named ${relativeTo} found`);
         }
-        if (prev.unit !== "ms") {
+        if (prev.unit !== unit) {
             throw new Error(`Measurement ${relativeTo} is not a "${unit}" measurement`);
         }
         reportedValue -= prev.value;
     }
-    currentScenario.iface.measurement(scenario ?? currentScenario.scenario.scenario.name, props
+    await currentScenario.iface.measurement(scenario ?? currentScenario.scenario.scenario.name, props
         ? {
             ...currentScenario.scenario.props,
             ...props,

package/dist/shell.js

@@ -91,7 +91,7 @@ class CommandImpl {
                 .filter((x) => x)
                 .map((x) => x.memory)
                 .reduce((a, b) => a + b, 0);
-            reportMeasurement(metricName, memUsage, "bytes", options);
+            await reportMeasurement(metricName, memUsage, "bytes", options);
         }
         catch (e) {
             // ignore

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vercel/devlow-bench",
-  "version": "0.1.3",
+  "version": "0.2.4",
   "description": "Benchmarking tool for the developer workflow",
   "type": "module",
   "main": "dist/index.js",
@@ -25,7 +25,8 @@
     "@types/inquirer": "^9.0.3",
     "@types/minimist": "^1.2.2",
     "@types/node": "^20.3.0",
-    "@types/split2": "^4.2.0"
+    "@types/split2": "^4.2.0",
+    "@turbo/eslint-config": "0.0.0"
   },
   "dependencies": {
     "@datadog/datadog-api-client": "^1.13.0",
@@ -38,6 +39,7 @@
     "tree-kill": "^1.2.2"
   },
   "scripts": {
+    "lint": "eslint src/",
     "prerelease": "pnpm run build:ts",
     "build:ts": "tsc"
   }