@vercel/devlow-bench 0.1.3 → 0.1.4

package/README.md

@@ -118,6 +118,10 @@ await session.softNavigationByClick("metric name", ".selector-to-click");
 await session.close();
 ```
 
+Run with `BROWSER_OUTPUT=1` to show the output of the browser.
+
+Run with `HEADLESS=false` to show the actual browser window.
+
 ## Shell operations
 
 The `devlow-bench` package provides a few helper functions to run operations in the shell.
@@ -151,6 +155,8 @@ shell.output
 await shell.kill();
 ```
 
+Run with `SHELL_OUTPUT=1` to show the output of the shell commands.
+
 ## File operations
 
 The `devlow-bench` package provides a few helper functions to run operations on the file system.

package/dist/browser.d.ts

@@ -1,7 +1,7 @@
 import type { Page } from "playwright-chromium";
 interface BrowserSession {
     close(): Promise<void>;
-    hardNavigation(url: string, metricName?: string): Promise<Page>;
+    hardNavigation(metricName: string, url: string): Promise<Page>;
     softNavigationByClick(metricName: string, selector: string): Promise<void>;
     reload(metricName: string): Promise<void>;
 }

package/dist/browser.js

@@ -1,5 +1,6 @@
 import { chromium } from "playwright-chromium";
 import { measureTime, reportMeasurement } from "./index.js";
+const browserOutput = !!process.env.BROWSER_OUTPUT;
 async function withRequestMetrics(metricName, page, fn) {
     const activePromises = [];
     const sizeByExtension = new Map();
@@ -25,8 +26,47 @@ async function withRequestMetrics(metricName, page, fn) {
             }
         })());
     };
+    let errorCount = 0;
+    let warningCount = 0;
+    let logCount = 0;
+    const consoleHandler = (message) => {
+        const type = message.type();
+        if (type === "error") {
+            errorCount++;
+        }
+        else if (type === "warning") {
+            warningCount++;
+        }
+        else {
+            logCount++;
+        }
+        if (browserOutput) {
+            activePromises.push((async () => {
+                const args = [];
+                try {
+                    const text = message.text();
+                    for (const arg of message.args()) {
+                        args.push(await arg.jsonValue());
+                    }
+                    console.log(`[${type}] ${text}`, ...args);
+                }
+                catch {
+                    // Ignore
+                }
+            })());
+        }
+    };
+    let uncaughtCount = 0;
+    const exceptionHandler = (error) => {
+        uncaughtCount++;
+        if (browserOutput) {
+            console.error(`[UNCAUGHT]`, error);
+        }
+    };
     try {
         page.on("response", responseHandler);
+        page.on("console", consoleHandler);
+        page.on("pageerror", exceptionHandler);
         await fn();
         await Promise.all(activePromises);
         let totalDownload = 0;
@@ -41,34 +81,63 @@ async function withRequestMetrics(metricName, page, fn) {
             totalRequests += count;
         }
         reportMeasurement(`${metricName}/requests`, totalRequests, "requests");
+        reportMeasurement(`${metricName}/console/logs`, logCount, "messages");
+        reportMeasurement(`${metricName}/console/warnings`, warningCount, "messages");
+        reportMeasurement(`${metricName}/console/errors`, errorCount, "messages");
+        reportMeasurement(`${metricName}/console/uncaught`, uncaughtCount, "messages");
+        reportMeasurement(`${metricName}/console`, logCount + warningCount + errorCount + uncaughtCount, "messages");
     }
     finally {
         page.off("response", responseHandler);
     }
 }
-function networkIdle(page) {
-    return new Promise((resolve) => {
+function networkIdle(page, delay = 300, rejectTimeout = 180000) {
+    return new Promise((resolve, reject) => {
         const cleanup = () => {
             page.off("request", requestHandler);
             page.off("requestfailed", requestFinishedHandler);
             page.off("requestfinished", requestFinishedHandler);
+            clearTimeout(fullTimeout);
+            if (timeout) {
+                clearTimeout(timeout);
+            }
         };
         let activeRequests = 0;
         let timeout = null;
-        const requestHandler = (request) => {
+        const requests = new Set();
+        const fullTimeout = setTimeout(() => {
+            cleanup();
+            reject(new Error(`Timeout while waiting for network idle. These requests are still pending: ${Array.from(requests).join(", ")}}`));
+        }, rejectTimeout);
+        const requestFilter = async (request) => {
+            return (await request.headers().accept) !== "text/event-stream";
+        };
+        const requestHandler = async (request) => {
+            requests.add(request.url());
             activeRequests++;
             if (timeout) {
                 clearTimeout(timeout);
                 timeout = null;
             }
+            // Avoid tracking some requests, but we only know this after awaiting
+            // so we need to do this weird stunt to ensure that
+            if (!(await requestFilter(request))) {
+                await requestFinishedInternal(request);
+            }
         };
-        const requestFinishedHandler = (request) => {
+        const requestFinishedHandler = async (request) => {
+            if (await requestFilter(request)) {
+                requestFinishedInternal(request);
+            }
+        };
+        const requestFinishedInternal = async (request) => {
+            requests.delete(request.url());
             activeRequests--;
             if (activeRequests === 0) {
                 timeout = setTimeout(() => {
                     cleanup();
                     resolve();
-                }, 300);
+                }, delay);
             }
         };
         page.on("request", requestHandler);
@@ -93,6 +162,7 @@ class BrowserSessionImpl {
         const page = (this.page = this.page ?? (await this.context.newPage()));
         await withRequestMetrics(metricName, page, async () => {
             measureTime(`${metricName}/start`);
+            const idle = networkIdle(page, 3000);
             await page.goto(url, {
                 waitUntil: "commit",
             });
@@ -107,8 +177,9 @@ class BrowserSessionImpl {
             measureTime(`${metricName}/load`, {
                 relativeTo: `${metricName}/start`,
             });
-            await page.waitForLoadState("networkidle");
+            await idle;
             measureTime(`${metricName}`, {
+                offset: 3000,
                 relativeTo: `${metricName}/start`,
             });
         });
@@ -122,7 +193,7 @@ class BrowserSessionImpl {
         await withRequestMetrics(metricName, page, async () => {
             measureTime(`${metricName}/start`);
             const firstResponse = new Promise((resolve) => page.once("response", () => resolve()));
-            const idle = networkIdle(page);
+            const idle = networkIdle(page, 3000);
             await page.click(selector);
             await firstResponse;
             measureTime(`${metricName}/firstResponse`, {
@@ -130,6 +201,7 @@ class BrowserSessionImpl {
             });
             await idle;
             measureTime(`${metricName}`, {
+                offset: 3000,
                 relativeTo: `${metricName}/start`,
             });
         });
@@ -141,6 +213,7 @@ class BrowserSessionImpl {
         }
         await withRequestMetrics(metricName, page, async () => {
             measureTime(`${metricName}/start`);
+            const idle = networkIdle(page, 3000);
             await page.reload({
                 waitUntil: "commit",
             });
@@ -155,8 +228,9 @@ class BrowserSessionImpl {
             measureTime(`${metricName}/load`, {
                 relativeTo: `${metricName}/start`,
             });
-            await page.waitForLoadState("networkidle");
+            await idle;
             measureTime(`${metricName}`, {
+                offset: 3000,
                 relativeTo: `${metricName}/start`,
             });
         });

package/dist/describe.d.ts

@@ -10,6 +10,7 @@ export declare function measureTime(name: string, options?: {
     relativeTo?: string | typeof PREVIOUS;
     scenario?: string;
     props?: Record<string, string | number | null>;
+    offset?: number;
 }): void;
 export declare function reportMeasurement(name: string, value: number, unit: string, options?: {
     relativeTo?: string | typeof PREVIOUS;

package/dist/describe.js

@@ -57,7 +57,7 @@ export function withCurrent(current, fn) {
 }
 export const PREVIOUS = Symbol("previous measurement with that unit");
 export function measureTime(name, options = {}) {
-    const end = Date.now();
+    const end = Date.now() - (options.offset || 0);
     reportMeasurement(name, end, "ms", { relativeTo: PREVIOUS, ...options });
 }
 export function reportMeasurement(name, value, unit, options = {}) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vercel/devlow-bench",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "Benchmarking tool for the developer workflow",
   "type": "module",
   "main": "dist/index.js",