@vercel/devlow-bench 0.1.2 → 0.1.3

package/dist/browser.d.ts

@@ -0,0 +1,13 @@
+import type { Page } from "playwright-chromium";
+interface BrowserSession {
+    close(): Promise<void>;
+    hardNavigation(url: string, metricName?: string): Promise<Page>;
+    softNavigationByClick(metricName: string, selector: string): Promise<void>;
+    reload(metricName: string): Promise<void>;
+}
+export declare function newBrowserSession(options: {
+    headless?: boolean;
+    devtools?: boolean;
+    baseURL?: string;
+}): Promise<BrowserSession>;
+export {};

package/dist/browser.js

@@ -0,0 +1,178 @@
+import { chromium } from "playwright-chromium";
+import { measureTime, reportMeasurement } from "./index.js";
+async function withRequestMetrics(metricName, page, fn) {
+    const activePromises = [];
+    const sizeByExtension = new Map();
+    const requestsByExtension = new Map();
+    const responseHandler = (response) => {
+        activePromises.push((async () => {
+            const url = response.request().url();
+            const status = response.status();
+            const extension = /^[^\?#]+\.([a-z0-9]+)(?:[\?#]|$)/i.exec(url)?.[1] ?? "none";
+            const currentRequests = requestsByExtension.get(extension) ?? 0;
+            requestsByExtension.set(extension, currentRequests + 1);
+            if (status >= 200 && status < 300) {
+                let body;
+                try {
+                    body = await response.body();
+                }
+                catch { }
+                if (body) {
+                    const size = body.length;
+                    const current = sizeByExtension.get(extension) ?? 0;
+                    sizeByExtension.set(extension, current + size);
+                }
+            }
+        })());
+    };
+    try {
+        page.on("response", responseHandler);
+        await fn();
+        await Promise.all(activePromises);
+        let totalDownload = 0;
+        for (const [extension, size] of sizeByExtension.entries()) {
+            reportMeasurement(`${metricName}/responseSizes/${extension}`, size, "bytes");
+            totalDownload += size;
+        }
+        reportMeasurement(`${metricName}/responseSizes`, totalDownload, "bytes");
+        let totalRequests = 0;
+        for (const [extension, count] of requestsByExtension.entries()) {
+            reportMeasurement(`${metricName}/requests/${extension}`, count, "requests");
+            totalRequests += count;
+        }
+        reportMeasurement(`${metricName}/requests`, totalRequests, "requests");
+    }
+    finally {
+        page.off("response", responseHandler);
+    }
+}
+function networkIdle(page) {
+    return new Promise((resolve) => {
+        const cleanup = () => {
+            page.off("request", requestHandler);
+            page.off("requestfailed", requestFinishedHandler);
+            page.off("requestfinished", requestFinishedHandler);
+        };
+        let activeRequests = 0;
+        let timeout = null;
+        const requestHandler = (request) => {
+            activeRequests++;
+            if (timeout) {
+                clearTimeout(timeout);
+                timeout = null;
+            }
+        };
+        const requestFinishedHandler = (request) => {
+            activeRequests--;
+            if (activeRequests === 0) {
+                timeout = setTimeout(() => {
+                    cleanup();
+                    resolve();
+                }, 300);
+            }
+        };
+        page.on("request", requestHandler);
+        page.on("requestfailed", requestFinishedHandler);
+        page.on("requestfinished", requestFinishedHandler);
+    });
+}
+class BrowserSessionImpl {
+    constructor(browser, context) {
+        this.browser = browser;
+        this.context = context;
+        this.page = null;
+    }
+    async close() {
+        if (this.page) {
+            await this.page.close();
+        }
+        await this.context.close();
+        await this.browser.close();
+    }
+    async hardNavigation(metricName, url) {
+        const page = (this.page = this.page ?? (await this.context.newPage()));
+        await withRequestMetrics(metricName, page, async () => {
+            measureTime(`${metricName}/start`);
+            await page.goto(url, {
+                waitUntil: "commit",
+            });
+            measureTime(`${metricName}/html`, {
+                relativeTo: `${metricName}/start`,
+            });
+            await page.waitForLoadState("domcontentloaded");
+            measureTime(`${metricName}/dom`, {
+                relativeTo: `${metricName}/start`,
+            });
+            await page.waitForLoadState("load");
+            measureTime(`${metricName}/load`, {
+                relativeTo: `${metricName}/start`,
+            });
+            await page.waitForLoadState("networkidle");
+            measureTime(`${metricName}`, {
+                relativeTo: `${metricName}/start`,
+            });
+        });
+        return page;
+    }
+    async softNavigationByClick(metricName, selector) {
+        const page = this.page;
+        if (!page) {
+            throw new Error("softNavigationByClick() must be called after hardNavigation()");
+        }
+        await withRequestMetrics(metricName, page, async () => {
+            measureTime(`${metricName}/start`);
+            const firstResponse = new Promise((resolve) => page.once("response", () => resolve()));
+            const idle = networkIdle(page);
+            await page.click(selector);
+            await firstResponse;
+            measureTime(`${metricName}/firstResponse`, {
+                relativeTo: `${metricName}/start`,
+            });
+            await idle;
+            measureTime(`${metricName}`, {
+                relativeTo: `${metricName}/start`,
+            });
+        });
+    }
+    async reload(metricName) {
+        const page = this.page;
+        if (!page) {
+            throw new Error("reload() must be called after hardNavigation()");
+        }
+        await withRequestMetrics(metricName, page, async () => {
+            measureTime(`${metricName}/start`);
+            await page.reload({
+                waitUntil: "commit",
+            });
+            measureTime(`${metricName}/html`, {
+                relativeTo: `${metricName}/start`,
+            });
+            await page.waitForLoadState("domcontentloaded");
+            measureTime(`${metricName}/dom`, {
+                relativeTo: `${metricName}/start`,
+            });
+            await page.waitForLoadState("load");
+            measureTime(`${metricName}/load`, {
+                relativeTo: `${metricName}/start`,
+            });
+            await page.waitForLoadState("networkidle");
+            measureTime(`${metricName}`, {
+                relativeTo: `${metricName}/start`,
+            });
+        });
+    }
+}
+export async function newBrowserSession(options) {
+    const browser = await chromium.launch({
+        headless: options.headless ?? process.env.HEADLESS !== "false",
+        timeout: 60000,
+    });
+    const context = await browser.newContext({
+        baseURL: options.baseURL ?? "http://localhost:3000",
+        viewport: { width: 1280, height: 720 },
+    });
+    context.setDefaultTimeout(120000);
+    context.setDefaultNavigationTimeout(120000);
+    return new BrowserSessionImpl(browser, context);
+}
+// see next.js/test/lib/browsers/playwright.ts

package/dist/cli.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/cli.js

@@ -0,0 +1,88 @@
+import minimist from "minimist";
+import { setCurrentScenarios } from "./describe.js";
+import { join } from "path";
+import { runScenarios } from "./index.js";
+import compose from "./interfaces/compose.js";
+import { pathToFileURL } from "url";
+(async () => {
+    const knownArgs = new Set([
+        "scenario",
+        "s",
+        "json",
+        "j",
+        "console",
+        "datadog",
+        "interactive",
+        "i",
+        "help",
+        "h",
+        "?",
+        "_",
+    ]);
+    const args = minimist(process.argv.slice(2), {
+        alias: {
+            s: "scenario",
+            j: "json",
+            i: "interactive",
+            "?": "help",
+            h: "help",
+        },
+    });
+    if (args.help || (Object.keys(args).length === 1 && args._.length === 0)) {
+        console.log("Usage: devlow-bench [options] <scenario files>");
+        console.log("## Selecting scenarios");
+        console.log("  --scenario=<filter>, -s=<filter>   Only run the scenario with the given name");
+        console.log("  --interactive, -i                  Select scenarios and variants interactively");
+        console.log("  --<prop>=<value>                   Filter by any variant property defined in scenarios");
+        console.log("## Output");
+        console.log("  --json=<path>, -j=<path>           Write the results to the given path as JSON");
+        console.log("  --console                          Print the results to the console");
+        console.log("  --datadog[=<hostname>]             Upload the results to Datadog");
+        console.log("                                     (requires DATADOG_API_KEY environment variables)");
+        console.log("## Help");
+        console.log("  --help, -h, -?                     Show this help");
+    }
+    const scenarios = [];
+    setCurrentScenarios(scenarios);
+    for (const path of args._) {
+        await import(pathToFileURL(join(process.cwd(), path)).toString());
+    }
+    setCurrentScenarios(null);
+    const cliIface = {
+        filterScenarios: async (scenarios) => {
+            if (args.scenario) {
+                const filter = [].concat(args.scenario);
+                return scenarios.filter((s) => filter.some((filter) => s.name.includes(filter)));
+            }
+            return scenarios;
+        },
+        filterScenarioVariants: async (variants) => {
+            const propEntries = Object.entries(args).filter(([key]) => !knownArgs.has(key));
+            if (propEntries.length === 0)
+                return variants;
+            for (const [key, value] of propEntries) {
+                const values = (Array.isArray(value) ? value : [value]).map((v) => v.toString());
+                variants = variants.filter((variant) => {
+                    const prop = variant.props[key];
+                    if (typeof prop === "undefined")
+                        return false;
+                    const str = prop.toString();
+                    return values.some((v) => str.includes(v));
+                });
+            }
+            return variants;
+        },
+    };
+    let ifaces = [
+        cliIface,
+        args.interactive && (await import("./interfaces/interactive.js")).default(),
+        args.json && (await import("./interfaces/json.js")).default(args.json),
+        args.datadog &&
+            (await import("./interfaces/datadog.js")).default(typeof args.datadog === "string" ? { host: args.datadog } : undefined),
+        args.console !== false &&
+            (await import("./interfaces/console.js")).default(),
+    ].filter((x) => x);
+    await runScenarios(scenarios, compose(...ifaces));
+})().catch((e) => {
+    console.error(e.stack);
+});

package/dist/describe.d.ts

@@ -0,0 +1,18 @@
+import type { ConfigFor, CurrentScenario, Scenario } from "./index.js";
+export declare function setCurrentScenarios(scenarios: Scenario[] | null): void;
+export declare function describe<P>(name: string, config: ConfigFor<P>, fn: (props: P) => Promise<void>): void;
+export declare namespace describe {
+    var only: <P>(name: string, config: ConfigFor<P>, fn: (props: P) => Promise<void>) => void;
+}
+export declare function withCurrent(current: CurrentScenario, fn: () => Promise<void>): Promise<void>;
+export declare const PREVIOUS: unique symbol;
+export declare function measureTime(name: string, options?: {
+    relativeTo?: string | typeof PREVIOUS;
+    scenario?: string;
+    props?: Record<string, string | number | null>;
+}): void;
+export declare function reportMeasurement(name: string, value: number, unit: string, options?: {
+    relativeTo?: string | typeof PREVIOUS;
+    scenario?: string;
+    props?: Record<string, string | number | null>;
+}): void;

package/dist/describe.js

@@ -0,0 +1,97 @@
+import compose from "./interfaces/compose.js";
+import { runScenarios } from "./runner.js";
+let currentScenarios = null;
+export function setCurrentScenarios(scenarios) {
+    currentScenarios = scenarios;
+}
+export function describe(name, config, fn) {
+    if (currentScenarios === null) {
+        const scenarios = (currentScenarios = []);
+        Promise.resolve().then(async () => {
+            const ifaceNames = process.env.INTERFACE || "interactive,console";
+            const ifaces = [];
+            for (const ifaceName of ifaceNames.split(",").map((s) => s.trim())) {
+                let iface;
+                try {
+                    iface = await import(`./interfaces/${ifaceName}.js`);
+                }
+                catch (e) {
+                    iface = await import(ifaceName);
+                }
+                iface = (iface && iface.default) || iface;
+                if (typeof iface === "function") {
+                    iface = await iface();
+                }
+                if (!iface) {
+                    throw new Error(`Interface ${ifaceName} is not a valid interface`);
+                }
+                ifaces.push(iface);
+            }
+            runScenarios(scenarios, compose(...ifaces));
+        });
+    }
+    const normalizedConfig = Object.fromEntries(Object.entries(config).map(([key, value]) => [
+        key,
+        typeof value === "boolean"
+            ? [value, !value]
+            : value,
+    ]));
+    currentScenarios.push({
+        name,
+        config: normalizedConfig,
+        only: false,
+        fn: fn,
+    });
+}
+describe.only = function describeOnly(name, config, fn) {
+    describe(name, config, fn);
+    currentScenarios[currentScenarios.length - 1].only = true;
+};
+let currentScenario = null;
+export function withCurrent(current, fn) {
+    const prev = currentScenario;
+    currentScenario = current;
+    return fn().finally(() => {
+        currentScenario = prev;
+    });
+}
+export const PREVIOUS = Symbol("previous measurement with that unit");
+export function measureTime(name, options = {}) {
+    const end = Date.now();
+    reportMeasurement(name, end, "ms", { relativeTo: PREVIOUS, ...options });
+}
+export function reportMeasurement(name, value, unit, options = {}) {
+    if (!currentScenario) {
+        throw new Error("reportMeasurement() must be called inside of describe()");
+    }
+    let { relativeTo, scenario, props } = options;
+    if (relativeTo === PREVIOUS) {
+        relativeTo = "previous";
+        for (const [prevName, prev] of currentScenario.measurements) {
+            if (prev.unit === unit) {
+                relativeTo = prevName;
+            }
+        }
+    }
+    currentScenario.measurements.set(name, {
+        value,
+        unit,
+    });
+    let reportedValue = value;
+    if (relativeTo) {
+        const prev = currentScenario.measurements.get(relativeTo);
+        if (!prev) {
+            throw new Error(`No measurement named ${relativeTo} found`);
+        }
+        if (prev.unit !== "ms") {
+            throw new Error(`Measurement ${relativeTo} is not a "${unit}" measurement`);
+        }
+        reportedValue -= prev.value;
+    }
+    currentScenario.iface.measurement(scenario ?? currentScenario.scenario.scenario.name, props
+        ? {
+            ...currentScenario.scenario.props,
+            ...props,
+        }
+        : currentScenario.scenario.props, name, reportedValue, unit, relativeTo);
+}

package/dist/file.d.ts

@@ -0,0 +1 @@
+export declare function waitForFile(path: string, timeout: number): Promise<void>;

package/dist/file.js

@@ -0,0 +1,54 @@
+import { watch } from "fs";
+import { access, constants } from "fs/promises";
+import { dirname } from "path";
+export async function waitForFile(path, timeout) {
+    let currentAction = "";
+    let timeoutRef;
+    const timeoutPromise = new Promise((resolve, reject) => {
+        timeoutRef = setTimeout(() => {
+            reject(new Error(`Timed out waiting for file ${path} (${currentAction}))`));
+        }, timeout || 60000);
+    });
+    const elements = [];
+    let current = path;
+    while (true) {
+        elements.push(current);
+        const parent = dirname(current);
+        if (parent === current) {
+            break;
+        }
+        current = parent;
+    }
+    elements.reverse();
+    try {
+        for (const path of elements) {
+            const checkAccess = () => access(path, constants.F_OK)
+                .then(() => true)
+                .catch(() => false);
+            if (!(await checkAccess())) {
+                let resolveCheckAgain = () => { };
+                const watcher = watch(dirname(path), () => {
+                    resolveCheckAgain();
+                });
+                currentAction = `waiting for ${path}`;
+                let checkAgainPromise = new Promise((resolve) => {
+                    resolveCheckAgain = resolve;
+                });
+                try {
+                    do {
+                        await Promise.race([timeoutPromise, checkAgainPromise]);
+                        checkAgainPromise = new Promise((resolve) => {
+                            resolveCheckAgain = resolve;
+                        });
+                    } while (!(await checkAccess()));
+                }
+                finally {
+                    watcher.close();
+                }
+            }
+        }
+    }
+    finally {
+        clearTimeout(timeoutRef);
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,34 @@
+export type ConfigFor<P> = {
+    [K in keyof P]: P[K] extends string ? string[] : P[K] extends number ? number[] : P[K] extends boolean ? boolean[] | boolean : never;
+};
+export interface Scenario {
+    name: string;
+    config: Record<string, (string | number | boolean)[]>;
+    only: boolean;
+    fn: (props: Record<string, string | number | boolean>) => Promise<void>;
+}
+export interface ScenarioVariant {
+    scenario: Scenario;
+    props: Record<string, string | number | boolean>;
+}
+export interface CurrentScenario {
+    scenario: ScenarioVariant;
+    iface: FullInterface;
+    measurements: Map<string, {
+        value: number;
+        unit: string;
+    }>;
+}
+export type Interface = Partial<FullInterface>;
+export interface FullInterface {
+    filterScenarios(scenarios: Scenario[]): Promise<Scenario[]>;
+    filterScenarioVariants(scenarioVariants: ScenarioVariant[]): Promise<ScenarioVariant[]>;
+    start(scenario: string, props: Record<string, string | number | boolean | null>): Promise<void>;
+    measurement(scenario: string, props: Record<string, string | number | boolean | null>, name: string, value: number, unit: string, relativeTo?: string): Promise<void>;
+    end(scenario: string, props: Record<string, string | number | boolean | null>): Promise<void>;
+    error(scenario: string, props: Record<string, string | number | boolean | null>, error: unknown): Promise<void>;
+    finish(): Promise<void>;
+}
+export declare function intoFullInterface(iface: Interface): FullInterface;
+export { describe, measureTime, reportMeasurement, PREVIOUS, } from "./describe.js";
+export { runScenarios } from "./runner.js";

package/dist/index.js

@@ -0,0 +1,14 @@
+export function intoFullInterface(iface) {
+    return {
+        filterScenarios: iface.filterScenarios ?? (async (scenarios) => scenarios),
+        filterScenarioVariants: iface.filterScenarioVariants ??
+            (async (scenarioVariants) => scenarioVariants),
+        start: iface.start ?? (async () => { }),
+        measurement: iface.measurement ?? (async () => { }),
+        end: iface.end ?? (async () => { }),
+        error: iface.error ?? (async () => { }),
+        finish: iface.finish ?? (async () => { }),
+    };
+}
+export { describe, measureTime, reportMeasurement, PREVIOUS, } from "./describe.js";
+export { runScenarios } from "./runner.js";

package/dist/interfaces/compose.d.ts

@@ -0,0 +1,2 @@
+import { Interface } from "../index.js";
+export default function compose(...ifaces: Interface[]): Interface;

package/dist/interfaces/compose.js

@@ -0,0 +1,33 @@
+export default function compose(...ifaces) {
+    const allKeys = new Set();
+    for (const iface of ifaces) {
+        for (const key of Object.keys(iface)) {
+            allKeys.add(key);
+        }
+    }
+    const composed = {};
+    for (const key of allKeys) {
+        if (key.startsWith("filter")) {
+            composed[key] = async (items, ...args) => {
+                for (const iface of ifaces) {
+                    const anyIface = iface;
+                    if (anyIface[key]) {
+                        items = await anyIface[key](items, ...args);
+                    }
+                }
+                return items;
+            };
+        }
+        else {
+            composed[key] = async (...args) => {
+                for (const iface of ifaces) {
+                    const anyIface = iface;
+                    if (anyIface[key]) {
+                        await anyIface[key](...args);
+                    }
+                }
+            };
+        }
+    }
+    return composed;
+}

package/dist/interfaces/console.d.ts

@@ -0,0 +1,2 @@
+import { Interface } from "../index.js";
+export default function createInterface(): Interface;

package/dist/interfaces/console.js

@@ -0,0 +1,17 @@
+import chalk from "chalk";
+import { formatUnit } from "../units.js";
+import { formatVariant } from "../utils.js";
+export default function createInterface() {
+    const iface = {
+        start: async (scenario, props) => {
+            console.log(chalk.bold.underline(`Running ${formatVariant(scenario, props)}...`));
+        },
+        measurement: async (scenario, props, name, value, unit, relativeTo) => {
+            console.log(chalk.bgCyan.bold.magenta(`${formatVariant(scenario, props)}: ${name} = ${formatUnit(value, unit)}${relativeTo ? ` (from ${relativeTo})` : ""}`));
+        },
+        error: async (scenario, props, error) => {
+            console.log(chalk.bold.red(`${formatVariant(scenario, props)}: ${(error && error.stack) || error}`));
+        },
+    };
+    return iface;
+}

package/dist/interfaces/datadog.d.ts

@@ -0,0 +1,6 @@
+import type { Interface } from "../index.js";
+export default function createInterface({ apiKey, appKey, host, }?: {
+    apiKey?: string;
+    appKey?: string;
+    host?: string;
+}): Interface;

package/dist/interfaces/datadog.js

@@ -0,0 +1,74 @@
+import datadogApiClient from "@datadog/datadog-api-client";
+import os from "os";
+function toIdentifier(str) {
+    return str.replace(/\//g, ".").replace(/ /g, "_");
+}
+const UNIT_MAPPING = {
+    ms: "millisecond",
+    requests: "request",
+    bytes: "byte",
+};
+export default function createInterface({ apiKey = process.env.DATADOG_API_KEY, appKey = process.env.DATADOG_APP_KEY, host = process.env.DATADOG_HOST || os.hostname(), } = {}) {
+    if (!apiKey)
+        throw new Error("Datadog API key is required (set DATADOG_API_KEY)");
+    const commonTags = [
+        `ci:${!!process.env.CI || "false"}`,
+        `os:${process.platform}`,
+        `os_release:${os.release()}`,
+        `cpus:${os.cpus().length}`,
+        `cpu_model:${os.cpus()[0].model}`,
+        `user:${os.userInfo().username}`,
+        `arch:${os.arch()}`,
+        `total_memory:${Math.round(os.totalmem() / 1024 / 1024 / 1024)}`,
+        `node_version:${process.version}`,
+    ];
+    const configuration = datadogApiClient.client.createConfiguration({
+        authMethods: {
+            apiKeyAuth: apiKey,
+            appKeyAuth: appKey,
+        },
+    });
+    const api = new datadogApiClient.v1.MetricsApi(configuration);
+    const dataPoints = [];
+    const metricMetadata = {};
+    const iface = {
+        measurement: async (scenario, props, name, value, unit, relativeTo) => {
+            const ts = Math.round(Date.now() / 1000);
+            const metric = toIdentifier(`devlow_bench/${scenario}/${name}`);
+            if (UNIT_MAPPING[unit]) {
+                metricMetadata[metric] = {
+                    unit: UNIT_MAPPING[unit],
+                };
+            }
+            dataPoints.push({
+                metric,
+                type: "distribution",
+                host,
+                tags: Object.entries(props)
+                    .filter(([, value]) => value !== null)
+                    .map(([key, value]) => `${toIdentifier(key)}:${toIdentifier(value.toString())}`)
+                    .concat(commonTags),
+                points: [[ts, [value]]],
+            });
+        },
+        end: async (scenario, props) => {
+            await api.submitDistributionPoints({
+                body: {
+                    series: dataPoints,
+                },
+            });
+            dataPoints.length = 0;
+        },
+        finish: async () => {
+            if (appKey) {
+                for (const [metric, metadata] of Object.entries(metricMetadata)) {
+                    await api.updateMetricMetadata({
+                        metricName: metric,
+                        body: metadata,
+                    });
+                }
+            }
+        },
+    };
+    return iface;
+}

package/dist/interfaces/interactive.d.ts

@@ -0,0 +1,2 @@
+import { Interface } from "../index.js";
+export default function createInterface(): Interface;

package/dist/interfaces/interactive.js

@@ -0,0 +1,41 @@
+import inquirer from "inquirer";
+import { formatVariant } from "../utils.js";
+export default function createInterface() {
+    const iface = {
+        filterScenarios: async (scenarios) => {
+            if (scenarios.length === 1) {
+                return scenarios;
+            }
+            let answer = await inquirer.prompt({
+                type: "checkbox",
+                name: "scenarios",
+                default: scenarios.slice(),
+                message: "Choose scenarios to run",
+                choices: scenarios.map((scenario) => ({
+                    name: scenario.name,
+                    value: scenario,
+                })),
+            });
+            return answer.scenarios;
+        },
+        filterScenarioVariants: async (variants) => {
+            if (variants.length === 1) {
+                return variants;
+            }
+            let answer = await inquirer.prompt({
+                type: "checkbox",
+                name: "variants",
+                default: variants.slice(),
+                message: "Choose variants to run",
+                choices: variants.map((variant) => {
+                    return {
+                        name: formatVariant(variant.scenario.name, variant.props),
+                        value: variant,
+                    };
+                }),
+            });
+            return answer.variants;
+        },
+    };
+    return iface;
+}

package/dist/interfaces/json.d.ts

@@ -0,0 +1,2 @@
+import { Interface } from "../index.js";
+export default function createInterface(file?: string): Interface;

package/dist/interfaces/json.js

@@ -0,0 +1,57 @@
+import { formatUnit } from "../units.js";
+import { writeFile } from "fs/promises";
+function filterProp(prop) {
+    const filteredProp = {};
+    for (const [key, value] of Object.entries(prop)) {
+        if (value !== null) {
+            filteredProp[key] = value;
+        }
+    }
+    return filteredProp;
+}
+export default function createInterface(file = (() => {
+    const file = process.env.JSON_OUTPUT_FILE;
+    if (!file) {
+        throw new Error("env var JSON_OUTPUT_FILE is not set");
+    }
+    return file;
+})()) {
+    const metrics = new Map();
+    const iface = {
+        measurement: async (scenario, props, name, value, unit, relativeTo) => {
+            const keyObject = {
+                scenario: scenario,
+                ...filterProp(props),
+                name: name,
+            };
+            const key = JSON.stringify(keyObject);
+            const current = metrics.get(key);
+            if (current) {
+                current.value += value;
+                current.count++;
+            }
+            else {
+                metrics.set(key, {
+                    key: keyObject,
+                    value,
+                    unit: unit,
+                    count: 1,
+                    relativeTo,
+                });
+            }
+        },
+        finish: async () => {
+            await writeFile(file, JSON.stringify([...metrics.values()].map(({ key, value, unit, count, relativeTo }) => {
+                return {
+                    key,
+                    value: value / count,
+                    unit,
+                    text: formatUnit(value / count, unit),
+                    datapoints: count,
+                    relativeTo,
+                };
+            })));
+        },
+    };
+    return iface;
+}

package/dist/runner.d.ts

@@ -0,0 +1,2 @@
+import { Interface, Scenario } from "./index.js";
+export declare function runScenarios(scenarios: Scenario[], iface: Interface): Promise<void>;

package/dist/runner.js

@@ -0,0 +1,54 @@
+import { withCurrent } from "./describe.js";
+import { intoFullInterface } from "./index.js";
+export async function runScenarios(scenarios, iface) {
+    const fullIface = intoFullInterface(iface);
+    if (scenarios.some((scenario) => scenario.only)) {
+        scenarios = scenarios.filter((scenario) => scenario.only);
+    }
+    scenarios = await fullIface.filterScenarios(scenarios);
+    let variants = [];
+    for (const scenario of scenarios) {
+        let props = [{}];
+        for (const [key, options] of Object.entries(scenario.config)) {
+            const newProps = [];
+            for (const prop of props) {
+                if (prop === "scenario" || prop === "name")
+                    throw new Error("Cannot use 'scenario' or 'name' as a property name");
+                for (const value of options) {
+                    newProps.push({
+                        ...prop,
+                        [key]: value,
+                    });
+                }
+            }
+            props = newProps;
+        }
+        variants.push(...props.map((props) => ({
+            scenario,
+            props,
+        })));
+    }
+    variants = await fullIface.filterScenarioVariants(variants);
+    for (const variant of variants) {
+        try {
+            const measurements = new Map();
+            await withCurrent({
+                iface: fullIface,
+                measurements,
+                scenario: variant,
+            }, async () => {
+                await fullIface.start(variant.scenario.name, variant.props);
+                measurements.set("start", {
+                    value: Date.now(),
+                    unit: "ms",
+                });
+                await variant.scenario.fn(variant.props);
+                await fullIface.end(variant.scenario.name, variant.props);
+            });
+        }
+        catch (e) {
+            await fullIface.error(variant.scenario.name, variant.props, e);
+        }
+    }
+    await fullIface.finish();
+}

package/dist/shell.d.ts

@@ -0,0 +1,19 @@
+import { PREVIOUS } from "./describe.js";
+export interface Command {
+    ok(): Promise<void>;
+    kill(): Promise<void>;
+    end(): Promise<number>;
+    waitForOutput(regex: RegExp): Promise<RegExpMatchArray>;
+    reportMemUsage(metricName: string, options: {
+        relativeTo?: string | typeof PREVIOUS;
+        scenario?: string;
+        props?: Record<string, string | number | null>;
+    }): Promise<void>;
+    stdout: string;
+    stderr: string;
+    output: string;
+}
+export declare function command(command: string, args: string[], options?: {
+    env?: Record<string, string>;
+    cwd?: string;
+}): Command;

package/dist/shell.js

@@ -0,0 +1,111 @@
+import { spawn } from "child_process";
+import split2 from "split2";
+import treeKill from "tree-kill";
+import pidusage from "pidusage-tree";
+import { reportMeasurement } from "./describe.js";
+const shellOutput = !!process.env.SHELL_OUTPUT;
+class CommandImpl {
+    constructor(process) {
+        this.process = process;
+        this.stdout = "";
+        this.stderr = "";
+        this.output = "";
+        this.waitingForOutput = [];
+        process.stdout?.pipe(split2()).on("data", (data) => {
+            const str = data.toString();
+            this.stdout += str + "\n";
+            this.output += str + "\n";
+            if (shellOutput) {
+                console.log(`[STDOUT] ${str}`);
+            }
+            if (this.waitingForOutput.length !== 0) {
+                const waitingForOutput = this.waitingForOutput;
+                this.waitingForOutput = [];
+                for (const fn of waitingForOutput) {
+                    fn();
+                }
+            }
+        });
+        process.stderr?.pipe(split2()).on("data", (data) => {
+            const str = data.toString();
+            this.stderr += str + "\n";
+            this.output += str + "\n";
+            if (shellOutput) {
+                console.log(`[STDERR] ${str}`);
+            }
+            if (this.waitingForOutput.length !== 0) {
+                const waitingForOutput = this.waitingForOutput;
+                this.waitingForOutput = [];
+                for (const fn of waitingForOutput) {
+                    fn();
+                }
+            }
+        });
+        this.exitPromise = new Promise((resolve, reject) => {
+            process.on("error", reject);
+            process.on("exit", resolve);
+        });
+    }
+    async ok() {
+        const exitCode = await this.exitPromise;
+        if (exitCode !== 0) {
+            throw new Error(`Command exited with code ${exitCode}\n\nOutput:\n${this.output}`);
+        }
+    }
+    async end() {
+        return await this.exitPromise;
+    }
+    async kill() {
+        const pid = this.process.pid;
+        await new Promise((resolve, reject) => treeKill(pid, (err) => {
+            if (err)
+                reject(err);
+            else
+                resolve();
+        }));
+        await this.exitPromise;
+    }
+    async waitForOutput(regex) {
+        let start = this.output.length;
+        while (true) {
+            const match = this.output.slice(start).match(regex);
+            if (match) {
+                return match;
+            }
+            const waitResult = await Promise.race([
+                this.exitPromise,
+                new Promise((resolve) => {
+                    this.waitingForOutput.push(resolve);
+                }).then(() => "output"),
+            ]);
+            if (waitResult !== "output") {
+                throw new Error(`Command exited with code ${waitResult}\n\nOutput:\n${this.output}`);
+            }
+        }
+    }
+    async reportMemUsage(metricName, options = {}) {
+        try {
+            const pid = this.process.pid;
+            const report = await pidusage(pid);
+            const memUsage = Object.values(report)
+                .filter((x) => x)
+                .map((x) => x.memory)
+                .reduce((a, b) => a + b, 0);
+            reportMeasurement(metricName, memUsage, "bytes", options);
+        }
+        catch (e) {
+            // ignore
+        }
+    }
+}
+export function command(command, args, options = {}) {
+    const process = spawn(command, args, {
+        shell: true,
+        ...options,
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+    if (shellOutput) {
+        console.log(`[SHELL] ${command} ${args.join(" ")} ${JSON.stringify(options)}`);
+    }
+    return new CommandImpl(process);
+}

package/dist/table.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/table.js

@@ -0,0 +1,117 @@
+import { readFile } from "fs/promises";
+import minimist from "minimist";
+(async () => {
+    const args = minimist(process.argv.slice(2), {
+        alias: {
+            r: "row",
+            c: "column",
+            "?": "help",
+            h: "help",
+        },
+    });
+    const knownArgs = new Set(["row", "r", "column", "c", "help", "h", "?", "_"]);
+    if (args.help || (Object.keys(args).length === 1 && args._.length === 0)) {
+        console.log("Usage: devlow-table <data.json>");
+        console.log("  --row=<name>      Key to show as row");
+        console.log("  --column=<name>   Key to show as column");
+        console.log("  --<prop>=<name>   Filter values");
+        console.log("  --help, -h, -?    Show this help");
+    }
+    let data = JSON.parse(await readFile(args._[0], "utf-8"));
+    const getValue = (data, name, includeKey) => {
+        if (name === "value") {
+            return data.text;
+        }
+        if (Array.isArray(name)) {
+            return name
+                .map((n) => getValue(data, n, true))
+                .filter((x) => x)
+                .join(" ");
+        }
+        const value = data.key[name];
+        if (value === undefined)
+            return "";
+        if (value === true)
+            return includeKey ? name : "true";
+        if (value === false)
+            return includeKey ? "" : "false";
+        if (value === null)
+            return "";
+        if (includeKey)
+            return `${name}=${value}`;
+        return value + "";
+    };
+    for (const [key, value] of Object.entries(args)) {
+        if (knownArgs.has(key))
+            continue;
+        const values = (Array.isArray(value) ? value : [value]).map((v) => v.toString());
+        data = data.filter((item) => {
+            const itemValue = getValue(item, key, false);
+            if (itemValue === "")
+                return false;
+            return values.some((v) => itemValue === v);
+        });
+    }
+    if (data.length === 0) {
+        console.log("No data");
+        return;
+    }
+    const row = args.row || "name";
+    const column = args.column || "scenario";
+    const getRow = (data) => getValue(data, row, false);
+    const getColumn = (data) => getValue(data, column, false);
+    const allRows = new Set(data.map(getRow));
+    const allColumns = new Set(data.map(getColumn));
+    const table = [];
+    const columnSizes = [...allColumns].map((c) => c.length);
+    for (const row of allRows) {
+        const rowData = [];
+        let i = 0;
+        for (const column of allColumns) {
+            let items = data
+                .filter((d) => getRow(d) === row && getColumn(d) === column)
+                .map((i) => i.text);
+            rowData.push(items.join(", "));
+            columnSizes[i] = Math.max(columnSizes[i], rowData[i].length);
+            i++;
+        }
+        table.push(rowData);
+    }
+    const pad = (str, size) => {
+        return " ".repeat(size - str.length) + str;
+    };
+    const firstColumnSize = Math.max(...[...allRows].map((r) => r.length));
+    // Header
+    {
+        let row = "| ";
+        let sepRow = "|:";
+        row += " ".repeat(firstColumnSize);
+        sepRow += "-".repeat(firstColumnSize);
+        const allColumnsArray = [...allColumns];
+        for (let i = 0; i < columnSizes.length; i++) {
+            row += " | ";
+            row += pad(allColumnsArray[i], columnSizes[i]);
+            sepRow += ":|-";
+            sepRow += "-".repeat(columnSizes[i]);
+        }
+        row += " |";
+        sepRow += ":|";
+        console.log(row);
+        console.log(sepRow);
+    }
+    // Separator
+    let r = 0;
+    for (const rowName of allRows) {
+        let row = "| ";
+        row += pad(rowName, firstColumnSize);
+        for (let i = 0; i < columnSizes.length; i++) {
+            row += " | ";
+            row += pad(table[r][i], columnSizes[i]);
+        }
+        row += " |";
+        console.log(row);
+        r++;
+    }
+})().catch((e) => {
+    console.error(e.stack);
+});

package/dist/units.d.ts

@@ -0,0 +1 @@
+export declare function formatUnit(value: number, unit: string): string;

package/dist/units.js

@@ -0,0 +1,24 @@
+const UNITS = {
+    ms: {
+        " s": 1000,
+    },
+    bytes: {
+        " GB": 1024 * 1024 * 1024,
+        " MB": 1024 * 1024,
+        " KB": 1024,
+    },
+    requests: {
+        "K requests": 1000,
+    },
+};
+export function formatUnit(value, unit) {
+    const conversion = UNITS[unit];
+    if (conversion) {
+        for (const [name, factor] of Object.entries(conversion)) {
+            if (value >= factor) {
+                return `${(value / factor).toFixed(2)}${name}`;
+            }
+        }
+    }
+    return `${value.toFixed(2).replace(/\.00$/, "")} ${unit}`;
+}

package/dist/utils.d.ts

@@ -0,0 +1 @@
+export declare function formatVariant(scenario: string, props: Record<string, string | number | boolean | null>): string;

package/dist/utils.js

@@ -0,0 +1,9 @@
+export function formatVariant(scenario, props) {
+    const keys = Object.keys(props)
+        .filter((key) => props[key] !== false && props[key] !== null)
+        .map((key) => (props[key] === true ? key : `${key}=${props[key]}`));
+    if (keys.length === 0) {
+        return scenario;
+    }
+    return `${scenario} ${keys.join(" ")}`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vercel/devlow-bench",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "description": "Benchmarking tool for the developer workflow",
   "type": "module",
   "main": "dist/index.js",