@vercel/devlow-bench 0.3.4 → 16.4.0-canary.24

package/dist/interfaces/json.js

@@ -1,5 +1,6 @@
-import { formatUnit } from "../units.js";
-import { writeFile } from "fs/promises";
+import { quantile, mean as statsMean } from '../statistics.js';
+import { formatUnit } from '../units.js';
+import { writeFile } from 'fs/promises';
 function filterProp(prop) {
     const filteredProp = {};
     for (const [key, value] of Object.entries(prop)) {
@@ -10,12 +11,14 @@ function filterProp(prop) {
     return filteredProp;
 }
 export default function createInterface(file = (() => {
-    const file = process.env.JSON_OUTPUT_FILE;
-    if (!file) {
-        throw new Error("env var JSON_OUTPUT_FILE is not set");
+    const fromEnv = process.env.JSON_OUTPUT_FILE;
+    if (!fromEnv) {
+        throw new Error('env var JSON_OUTPUT_FILE is not set');
     }
-    return file;
-})()) {
+    return fromEnv;
+})(), options = {}) {
+    const n = options.n ?? 1;
+    // Per-(scenario, props, name) sample accumulator.
     const metrics = new Map();
     const iface = {
         measurement: async (scenario, props, name, value, unit, relativeTo) => {
@@ -27,30 +30,46 @@ export default function createInterface(file = (() => {
             const key = JSON.stringify(keyObject);
             const current = metrics.get(key);
             if (current) {
-                current.value += value;
-                current.count++;
+                current.samples.push(value);
             }
             else {
                 metrics.set(key, {
                     key: keyObject,
-                    value,
-                    unit: unit,
-                    count: 1,
+                    samples: [value],
+                    unit,
                     relativeTo,
                 });
             }
         },
         finish: async () => {
-            await writeFile(file, JSON.stringify([...metrics.values()].map(({ key, value, unit, count, relativeTo }) => {
+            const results = [...metrics.values()].map(({ key, samples, unit, relativeTo }) => {
+                if (samples.length === 1) {
+                    // Preserve the original single-run shape exactly for back-compat.
+                    return {
+                        key,
+                        value: samples[0],
+                        unit,
+                        text: formatUnit(samples[0], unit),
+                        datapoints: 1,
+                        relativeTo,
+                    };
+                }
+                const m = statsMean(samples);
                 return {
                     key,
-                    value: value / count,
+                    value: m,
                     unit,
-                    text: formatUnit(value / count, unit),
-                    datapoints: count,
+                    text: formatUnit(m, unit),
+                    datapoints: samples.length,
+                    mean: m,
+                    p50: quantile(samples, 0.5),
+                    p90: quantile(samples, 0.9),
+                    samples,
                     relativeTo,
                 };
-            })));
+            });
+            const payload = n > 1 ? { results } : results;
+            await writeFile(file, JSON.stringify(payload));
         },
     };
     return iface;

package/dist/interfaces/shell-test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/interfaces/shell-test.js

@@ -0,0 +1,56 @@
+import assert from 'assert';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { test } from 'node:test';
+import { command } from '../shell.js';
+async function withTempScript(scriptContents, fn) {
+    const dir = await mkdtemp(join(tmpdir(), 'devlow-shell-test-'));
+    const scriptPath = join(dir, 'script.js');
+    try {
+        await writeFile(scriptPath, scriptContents, 'utf8');
+        await fn(scriptPath);
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+}
+test('waitForOutput handles sequential waits after buffered output', async () => {
+    await withTempScript(`
+console.log('FIRST_MARKER')
+console.log('SECOND_MARKER')
+setTimeout(() => {}, 1000)
+`, async (scriptPath) => {
+        const shell = command('node', [scriptPath]);
+        try {
+            const first = await shell.waitForOutput(/FIRST_MARKER\n/, {
+                timeoutMs: 2_000,
+            });
+            assert.equal(first[0], 'FIRST_MARKER\n');
+            const second = await shell.waitForOutput(/SECOND_MARKER\n/, {
+                timeoutMs: 2_000,
+            });
+            assert.equal(second[0], 'SECOND_MARKER\n');
+        }
+        finally {
+            await shell.kill();
+        }
+    });
+});
+test('waitForOutput times out when output never appears', async () => {
+    await withTempScript(`
+console.log('FIRST_MARKER')
+setTimeout(() => {}, 1000)
+`, async (scriptPath) => {
+        const shell = command('node', [scriptPath]);
+        try {
+            await shell.waitForOutput(/FIRST_MARKER\n/, {
+                timeoutMs: 2_000,
+            });
+            await assert.rejects(shell.waitForOutput(/NEVER_SEEN\n/, { timeoutMs: 100 }), /Timed out waiting for output matching/);
+        }
+        finally {
+            await shell.kill();
+        }
+    });
+});

package/dist/interfaces/snapshot.d.ts

@@ -0,0 +1,6 @@
+import type { Interface } from '../index.js';
+export default function createInterface(options?: {
+    path?: string;
+}): Interface & {
+    resolvedPath: string;
+};

package/dist/interfaces/snapshot.js

@@ -0,0 +1,60 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import picocolors from 'picocolors';
+import { defaultSnapshotPath, writeSnapshot, } from '../snapshot.js';
+import { formatVariantProps } from '../utils.js';
+const execFileAsync = promisify(execFile);
+async function readGitInfo() {
+    const sha = process.env.GITHUB_SHA || (await tryGit(['rev-parse', 'HEAD']));
+    const branch = process.env.GITHUB_REF_NAME ||
+        (await tryGit(['rev-parse', '--abbrev-ref', 'HEAD']));
+    return { sha, branch };
+}
+async function tryGit(args) {
+    try {
+        const { stdout } = await execFileAsync('git', args);
+        return stdout.trim();
+    }
+    catch {
+        return '';
+    }
+}
+export default function createInterface(options = {}) {
+    const path = options.path ?? defaultSnapshotPath();
+    const rows = [];
+    const timestamp = new Date().toISOString();
+    const iface = {
+        resolvedPath: path,
+        variantStatistics: async (scenario, props, stats) => {
+            const variant = formatVariantProps(props);
+            for (const [metric, s] of Object.entries(stats)) {
+                for (let i = 0; i < s.samples.length; i++) {
+                    rows.push({
+                        timestamp,
+                        scenario,
+                        variant,
+                        metric,
+                        sample_idx: i + 1,
+                        value: s.samples[i],
+                        unit: s.unit,
+                        relative_to: s.relativeTo ?? '',
+                        git_sha: '',
+                        git_branch: '',
+                    });
+                }
+            }
+        },
+        finish: async () => {
+            if (rows.length === 0)
+                return;
+            const { sha, branch } = await readGitInfo();
+            for (const r of rows) {
+                r.git_sha = sha;
+                r.git_branch = branch;
+            }
+            await writeSnapshot(path, rows);
+            console.log(picocolors.dim(`Wrote ${rows.length} rows to ${path}`));
+        },
+    };
+    return iface;
+}

package/dist/interfaces/snowflake-test.js

@@ -1,6 +1,6 @@
-import assert from "assert";
-import createInterface from "./snowflake.js";
-import { mock, test } from "node:test";
+import assert from 'assert';
+import createInterface from './snowflake.js';
+import { mock, test } from 'node:test';
 test('sending measurements to the batch endpoint', async () => {
     const requests = [];
     const fetchMock = mock.method(global, 'fetch', (req, options) => {

package/dist/interfaces/snowflake.d.ts

@@ -1,4 +1,4 @@
-import type { Interface } from "../index.js";
+import type { Interface } from '../index.js';
 export default function createInterface({ gatewayUri, topicName, schemaId, }?: {
     gatewayUri?: string;
     topicName?: string;

package/dist/interfaces/snowflake.js

@@ -1,13 +1,15 @@
-import os from "os";
-import { CPU_ARCH, CPU_MODEL, GIT_BRANCH, GIT_SHA, IS_CI, NODE_VERSION, NUM_CPUS, OS, OS_RELEASE, USERNAME } from "./constants.js";
-import { randomUUID } from "crypto";
-export default function createInterface({ gatewayUri = process.env.SNOWFLAKE_BATCH_URI, topicName = process.env.SNOWFLAKE_TOPIC_NAME, schemaId = process.env.SNOWFLAKE_SCHEMA_ID ? parseInt(process.env.SNOWFLAKE_SCHEMA_ID, 10) : undefined, } = {}) {
+import os from 'os';
+import { CPU_ARCH, CPU_MODEL, GIT_BRANCH, GIT_SHA, IS_CI, NODE_VERSION, NUM_CPUS, OS, OS_RELEASE, USERNAME, } from './constants.js';
+import { randomUUID } from 'crypto';
+export default function createInterface({ gatewayUri = process.env.SNOWFLAKE_BATCH_URI, topicName = process.env.SNOWFLAKE_TOPIC_NAME, schemaId = process.env.SNOWFLAKE_SCHEMA_ID
+    ? parseInt(process.env.SNOWFLAKE_SCHEMA_ID, 10)
+    : undefined, } = {}) {
     if (!gatewayUri)
-        throw new Error("Snowflake gateway URI is required (set SNOWFLAKE_GATEWAY_URI)");
+        throw new Error('Snowflake gateway URI is required (set SNOWFLAKE_GATEWAY_URI)');
     if (!topicName)
-        throw new Error("Snowflake topic name is required (set SNOWFLAKE_TOPIC_NAME)");
+        throw new Error('Snowflake topic name is required (set SNOWFLAKE_TOPIC_NAME)');
     if (!schemaId)
-        throw new Error("Snowflake schema ID is required (set SNOWFLAKE_SCHEMA_ID to a valid integer)");
+        throw new Error('Snowflake schema ID is required (set SNOWFLAKE_SCHEMA_ID to a valid integer)');
     const records = [];
     const iface = {
         measurement: async (scenario, props, name, value, unit, relativeTo) => {
@@ -32,7 +34,7 @@ export default function createInterface({ gatewayUri = process.env.SNOWFLAKE_BAT
                 git_branch: GIT_BRANCH,
             });
         },
-        end: async (scenario, props) => {
+        end: async (_scenario, _props) => {
             await trackAnalytics(gatewayUri, topicName, schemaId, records);
         },
     };
@@ -41,10 +43,10 @@ export default function createInterface({ gatewayUri = process.env.SNOWFLAKE_BAT
 async function trackAnalytics(batchUri, topic, schemaId, records) {
     try {
         const res = await fetch(batchUri, {
-            method: "POST",
+            method: 'POST',
             headers: {
-                "Client-Id": "nextjs",
-                "Content-Type": "application/json",
+                'Client-Id': 'nextjs',
+                'Content-Type': 'application/json',
             },
             body: JSON.stringify({
                 schema_id: schemaId,
@@ -53,7 +55,7 @@ async function trackAnalytics(batchUri, topic, schemaId, records) {
                     ...record,
                     id: randomUUID(),
                     props_json: JSON.stringify(record.props),
-                }, ["props"])),
+                }, ['props'])),
             }),
         });
         if (!res.ok) {
@@ -61,7 +63,7 @@ async function trackAnalytics(batchUri, topic, schemaId, records) {
         }
     }
     catch (e) {
-        const wrappedError = new Error("Unexpected error tracking analytics event");
+        const wrappedError = new Error('Unexpected error tracking analytics event');
         wrappedError.cause = e;
         console.error(wrappedError);
     }

package/dist/runner.d.ts

@@ -1,2 +1,5 @@
-import { Interface, Scenario } from "./index.js";
-export declare function runScenarios(scenarios: Scenario[], iface: Interface): Promise<void>;
+import type { Interface, Scenario } from './index.js';
+export declare function runScenarios(scenarios: Scenario[], iface: Interface, options?: {
+    n?: number;
+    warmup?: number;
+}): Promise<void>;

package/dist/runner.js

@@ -1,20 +1,31 @@
-import { withCurrent } from "./describe.js";
-import { intoFullInterface } from "./index.js";
-export async function runScenarios(scenarios, iface) {
+import { withCurrent } from './describe.js';
+import { intoFullInterface } from './index.js';
+import { summary } from './statistics.js';
+import { formatVariant } from './utils.js';
+// Total attempts per variant are capped at this multiple of the requested
+// `warmup + n`. Allows retrying flaky runs without burning unbounded time when
+// every attempt fails.
+const MAX_ATTEMPT_MULTIPLIER = 2;
+export async function runScenarios(scenarios, iface, options = {}) {
+    const n = Math.max(1, Math.floor(options.n ?? 1));
+    const warmup = Math.max(0, Math.floor(options.warmup ?? 0));
     const fullIface = intoFullInterface(iface);
     if (scenarios.some((scenario) => scenario.only)) {
         scenarios = scenarios.filter((scenario) => scenario.only);
     }
     scenarios = await fullIface.filterScenarios(scenarios);
+    let totalFailedAttempts = 0;
+    let variantsWithRetries = 0;
+    let variantsShortOfTarget = 0;
     let variants = [];
     for (const scenario of scenarios) {
         let props = [{}];
-        for (const [key, options] of Object.entries(scenario.config)) {
+        for (const [key, values] of Object.entries(scenario.config)) {
             const newProps = [];
             for (const prop of props) {
-                if (prop === "scenario" || prop === "name")
+                if (prop === 'scenario' || prop === 'name')
                     throw new Error("Cannot use 'scenario' or 'name' as a property name");
-                for (const value of options) {
+                for (const value of values) {
                     newProps.push({
                         ...prop,
                         [key]: value,
@@ -23,32 +34,129 @@ export async function runScenarios(scenarios, iface) {
             }
             props = newProps;
         }
-        variants.push(...props.map((props) => ({
+        variants.push(...props.map((variantProps) => ({
             scenario,
-            props,
+            props: variantProps,
         })));
     }
     variants = await fullIface.filterScenarioVariants(variants);
     for (const variant of variants) {
-        try {
-            const measurements = new Map();
-            await withCurrent({
-                iface: fullIface,
-                measurements,
-                scenario: variant,
-            }, async () => {
-                await fullIface.start(variant.scenario.name, variant.props);
-                measurements.set("start", {
-                    value: Date.now(),
-                    unit: "ms",
+        const samplesByMetric = new Map();
+        // Each attempt buffers its measurements here. On success we merge into
+        // `samplesByMetric`; on failure we discard, so a partial run can't
+        // pollute the aggregated stats. Downstream per-sample reporters (Datadog,
+        // console, …) still receive every measurement inline — only the
+        // aggregated per-variant samples are gated on success.
+        let collecting = false;
+        let perRun = new Map();
+        const wrappedIface = {
+            ...fullIface,
+            measurement: async (scenario, props, name, value, unit, relativeTo) => {
+                if (collecting) {
+                    let list = perRun.get(name);
+                    if (!list) {
+                        list = [];
+                        perRun.set(name, list);
+                    }
+                    list.push({ value, unit, relativeTo });
+                }
+                await fullIface.measurement(scenario, props, name, value, unit, relativeTo);
+            },
+        };
+        const maxAttempts = (warmup + n) * MAX_ATTEMPT_MULTIPLIER;
+        const showRunInfo = warmup + n > 1;
+        let warmupDone = 0;
+        let collectedRuns = 0;
+        let totalAttempts = 0;
+        while ((warmupDone < warmup || collectedRuns < n) &&
+            totalAttempts < maxAttempts) {
+            totalAttempts++;
+            const isWarmup = warmupDone < warmup;
+            collecting = !isWarmup;
+            perRun = new Map();
+            const runInfo = showRunInfo
+                ? isWarmup
+                    ? { run: warmupDone + 1, total: warmup, warmup: true }
+                    : { run: collectedRuns + 1, total: n, warmup: false }
+                : undefined;
+            try {
+                const measurements = new Map();
+                await withCurrent({
+                    iface: wrappedIface,
+                    measurements,
+                    scenario: variant,
+                }, async () => {
+                    await wrappedIface.start(variant.scenario.name, variant.props, runInfo);
+                    measurements.set('start', {
+                        value: Date.now(),
+                        unit: 'ms',
+                    });
+                    await variant.scenario.fn(variant.props);
+                    await wrappedIface.end(variant.scenario.name, variant.props);
                 });
-                await variant.scenario.fn(variant.props);
-                await fullIface.end(variant.scenario.name, variant.props);
-            });
+                if (collecting) {
+                    // Commit the buffered samples for this run into the aggregate.
+                    for (const [name, entries] of perRun) {
+                        let set = samplesByMetric.get(name);
+                        if (!set) {
+                            const first = entries[0];
+                            set = {
+                                samples: [],
+                                unit: first.unit,
+                                relativeTo: first.relativeTo,
+                            };
+                            samplesByMetric.set(name, set);
+                        }
+                        for (const e of entries) {
+                            set.samples.push(e.value);
+                        }
+                    }
+                    collectedRuns++;
+                }
+                else {
+                    warmupDone++;
+                }
+            }
+            catch (e) {
+                await wrappedIface.error(variant.scenario.name, variant.props, e);
+                // Loop continues; the buffered samples for this attempt are dropped
+                // when `perRun` is replaced on the next iteration.
+            }
+        }
+        const failedAttempts = totalAttempts - warmupDone - collectedRuns;
+        if (failedAttempts > 0) {
+            const label = formatVariant(variant.scenario.name, variant.props);
+            console.log(`${label}: collected ${collectedRuns}/${n} samples after ${totalAttempts} attempts (${failedAttempts} failed)`);
+            totalFailedAttempts += failedAttempts;
+            variantsWithRetries++;
+        }
+        if (collectedRuns < n) {
+            process.exitCode = 1;
+            variantsShortOfTarget++;
+        }
+        if (samplesByMetric.size > 0) {
+            const stats = {};
+            for (const [name, set] of samplesByMetric) {
+                stats[name] = {
+                    samples: set.samples,
+                    unit: set.unit,
+                    relativeTo: set.relativeTo,
+                    ...summary(set.samples),
+                };
+            }
+            await fullIface.variantStatistics(variant.scenario.name, variant.props, stats);
+        }
+    }
+    if (totalFailedAttempts > 0) {
+        const recovered = variantsWithRetries - variantsShortOfTarget;
+        const parts = [];
+        if (recovered > 0) {
+            parts.push(`${recovered} variant${recovered === 1 ? '' : 's'} hit n=${n} after retries`);
         }
-        catch (e) {
-            await fullIface.error(variant.scenario.name, variant.props, e);
+        if (variantsShortOfTarget > 0) {
+            parts.push(`${variantsShortOfTarget} variant${variantsShortOfTarget === 1 ? '' : 's'} fell short of n=${n}`);
         }
+        console.log(`\n${totalFailedAttempts} failed attempt${totalFailedAttempts === 1 ? '' : 's'} across ${variantsWithRetries} variant${variantsWithRetries === 1 ? '' : 's'}: ${parts.join('; ')}.`);
     }
     await fullIface.finish();
 }

package/dist/shell.d.ts

@@ -1,9 +1,11 @@
-import { PREVIOUS } from "./describe.js";
+import type { PREVIOUS } from './describe.js';
 export interface Command {
     ok(): Promise<void>;
     kill(): Promise<void>;
     end(): Promise<number>;
-    waitForOutput(regex: RegExp): Promise<RegExpMatchArray>;
+    waitForOutput(regex: RegExp, options?: {
+        timeoutMs?: number;
+    }): Promise<RegExpMatchArray>;
     reportMemUsage(metricName: string, options: {
         relativeTo?: string | typeof PREVIOUS;
         scenario?: string;
@@ -13,7 +15,7 @@ export interface Command {
     stderr: string;
     output: string;
 }
-export declare function command(command: string, args: string[], options?: {
+export declare function command(executable: string, args: string[], options?: {
     env?: Record<string, string>;
     cwd?: string;
 }): Command;

package/dist/shell.js

@@ -1,22 +1,23 @@
-import { spawn } from "child_process";
-import split2 from "split2";
-import treeKill from "tree-kill";
-import pidusage from "pidusage-tree";
-import { reportMeasurement } from "./describe.js";
+import { spawn } from 'child_process';
+import split2 from 'split2';
+import treeKill from 'tree-kill';
+import pidusage from 'pidusage-tree';
+import { reportMeasurement } from './describe.js';
 const shellOutput = !!process.env.SHELL_OUTPUT;
 class CommandImpl {
     process;
-    stdout = "";
-    stderr = "";
-    output = "";
+    stdout = '';
+    stderr = '';
+    output = '';
+    outputCursor = 0;
     exitPromise;
     waitingForOutput = [];
     constructor(process) {
         this.process = process;
-        process.stdout?.pipe(split2()).on("data", (data) => {
+        process.stdout?.pipe(split2()).on('data', (data) => {
             const str = data.toString();
-            this.stdout += str + "\n";
-            this.output += str + "\n";
+            this.stdout += str + '\n';
+            this.output += str + '\n';
             if (shellOutput) {
                 console.log(`[STDOUT] ${str}`);
             }
@@ -28,10 +29,10 @@ class CommandImpl {
                 }
             }
         });
-        process.stderr?.pipe(split2()).on("data", (data) => {
+        process.stderr?.pipe(split2()).on('data', (data) => {
             const str = data.toString();
-            this.stderr += str + "\n";
-            this.output += str + "\n";
+            this.stderr += str + '\n';
+            this.output += str + '\n';
             if (shellOutput) {
                 console.log(`[STDERR] ${str}`);
             }
@@ -44,8 +45,8 @@ class CommandImpl {
             }
         });
         this.exitPromise = new Promise((resolve, reject) => {
-            process.on("error", reject);
-            process.on("exit", resolve);
+            process.on('error', reject);
+            process.on('exit', resolve);
         });
     }
     async ok() {
@@ -67,20 +68,47 @@ class CommandImpl {
         }));
         await this.exitPromise;
     }
-    async waitForOutput(regex) {
-        let start = this.output.length;
+    async waitForOutput(regex, options = {}) {
+        let start = this.outputCursor;
+        const deadline = options.timeoutMs === undefined
+            ? undefined
+            : Date.now() + options.timeoutMs;
         while (true) {
-            const match = this.output.slice(start).match(regex);
+            const outputToSearch = this.output.slice(start);
+            const match = outputToSearch.match(regex);
             if (match) {
+                const matchIndex = match.index ?? outputToSearch.search(regex);
+                if (matchIndex >= 0) {
+                    this.outputCursor = start + matchIndex + match[0].length;
+                }
+                else {
+                    this.outputCursor = this.output.length;
+                }
                 return match;
             }
-            const waitResult = await Promise.race([
+            const promises = [
                 this.exitPromise,
                 new Promise((resolve) => {
                     this.waitingForOutput.push(resolve);
-                }).then(() => "output"),
-            ]);
-            if (waitResult !== "output") {
+                }).then(() => 'output'),
+            ];
+            let timeoutId;
+            if (deadline !== undefined) {
+                const timeoutMs = deadline - Date.now();
+                if (timeoutMs <= 0) {
+                    throw new Error(`Timed out waiting for output matching ${regex}\n\nOutput:\n${this.output}`);
+                }
+                promises.push(new Promise((resolve) => {
+                    timeoutId = setTimeout(() => resolve('timeout'), timeoutMs);
+                }));
+            }
+            const waitResult = await Promise.race(promises);
+            if (timeoutId)
+                clearTimeout(timeoutId);
+            if (waitResult !== 'output') {
+                if (waitResult === 'timeout') {
+                    throw new Error(`Timed out waiting for output matching ${regex}\n\nOutput:\n${this.output}`);
+                }
                 throw new Error(`Command exited with code ${waitResult}\n\nOutput:\n${this.output}`);
             }
         }
@@ -93,21 +121,21 @@ class CommandImpl {
                 .filter((x) => x)
                 .map((x) => x.memory)
                 .reduce((a, b) => a + b, 0);
-            await reportMeasurement(metricName, memUsage, "bytes", options);
+            await reportMeasurement(metricName, memUsage, 'bytes', options);
         }
         catch (e) {
             // ignore
         }
     }
 }
-export function command(command, args, options = {}) {
-    const process = spawn(command, args, {
+export function command(executable, args, options = {}) {
+    const process = spawn(executable, args, {
         shell: true,
         ...options,
-        stdio: ["ignore", "pipe", "pipe"],
+        stdio: ['ignore', 'pipe', 'pipe'],
     });
     if (shellOutput) {
-        console.log(`[SHELL] ${command} ${args.join(" ")} ${JSON.stringify(options)}`);
+        console.log(`[SHELL] ${executable} ${args.join(' ')} ${JSON.stringify(options)}`);
     }
     return new CommandImpl(process);
 }