@hona/strobe-bench 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,38 @@
-# Temporary Holding Version
+# @hona/strobe-bench
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Types and helpers for [strobe](https://github.com/Hona/strobe) bench files.
+
+strobe runs a bench file with its own copy of this package, so a bench file runs without installing
+anything. Install the package for your editor and `tsc`: the types catch a renamed option before a
+long run.
+
+```sh
+bun add -d @hona/strobe-bench
+```
+
+```ts
+import { bench, web } from "@hona/strobe-bench"
+
+export default bench({
+  targets: { app: web({ url: "/" }) },
+  cases: {
+    "open settings": {
+      primary: "main.work",
+      run: (ctx) => ctx.page.click("[data-action=settings]", { until: "[role=dialog]" }),
+      reset: (ctx) => ctx.page.press("Escape", { until: "idle" }),
+    },
+  },
+})
+```
+
+```sh
+strobe bench perf/settings.bench.ts --check            # each case once: does it work?
+strobe bench perf/settings.bench.ts --base git:main    # is this branch faster?
+```
+
+- **Targets:** `web()`, `url()`, `app()`, `exe()`, `command()` (a dev command that launches an app)
+  and `script()` (a Bun or Node module whose exports cases call).
+- **Cases:** `run` measures work inside an open target. `launch` measures the launch itself, from
+  process spawn to a selector, with milestones on the way.
+
+The package version matches the strobe version whose SDK it describes.

package/dist/index.d.ts

@@ -0,0 +1,290 @@
+/** A visible element: a selector, or one whose text contains `text` (or is exactly it, trimmed). */
+export type Selector = string | {
+    selector: string;
+    text?: string;
+    exact?: boolean;
+};
+/** When an input call resolves: `"idle"` (no DOM change or long frame for 300 ms), `"frame"` (after next paint), or a visible element. */
+export type Until = "idle" | "frame" | Selector;
+export interface Options {
+    until?: Until;
+    /** Milliseconds before the call fails the block. Default 30000. */
+    timeout?: number;
+}
+export interface Page {
+    /** Opens `url`; a relative path resolves against the current page. */
+    goto(url: string, opts?: Options): Promise<void>;
+    click(selector: string, opts?: Options): Promise<void>;
+    /** Moves the pointer onto `selector` without a click: reveals hover-only controls, or parks the pointer. */
+    hover(selector: string, opts?: Options): Promise<void>;
+    /** Moves the pointer to a point in the viewport (CSS pixels), such as away from anything that reacts to hover. */
+    move(x: number, y: number, opts?: Options): Promise<void>;
+    /** Focuses `selector` (clicking it if focus is elsewhere), then sends one key event per character. */
+    type(selector: string, text: string, opts?: Options & {
+        delay?: number;
+    }): Promise<void>;
+    /** A key or chord: `"Enter"`, `"Control+a"`, `"ControlOrMeta+Shift+P"`. */
+    press(key: string, opts?: Options): Promise<void>;
+    scroll(selector: string, delta: {
+        by?: number;
+        byX?: number;
+        speed?: number;
+    }, opts?: Options): Promise<void>;
+    /** Resolves when a visible element matches `selector` (holding `text`, if given). Re-checked on every DOM and focus change, so `:focus` works. */
+    waitFor(selector: string, opts?: {
+        text?: string;
+        exact?: boolean;
+        timeout?: number;
+    }): Promise<void>;
+    /** Resolves when nothing in the DOM changed and no frame ran long for 300 ms. */
+    idle(opts?: {
+        timeout?: number;
+    }): Promise<void>;
+    wait(ms: number): Promise<void>;
+    /** Runs `fn` in the page. Only in `prepare`, `setup` and `reset`: it costs main-thread time. */
+    eval<A extends unknown[], T>(fn: (...args: A) => T, ...args: A): Promise<Awaited<T>>;
+}
+export interface Context<P> {
+    page: Page;
+    /** The case this block runs: a block always belongs to one case. */
+    case: string;
+    /** Marks a named span inside a run; every metric can be sliced by it. */
+    phase<T>(name: string, fn: () => T | Promise<T>): Promise<T>;
+    /** Adds `n` (default 1) to the `count.<name>` metric of the current run. Outside a run it counts nothing. */
+    count(name: string, n?: number): void;
+    /** Calls an export of a `script()` target's module in its process and returns what it returned (JSON). */
+    call<T = unknown>(name: string, ...args: unknown[]): Promise<T>;
+    params: P;
+    target: string;
+    variant: "base" | "head";
+    /** The URL this block opened (empty for app, command and script targets). */
+    url: string;
+    /** The variant's folder: a git ref's worktree, or the directory it names. */
+    root: string;
+    /** What the target's `prepare` returned for this block. */
+    prepared: any;
+}
+/** A request a `mock` handler answers. */
+export interface MockRequest {
+    url: string;
+    method: string;
+    headers: Record<string, string>;
+    /** The request body as text, if it had one. */
+    body?: string;
+    json<T = any>(): T;
+}
+/** A mocked response: `body` as text, bytes, or any other value as JSON. */
+export interface MockResponse {
+    status?: number;
+    headers?: Record<string, string>;
+    body?: string | Uint8Array | unknown;
+}
+/** Answers a request, or returns `undefined` to let it through to the network. */
+export type MockHandler = (request: MockRequest) => MockResponse | undefined | Promise<MockResponse | undefined>;
+export interface PrepareContext<P> {
+    /** The case this block runs, so one file can give each case its own fixture. */
+    case: string;
+    params: P;
+    target: string;
+    variant: "base" | "head";
+    /** The URL strobe opens after `prepare`, such as the variant's server; mock its API under `new URL(url).origin`. */
+    url: string;
+    /** The variant's folder: a git ref's worktree, or the directory it names. */
+    root: string;
+    /** Playwright, connected to this block's browser. Only on targets with a debug port. */
+    playwright: {
+        browser: any;
+        context: any;
+        page: any;
+    };
+    /**
+     * Answers requests whose URL matches `pattern` (`*` is any run of characters) in this block's page.
+     * Unlike Playwright's `page.route`, the browser's HTTP cache stays on: scripts and styles a reload
+     * fetches come from the cache, as for a person. Later calls add patterns and are asked first; a
+     * handler that returns `undefined` passes the request to earlier ones, as Playwright's routes do.
+     */
+    mock(pattern: string, handler: MockHandler): Promise<void>;
+}
+/** What a launch's `before` hook gets: the launch has not started yet. */
+export interface LaunchContext<P> {
+    case: string;
+    params: P;
+    target: string;
+    variant: "base" | "head";
+    /** The variant's folder: a git ref's worktree, or the directory it names. */
+    root: string;
+    /** The private home the app launches in (its `HOME`, `APPDATA`, XDG folders); null for web targets. Ready before the hook: seed and profile files are in. */
+    home: string | null;
+    /** App and exe targets: the whole environment the app launches with, to start what it expects to find (a background service) as it would. */
+    env: Record<string, string> | null;
+    /** App and exe targets: the executable this variant launches. */
+    exe: string | null;
+}
+/** A point in a launch: the first line matching `match` in `in`, timed from process spawn. */
+export interface Milestone {
+    /** Text the line contains, or a regular expression it matches. A group named `at` holding epoch ms (`/entry: (?<at>\d{13})/`) is the time, for marks an app writes into its log; otherwise the time strobe read the line. */
+    match: string | RegExp;
+    /** `"output"` (stdout and stderr, the default), `"page"` (the page's `performance.mark` names), or a glob of files under the launch's home. */
+    in?: "output" | "page" | (string & {});
+}
+export interface Launch<P> {
+    /** The launch is ready when this is visible in any of its windows. The primary is `launch.ready`. */
+    until: Selector;
+    /** Named points on the way, each a metric `launch.<name>`. One a case declares must appear in every launch. */
+    milestones?: Record<string, Milestone>;
+    /** `fresh` (default): a new home every launch. `warm`: one home per series and variant, kept across its launches, so caches and state from earlier launches are there. */
+    profile?: "fresh" | "warm";
+    /** Before every launch, never timed: delete caches for a cold start, start or stop a service, seed the home. */
+    before?(ctx: LaunchContext<P>): unknown;
+    /** Starts the main process with `--inspect` and reads Node's own bootstrap timing after the launch (`launch.node.*`). App and exe targets. */
+    inspect?: boolean;
+    /** Times `launch.painted` and `launch.settled` from the window's pixels (Windows; default on for apps). */
+    screen?: boolean;
+    /** Milliseconds before a launch that never shows `until` fails. Default 120000. */
+    timeout?: number;
+}
+interface CaseBase {
+    /** The metric this case is about, declared before the run. */
+    primary: string;
+    /** Extra secondary metrics to report: any catalog metric, `count.<name>`, `net.requests{<url glob>}`, `layout.shift`, `launch.<milestone>`, `js.heap.retained` (scripts). */
+    metrics?: string[];
+    /** The targets this case runs on. Default: every target. */
+    targets?: string[];
+}
+/** A case of runs inside an open target: measured `run`, unmeasured `reset`. */
+export interface RunCase<P> extends CaseBase {
+    /** Measured. */
+    run(ctx: Context<P>): unknown;
+    /** Puts the app back between runs. Never measured. */
+    reset?(ctx: Context<P>): unknown;
+    launch?: never;
+}
+/** A case that measures the launch itself: one cold launch per block. */
+export interface LaunchCase<P> extends CaseBase {
+    launch: Launch<P>;
+    run?: never;
+    reset?: never;
+}
+export type Case<P> = RunCase<P> | LaunchCase<P>;
+/** The first page a block opens: a path, or a function of what `prepare` returned. */
+export type FirstPage<P> = string | ((ctx: {
+    case: string;
+    prepared: any;
+    params: P;
+}) => string | Promise<string>);
+interface Hooks<P> {
+    prepare?: (ctx: PrepareContext<P>) => unknown;
+    /** When the block closes, for what `prepare` started (a server, a temporary folder). */
+    teardown?: (ctx: PrepareContext<P>) => unknown;
+}
+export type Target<P> = ({
+    kind: "web";
+    url: FirstPage<P>;
+    viewport?: Viewport;
+    scale?: number;
+} & Hooks<P>) | ({
+    kind: "url";
+    href: string;
+    viewport?: Viewport;
+    scale?: number;
+} & Hooks<P>) | {
+    kind: "app";
+    id: string;
+    home?: string;
+    viewport?: Viewport;
+} | {
+    kind: "exe";
+    path: string;
+    app?: string;
+    home?: string;
+    viewport?: Viewport;
+} | ({
+    kind: "command";
+} & CommandOptions) | ({
+    kind: "script";
+} & ScriptOptions);
+export interface Viewport {
+    width: number;
+    height: number;
+}
+export interface CommandOptions {
+    /** The command line, run by the shell in `cwd`, such as `bun run dev:desktop`. */
+    run: string;
+    /** Relative to the variant's folder. Default: the folder itself. */
+    cwd?: string;
+    /** Extra environment; `{home}` (the launch's private home) and `{port}` (a free port) are filled in. */
+    env?: Record<string, string>;
+    /** Environment variables to remove; `PREFIX_*` removes a family. */
+    unset?: string[];
+    /** The text before the DevTools endpoint in the command's output. Default `"DevTools listening on"`. */
+    attach?: string;
+    /** Milliseconds the command may take to print its endpoint. Default 600000. */
+    timeout?: number;
+    /** Run once per variant before its first block, in its folder. */
+    build?: string;
+    viewport?: Viewport;
+}
+export interface ScriptOptions {
+    /**
+     * The harness module whose exports cases call (`ctx.call`), relative to the bench file. strobe
+     * runs the same file for both variants, placed at the same path in a git ref's worktree, so its
+     * relative imports reach that variant's code. Keep the harness in one file.
+     */
+    entry: string;
+    /** Default `"bun"`. */
+    runtime?: "bun" | "node";
+    env?: Record<string, string>;
+    /** Run once per variant before its first block, in its folder. */
+    build?: string;
+}
+/** A parameter's values: all run, or `run` by default and `more` with `--params all`. */
+export type ParamValues = readonly unknown[] | {
+    readonly run: readonly unknown[];
+    readonly more?: readonly unknown[];
+};
+type Values<T> = {
+    [K in keyof T]: T[K] extends readonly (infer V)[] ? V : T[K] extends {
+        run: readonly (infer V)[];
+        more: readonly (infer W)[];
+    } ? V | W : T[K] extends {
+        run: readonly (infer V)[];
+    } ? V : never;
+};
+export interface Bench<Params extends Record<string, ParamValues>> {
+    targets: Record<string, Target<Values<Params>>>;
+    /** Every combination is its own series. */
+    params?: Params;
+    /** Once per block, after load. Never measured. */
+    setup?(ctx: Context<Values<Params>>): unknown;
+    /** Once per block, when it closes. Never measured. */
+    teardown?(ctx: Context<Values<Params>>): unknown;
+    cases: Record<string, Case<Values<Params>>>;
+}
+/** The project's production build, served per variant from `strobe.json`, in a strobe-launched Chromium. */
+export declare function web<P = any>(opts?: {
+    url?: FirstPage<P>;
+    viewport?: Viewport;
+    scale?: number;
+} & Hooks<P>): Target<P>;
+/** A fixed page; the same for both variants unless a variant is `url:<href>`. */
+export declare function url<P = any>(href: string, opts?: {
+    viewport?: Viewport;
+    scale?: number;
+} & Hooks<P>): Target<P>;
+/** An installed Electron app, by its profile in strobe.json `apps`, in a fresh isolated home seeded from `home`. */
+export declare function app<P = any>(id: string, opts?: {
+    home?: string;
+    viewport?: Viewport;
+}): Target<P>;
+/** A packaged Electron app by path; `app` borrows a profile from strobe.json `apps` for it. */
+export declare function exe<P = any>(path: string, opts?: {
+    app?: string;
+    home?: string;
+    viewport?: Viewport;
+}): Target<P>;
+/** An app started by a command in the variant's folder (a dev server that launches it), attached through the DevTools endpoint it prints. */
+export declare function command<P = any>(run: string, opts?: Omit<CommandOptions, "run">): Target<P>;
+/** A Bun or Node module of the variant, loaded in a fresh process per block; cases call its exports with `ctx.call`. */
+export declare function script<P = any>(entry: string, opts?: Omit<ScriptOptions, "entry">): Target<P>;
+export declare function bench<const Params extends Record<string, ParamValues> = {}>(spec: Bench<Params>): Bench<Params>;
+export {};

package/dist/index.js

@@ -0,0 +1,31 @@
+// src/index.ts
+function web(opts = {}) {
+  return { kind: "web", url: opts.url ?? "/", prepare: opts.prepare, teardown: opts.teardown, viewport: opts.viewport, scale: opts.scale };
+}
+function url(href, opts = {}) {
+  return { kind: "url", href, ...opts };
+}
+function app(id, opts = {}) {
+  return { kind: "app", id, ...opts };
+}
+function exe(path, opts = {}) {
+  return { kind: "exe", path, ...opts };
+}
+function command(run, opts = {}) {
+  return { kind: "command", run, ...opts };
+}
+function script(entry, opts = {}) {
+  return { kind: "script", entry, ...opts };
+}
+function bench(spec) {
+  return spec;
+}
+export {
+  app,
+  bench,
+  command,
+  exe,
+  script,
+  url,
+  web
+};

package/package.json

@@ -1,6 +1,29 @@
 {
   "name": "@hona/strobe-bench",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Types and helpers for strobe bench files: measure web pages, Electron apps and scripts with paired, valid statistics",
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": ["dist", "README.md"],
+  "sideEffects": false,
+  "scripts": {
+    "build": "bun build src/index.ts --outdir dist --target node --format esm && tsc -p tsconfig.json",
+    "prepublishOnly": "bun run build"
+  },
+  "devDependencies": {
+    "typescript": "^5.9.0"
+  },
+  "keywords": ["benchmark", "performance", "strobe", "playwright", "electron", "chromium"],
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  }
+}