@soroush.tech/bench 0.1.0

package/Dockerfile

@@ -0,0 +1,10 @@
+# Sandbox image for @soroush.tech/bench.
+# Generic and content-stable: the harness and bench files are mounted at run
+# time, so this image is built once and cached across every benchmark.
+FROM node:24-slim
+
+# tsx runs the TypeScript harness and bench files. mitata is bundled into the
+# harness at build time, so it is not installed here.
+RUN npm install -g tsx@^4
+
+WORKDIR /repo

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Masoud Soroush <https://soroush.tech>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,143 @@
+# @soroush.tech/bench
+
+Run A/B function benchmarks inside a **CPU/RAM-pinned Docker sandbox** so the
+numbers stay stable run-to-run and are not skewed by whatever else the machine
+is doing. Built on [mitata](https://github.com/evanwashere/mitata).
+
+Compare:
+
+- two (or more) implementations of the same function, or
+- two-or-more **versions of one npm package** (e.g. `lodash@4` vs `lodash@5`).
+
+> **What pinning does and does not buy you.** Pinning to a single core
+> (`--cpuset-cpus`), a hard CPU quota (`--cpus`), and fixed memory with swap off
+> (`--memory` + `--memory-swap`) removes the noise from background load, turbo
+> boost variance, and paging — so repeated runs **on the same host** agree. It
+> does **not** make results identical across different physical CPUs (a
+> container shares the host kernel and silicon). For numbers that are identical
+> on any machine, count operations instead of measuring time.
+
+## Requirements
+
+Docker must be installed and running on the host. The sandbox image is built
+once and cached; subsequent runs only re-run the container.
+
+## Usage
+
+Write a `*.bench.ts` file that default-exports a `defineBench(...)` config:
+
+```ts
+// clone.bench.ts
+import defineBench from '@soroush.tech/bench'
+
+const data = { a: 1, nested: { b: [1, 2, 3] } }
+
+export default defineBench({
+  name: 'deep clone',
+  cases: {
+    structuredClone: () => structuredClone(data),
+    jsonRoundtrip: () => JSON.parse(JSON.stringify(data)),
+  },
+})
+```
+
+Run it in the pinned sandbox:
+
+```sh
+bench ./clone.bench.ts
+```
+
+### Comparing versions of one package
+
+Declare each version under an alias in `packages`; the harness installs them
+side by side and injects the resolved modules into every case via
+`ctx.modules`.
+
+### Comparing a local workspace package against npm
+
+Because the sandbox mounts the whole repo, a local workspace package is reached
+with a normal top-level import, while the npm release is installed under an
+alias. This benchmarks the local `@soroush.tech/styled-system` rewrite against
+the original `styled-system` it replaces:
+
+```ts
+// styled-system-color.bench.ts
+import { color } from '@soroush.tech/styled-system/color' // local source (mounted repo)
+import defineBench from '@soroush.tech/bench'
+
+const props = { theme: { colors: { brand: '#0af', muted: '#666' } }, color: 'brand', bg: 'muted' }
+type ColorFn = (props: typeof props) => unknown
+
+export default defineBench({
+  name: 'styled-system color()',
+  packages: { upstream: 'styled-system@5.1.5' }, // npm release
+  cases: {
+    local: () => color(props),
+    upstream: ({ modules }) => (modules.upstream as { color: ColorFn }).color(props),
+  },
+})
+```
+
+Run it from the repo root so the mount includes both the workspace package and
+its `node_modules` link:
+
+```sh
+bench ./examples/bench/styled-system-color.bench.ts
+```
+
+> The same shape compares **two-or-more versions of one npm package** — give
+> several aliases the same name at different versions (e.g.
+> `{ v4: 'lodash@4.17.21', v5: 'lodash@5.0.0' }`) and read each off `ctx.modules`.
+
+### Options
+
+| Flag       | Default               | Meaning                                  |
+| ---------- | --------------------- | ---------------------------------------- |
+| `--cpus`   | `1`                   | CPU quota (`docker run --cpus`).         |
+| `--cpuset` | `0`                   | Logical CPU(s) to pin to (`--cpuset-cpus`, e.g. `0,2`). |
+| `--memory` | `512m`                | Memory cap; swap is pinned to the same.  |
+| `--tag`    | `soroush-bench:latest`| Sandbox image tag.                       |
+| `--mount`  | —                     | Extra `host:container[:ro]` volume, passed through to `docker -v`. Repeatable. |
+
+### pnpm workspaces
+
+pnpm links workspace packages with **absolute** symlinks. Inside the sandbox
+those only resolve if the repo is also mounted at its host-visible path. With
+Docker Desktop that path is `/mnt/host/<drive>/…`, so to benchmark a local
+workspace package add:
+
+```sh
+bench ./examples/bench/styled-system-color.bench.ts \
+  --cpuset 0,2 --cpus 2 --memory 1g \
+  --mount "$PWD:/mnt/host/c/Users/you/your-repo:ro"
+```
+
+(npm/yarn flat `node_modules` need no extra mount — the package is copied, not
+symlinked out of the repo.)
+
+### Pinning on hybrid CPUs (P/E cores) & Docker Desktop
+
+On Intel 12th–14th gen (P-cores + E-cores), prefer a pin that avoids SMT
+siblings: `--cpuset 0` (a single thread — most reproducible, and enough since a
+mitata run is single-threaded) or `--cpuset 0,2` (two **distinct** physical
+cores), not `0-1` (which is the two hyperthreads of one core → SMT contention).
+Confirm a pin with `cat /sys/devices/system/cpu/cpu<N>/topology/thread_siblings_list`.
+
+Under **Docker Desktop (WSL2)** the hybrid topology is virtualized — `lscpu`
+reports a flat layout with no P/E distinction — so `--cpuset` selects VM vCPUs
+and the Windows scheduler/Thread Director decides their physical placement.
+Physical P-core residency is therefore **not** guaranteed here; for that, use
+native Linux Docker, or pin at the `.wslconfig`/Windows-affinity layer.
+
+For flat run-to-run numbers the biggest lever is locking the clock: cap power in
+BIOS (PL1/PL2) or disable Turbo (BIOS, or Windows power plan max state 99%), so a
+thermal/turbo spike mid-run can't skew a sample.
+
+## API
+
+```ts
+import defineBench, { type BenchConfig, type BenchContext } from '@soroush.tech/bench'
+```
+
+`defineBench(config)` validates and freezes the config. It requires a non-empty
+`name` and at least two `cases`; each case is `(ctx: BenchContext) => unknown`.

package/dist/bin.mjs

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+//#region src/docker.ts
+/** Argv for `docker build` of the sandbox image. */
+function buildImageArgs(opts) {
+	return [
+		"build",
+		"-t",
+		opts.imageTag,
+		opts.contextDir
+	];
+}
+/**
+* Argv for the pinned `docker run`. The pinning flags — single-core
+* `--cpuset-cpus`, a hard `--cpus` quota, and `--memory` with `--memory-swap`
+* set equal (swap off) — are what make timings stable run-to-run regardless of
+* host load.
+*/
+function buildRunArgs(opts) {
+	return [
+		"run",
+		"--rm",
+		"--cpuset-cpus",
+		opts.cpuset,
+		"--cpus",
+		String(opts.cpus),
+		"--memory",
+		opts.memory,
+		"--memory-swap",
+		opts.memory,
+		"--tmpfs",
+		"/bench",
+		"-v",
+		`${opts.repoDir}:/repo:ro`,
+		"-v",
+		`${opts.appDir}:/app:ro`,
+		...opts.extraMounts.flatMap((mount) => ["-v", mount]),
+		"-w",
+		"/repo",
+		opts.imageTag,
+		"tsx",
+		"/app/dist/harness.mjs",
+		`/repo/${opts.benchRelPath}`
+	];
+}
+/** Builds the image then runs the pinned benchmark; throws on a non-zero exit. */
+async function runSandbox(opts, exec) {
+	const buildCode = await exec("docker", buildImageArgs(opts));
+	if (buildCode !== 0) throw new Error(`docker build failed with exit code ${buildCode}`);
+	const runCode = await exec("docker", buildRunArgs(opts));
+	if (runCode !== 0) throw new Error(`benchmark run failed with exit code ${runCode}`);
+}
+//#endregion
+//#region src/cli.ts
+const DEFAULTS = {
+	cpus: 1,
+	cpuset: "0",
+	memory: "512m",
+	imageTag: "soroush-bench:latest"
+};
+function requireValue(value, flag) {
+	if (value === void 0) throw new Error(`bench: ${flag} requires a value`);
+	return value;
+}
+function parseCpus(value) {
+	const cpus = Number(requireValue(value, "--cpus"));
+	if (!Number.isFinite(cpus) || cpus <= 0) throw new Error("bench: --cpus must be a positive number");
+	return cpus;
+}
+/** Parses `bench <file> [--cpus N] [--cpuset C] [--memory M] [--tag T]`. */
+function parseCliArgs(argv) {
+	const opts = { ...DEFAULTS };
+	const mounts = [];
+	let benchFile;
+	for (let i = 0; i < argv.length; i += 1) {
+		const arg = argv[i];
+		switch (arg) {
+			case "--cpu":
+			case "--cpus":
+				opts.cpus = parseCpus(argv[i += 1]);
+				break;
+			case "--cpuset":
+				opts.cpuset = requireValue(argv[i += 1], "--cpuset");
+				break;
+			case "--memory":
+				opts.memory = requireValue(argv[i += 1], "--memory");
+				break;
+			case "--tag":
+				opts.imageTag = requireValue(argv[i += 1], "--tag");
+				break;
+			case "--mount":
+				mounts.push(requireValue(argv[i += 1], "--mount"));
+				break;
+			default:
+				if (arg.startsWith("--")) throw new Error(`bench: unknown option ${arg}`);
+				if (benchFile !== void 0) throw new Error("bench: only one bench file may be given");
+				benchFile = arg;
+		}
+	}
+	if (benchFile === void 0) throw new Error("bench: a bench file path is required");
+	return {
+		benchFile,
+		...opts,
+		mounts
+	};
+}
+/** Maps parsed options + host paths into the concrete sandbox run options. */
+function resolveSandboxOptions(cli, cwd, packageRoot) {
+	const rel = relative(cwd, resolve(cwd, cli.benchFile));
+	if (rel.startsWith("..") || isAbsolute(rel)) throw new Error("bench: the bench file must live inside the current directory");
+	return {
+		imageTag: cli.imageTag,
+		contextDir: packageRoot,
+		repoDir: cwd,
+		appDir: packageRoot,
+		benchRelPath: rel.split(sep).join("/"),
+		cpuset: cli.cpuset,
+		cpus: cli.cpus,
+		memory: cli.memory,
+		extraMounts: cli.mounts
+	};
+}
+/** Parses argv, resolves the run, and executes it in the pinned sandbox. */
+async function main(argv, deps) {
+	await runSandbox(resolveSandboxOptions(parseCliArgs(argv), deps.cwd, deps.packageRoot), deps.exec);
+}
+//#endregion
+//#region src/bin.ts
+/** Real process runner: inherits stdio so Docker/mitata output streams through. */
+const exec = (command, args) => new Promise((resolvePromise, reject) => {
+	const child = spawn(command, args, { stdio: "inherit" });
+	child.on("error", reject);
+	child.on("close", (code) => resolvePromise(code ?? 0));
+});
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+main(process.argv.slice(2), {
+	cwd: process.cwd(),
+	packageRoot,
+	exec
+}).catch((error) => {
+	console.error(error instanceof Error ? error.message : error);
+	process.exitCode = 1;
+});
+//#endregion
+export {};