@omni-oss/task-bench 0.0.0-beta.1

package/src/generate/index.ts

@@ -0,0 +1,215 @@
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { execa } from "execa";
+import {
+    type HarnessConfig,
+    type HarnessConfigInput,
+    resolveConfig,
+} from "../config";
+import { buildGraph, type ProjectNode, taskNames } from "../graph";
+import { getAdapters, resolveToolVersions, type ToolAdapter } from "../tools";
+import { sourceFile, taskRunner } from "./templates";
+
+export interface GenerateResult {
+    rootDir: string;
+    config: HarnessConfig;
+    projects: ProjectNode[];
+    /** Every file written, workspace-relative. */
+    files: string[];
+}
+
+async function writeText(
+    rootDir: string,
+    relPath: string,
+    contents: string,
+    written: string[],
+): Promise<void> {
+    const abs = join(rootDir, relPath);
+    await mkdir(dirname(abs), { recursive: true });
+    await writeFile(abs, contents);
+    written.push(relPath);
+}
+
+function rootPackageJson(
+    config: HarnessConfig,
+    adapters: ToolAdapter[],
+): string {
+    // Each tool contributes its own npm dependencies (decoupled).
+    const devDependencies: Record<string, string> = {};
+    for (const adapter of adapters) {
+        Object.assign(devDependencies, adapter.devDependencies(config));
+    }
+
+    return `${JSON.stringify(
+        {
+            name: "task-bench-harness",
+            private: true,
+            packageManager: `bun@${config.versions.bun}`,
+            workspaces: ["packages/*"],
+            ...(Object.keys(devDependencies).length ? { devDependencies } : {}),
+        },
+        null,
+        2,
+    )}\n`;
+}
+
+function projectPackageJson(
+    config: HarnessConfig,
+    project: ProjectNode,
+    projects: ProjectNode[],
+): string {
+    const scripts: Record<string, string> = {};
+    for (const task of taskNames(config)) {
+        scripts[task] = `node ./task.mjs ${task}`;
+    }
+    const dependencies: Record<string, string> = {};
+    for (const depIndex of project.dependencies) {
+        const dep = projects[depIndex];
+        if (dep) dependencies[dep.name] = "workspace:*";
+    }
+    return `${JSON.stringify(
+        {
+            name: project.name,
+            version: "0.0.0",
+            private: true,
+            scripts,
+            ...(Object.keys(dependencies).length ? { dependencies } : {}),
+        },
+        null,
+        2,
+    )}\n`;
+}
+
+function gitignore(): string {
+    return [
+        "node_modules",
+        "dist",
+        ".turbo",
+        ".nx",
+        ".omni",
+        ".moon/cache",
+        "*.log",
+        "",
+    ].join("\n");
+}
+
+function readme(config: HarnessConfig): string {
+    return [
+        "# task-bench harness",
+        "",
+        "Auto-generated benchmark workspace. Do not edit by hand; regenerate via",
+        "`@omni-oss/task-bench` instead.",
+        "",
+        `- projects: **${config.projects}**`,
+        `- tasks per project: **${config.tasksPerProject}**`,
+        `- dependency strategy: **${config.dependency.strategy}**`,
+        `- tools: **${config.tools.join(", ")}**`,
+        "",
+        "The exact configuration is captured in `bench.config.json`.",
+        "",
+    ].join("\n");
+}
+
+/**
+ * moon only enables its cache when the workspace is a git repository, so
+ * initialize + commit one. This is harmless (and realistic) for the other
+ * runners. Failures (e.g. git missing) are ignored.
+ */
+async function initGitRepo(rootDir: string): Promise<void> {
+    const opts = { cwd: rootDir, reject: false, stdio: "ignore" as const };
+    await execa("git", ["init", "-q"], opts);
+    await execa("git", ["add", "-A"], opts);
+    await execa(
+        "git",
+        [
+            "-c",
+            "user.email=bench@task-bench.local",
+            "-c",
+            "user.name=task-bench",
+            "commit",
+            "-qm",
+            "generated benchmark workspace",
+        ],
+        opts,
+    );
+}
+
+/**
+ * Generate a complete benchmark workspace at `rootDir`. Existing contents at
+ * `rootDir` are removed first so the workspace is reproducible.
+ *
+ * The generator only produces the neutral, tool-agnostic workspace (projects,
+ * sources, task runners). Each enabled tool then writes its own configuration
+ * via its adapter, keeping the tools fully decoupled.
+ */
+export async function generateWorkspace(
+    rootDir: string,
+    input?: HarnessConfigInput,
+): Promise<GenerateResult> {
+    const config = resolveConfig(input);
+    const projects = buildGraph(config);
+    const written: string[] = [];
+    const adapters = getAdapters(config.tools);
+
+    await rm(rootDir, { recursive: true, force: true });
+    await mkdir(rootDir, { recursive: true });
+
+    // Validate every enabled tool's version up-front (fails fast).
+    const versions = await resolveToolVersions(config, rootDir);
+
+    // Neutral, tool-agnostic workspace.
+    await writeText(
+        rootDir,
+        "package.json",
+        rootPackageJson(config, adapters),
+        written,
+    );
+    await writeText(rootDir, ".gitignore", gitignore(), written);
+    await writeText(rootDir, "README.md", readme(config), written);
+    await writeText(
+        rootDir,
+        "bench.config.json",
+        `${JSON.stringify(config, null, 2)}\n`,
+        written,
+    );
+
+    for (const project of projects) {
+        const dir = project.dir;
+        await writeText(
+            rootDir,
+            `${dir}/package.json`,
+            projectPackageJson(config, project, projects),
+            written,
+        );
+        await writeText(
+            rootDir,
+            `${dir}/src/index.js`,
+            sourceFile(project),
+            written,
+        );
+        await writeText(
+            rootDir,
+            `${dir}/task.mjs`,
+            taskRunner(config, project),
+            written,
+        );
+    }
+
+    // Let each tool derive and write its own configuration.
+    const write = (relPath: string, contents: string) =>
+        writeText(rootDir, relPath, contents, written);
+    for (const adapter of adapters) {
+        await adapter.setup({
+            rootDir,
+            config,
+            projects,
+            version: versions.get(adapter.tool) ?? null,
+            write,
+        });
+    }
+
+    // moon requires a git repo to enable its cache.
+    await initGitRepo(rootDir);
+
+    return { rootDir, config, projects, files: written };
+}

package/src/generate/templates.ts

@@ -0,0 +1,87 @@
+import type { HarnessConfig } from "../config";
+import type { ProjectNode } from "../graph";
+
+/**
+ * The minimal source file each project ships. It exists so that every runner
+ * has a real input file to hash for cache keys.
+ */
+export function sourceFile(project: ProjectNode): string {
+    return [
+        `// Auto-generated source for ${project.name}.`,
+        `// Edit the harness config, not this file.`,
+        `export const id = ${JSON.stringify(project.name)};`,
+        `export const index = ${project.index};`,
+        `export const answer = 42;`,
+        "",
+    ].join("\n");
+}
+
+/**
+ * The task runner every project uses. It is intentionally tiny but does real
+ * work that exercises each runner's caching + log-capture machinery:
+ *   - reads its own source (a cache input),
+ *   - performs a deterministic CPU loop,
+ *   - prints a configurable number of log lines to stdout,
+ *   - writes deterministic output file(s) into dist/ (cache outputs).
+ *
+ * Determinism matters: identical inputs => identical outputs => cache hits on
+ * warm runs, which is exactly what we want to measure.
+ */
+export function taskRunner(
+    config: HarnessConfig,
+    project: ProjectNode,
+): string {
+    const { logLines, workIterations, outputFiles } = config.task;
+    return `#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const PROJECT = ${JSON.stringify(project.name)};
+const LOG_LINES = ${logLines};
+const WORK_ITERATIONS = ${workIterations};
+const OUTPUT_FILES = ${outputFiles};
+
+const task = process.argv[2] ?? "task";
+
+// Ground-truth execution marker: this line is only ever reached when the task
+// actually runs. Cache hits skip the process entirely, so counting the lines
+// in this out-of-tree log yields a tool-agnostic cache-hit rate. The log path
+// lives outside the workspace so no runner hashes it as an input.
+const execLog = process.env.TASK_BENCH_EXEC_LOG;
+if (execLog) {
+    appendFileSync(execLog, \`\${PROJECT}\\t\${task}\\n\`);
+}
+
+const source = readFileSync(join(HERE, "src", "index.js"), "utf8");
+
+// Cheap, deterministic CPU work.
+let acc = 0;
+for (let i = 0; i < WORK_ITERATIONS; i++) {
+    acc = (acc + Math.imul(i ^ acc, 2654435761)) >>> 0;
+}
+
+const digest = createHash("sha256")
+    .update(source)
+    .update(task)
+    .update(String(acc))
+    .digest("hex");
+
+for (let i = 1; i <= LOG_LINES; i++) {
+    console.log(\`[\${PROJECT}] \${task}: step \${i}/\${LOG_LINES} digest=\${digest.slice(0, 12)}\`);
+}
+
+const outDir = join(HERE, "dist");
+mkdirSync(outDir, { recursive: true });
+for (let f = 0; f < OUTPUT_FILES; f++) {
+    writeFileSync(
+        join(outDir, \`\${task}.\${f}.txt\`),
+        \`\${PROJECT}\\t\${task}\\t\${digest}\\n\`,
+    );
+}
+
+console.log(\`[\${PROJECT}] \${task}: complete (\${OUTPUT_FILES} output file(s))\`);
+`;
+}

package/src/graph.spec.ts

@@ -0,0 +1,119 @@
+import { describe, expect, it } from "vitest";
+import { resolveConfig } from "./config";
+import { buildGraph, projectName, taskNames } from "./graph";
+
+describe("resolveConfig", () => {
+    it("applies defaults", () => {
+        const config = resolveConfig();
+        expect(config.projects).toBe(50);
+        expect(config.tasksPerProject).toBe(3);
+        expect(config.dependency.strategy).toBe("layered");
+        expect(config.tools).toEqual(["omni", "turbo", "nx", "moon"]);
+    });
+
+    it("merges partial overrides", () => {
+        const config = resolveConfig({
+            projects: 8,
+            dependency: { strategy: "chain" },
+        });
+        expect(config.projects).toBe(8);
+        expect(config.dependency.strategy).toBe("chain");
+        // Untouched nested defaults remain.
+        expect(config.dependency.fanout).toBe(3);
+    });
+});
+
+describe("projectName", () => {
+    it("zero-pads to a stable width", () => {
+        const config = resolveConfig({
+            projects: 12,
+            projectPrefix: "bench-p",
+        });
+        expect(projectName(config, 0)).toBe("bench-p0000");
+        expect(projectName(config, 11)).toBe("bench-p0011");
+    });
+});
+
+describe("taskNames", () => {
+    it("generates t0..tN-1", () => {
+        expect(taskNames(resolveConfig({ tasksPerProject: 3 }))).toEqual([
+            "t0",
+            "t1",
+            "t2",
+        ]);
+    });
+});
+
+describe("buildGraph strategies", () => {
+    const base = { projects: 10 } as const;
+
+    it("isolated has no edges", () => {
+        const nodes = buildGraph(
+            resolveConfig({ ...base, dependency: { strategy: "isolated" } }),
+        );
+        expect(nodes.every((n) => n.dependencies.length === 0)).toBe(true);
+    });
+
+    it("chain links each project to its predecessor", () => {
+        const nodes = buildGraph(
+            resolveConfig({ ...base, dependency: { strategy: "chain" } }),
+        );
+        expect(nodes[0]?.dependencies).toEqual([]);
+        expect(nodes[5]?.dependencies).toEqual([4]);
+    });
+
+    it("fan-out points every project at the root", () => {
+        const nodes = buildGraph(
+            resolveConfig({ ...base, dependency: { strategy: "fan-out" } }),
+        );
+        expect(nodes[0]?.dependencies).toEqual([]);
+        for (let i = 1; i < nodes.length; i++) {
+            expect(nodes[i]?.dependencies).toEqual([0]);
+        }
+    });
+
+    it("layered only depends on the previous layer", () => {
+        const config = resolveConfig({
+            projects: 10,
+            dependency: { strategy: "layered", layers: 5, fanout: 2 },
+        });
+        const nodes = buildGraph(config);
+        const perLayer = Math.ceil(10 / 5); // 2
+        for (const node of nodes) {
+            const layer = Math.floor(node.index / perLayer);
+            for (const dep of node.dependencies) {
+                expect(Math.floor(dep / perLayer)).toBe(layer - 1);
+            }
+        }
+    });
+
+    it("is always acyclic (deps point to lower indices)", () => {
+        for (const strategy of [
+            "chain",
+            "fan-out",
+            "layered",
+            "random",
+        ] as const) {
+            const nodes = buildGraph(
+                resolveConfig({ projects: 25, dependency: { strategy } }),
+            );
+            for (const node of nodes) {
+                for (const dep of node.dependencies) {
+                    expect(dep).toBeLessThan(node.index);
+                }
+            }
+        }
+    });
+
+    it("random is deterministic for a fixed seed", () => {
+        const mk = () =>
+            buildGraph(
+                resolveConfig({
+                    projects: 30,
+                    seed: 7,
+                    dependency: { strategy: "random", edgeProbability: 0.4 },
+                }),
+            );
+        expect(mk()).toEqual(mk());
+    });
+});

package/src/graph.ts

@@ -0,0 +1,120 @@
+import type { HarnessConfig } from "./config";
+
+/** A single generated project and the upstream projects it depends on. */
+export interface ProjectNode {
+    /** Zero-based index in the generated set. */
+    index: number;
+    /** Package name, e.g. `bench-p0007`. */
+    name: string;
+    /** Workspace-relative POSIX directory, e.g. `packages/bench-p0007`. */
+    dir: string;
+    /** Indices of upstream projects this project depends on. */
+    dependencies: number[];
+}
+
+/**
+ * Deterministic PRNG (mulberry32) so a given seed always yields the same graph.
+ */
+function makeRng(seed: number): () => number {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+
+function padWidth(count: number): number {
+    return Math.max(4, String(count - 1).length);
+}
+
+export function projectName(config: HarnessConfig, index: number): string {
+    const width = padWidth(config.projects);
+    return `${config.projectPrefix}${String(index).padStart(width, "0")}`;
+}
+
+/** Evenly sample up to `count` indices from the inclusive range [start, end]. */
+function evenlySample(start: number, end: number, count: number): number[] {
+    const size = end - start + 1;
+    if (size <= 0 || count <= 0) return [];
+    if (count >= size) {
+        return Array.from({ length: size }, (_, i) => start + i);
+    }
+    const picked: number[] = [];
+    for (let i = 0; i < count; i++) {
+        // Spread picks across the range deterministically.
+        const offset = Math.round((i * (size - 1)) / (count - 1 || 1));
+        picked.push(start + offset);
+    }
+    return [...new Set(picked)];
+}
+
+function computeDependencies(
+    config: HarnessConfig,
+    index: number,
+    rng: () => number,
+): number[] {
+    const { strategy, layers, fanout, edgeProbability } = config.dependency;
+
+    switch (strategy) {
+        case "isolated":
+            return [];
+
+        case "chain":
+            return index > 0 ? [index - 1] : [];
+
+        case "fan-out":
+            // Every project (except the root) depends on the single root project.
+            return index > 0 ? [0] : [];
+
+        case "layered": {
+            const perLayer = Math.ceil(config.projects / layers);
+            const layer = Math.floor(index / perLayer);
+            if (layer === 0) return [];
+            const prevStart = (layer - 1) * perLayer;
+            const prevEnd = Math.min(layer * perLayer, config.projects) - 1;
+            return evenlySample(prevStart, prevEnd, fanout);
+        }
+
+        case "random": {
+            if (index === 0) return [];
+            const deps: number[] = [];
+            for (let j = 0; j < index; j++) {
+                if (rng() < edgeProbability) deps.push(j);
+            }
+            if (fanout > 0 && deps.length > fanout) {
+                // Keep the `fanout` closest ancestors for a shallower graph.
+                return deps.slice(deps.length - fanout);
+            }
+            return deps;
+        }
+
+        default:
+            return [];
+    }
+}
+
+/**
+ * Build the full project graph for a config. Dependencies always point to
+ * lower indices, guaranteeing an acyclic graph.
+ */
+export function buildGraph(config: HarnessConfig): ProjectNode[] {
+    const rng = makeRng(config.seed);
+    const nodes: ProjectNode[] = [];
+    for (let index = 0; index < config.projects; index++) {
+        const name = projectName(config, index);
+        nodes.push({
+            index,
+            name,
+            dir: `packages/${name}`,
+            dependencies: computeDependencies(config, index, rng),
+        });
+    }
+    return nodes;
+}
+
+/** The task names generated for every project: `t0`, `t1`, ... */
+export function taskNames(config: HarnessConfig): string[] {
+    return Array.from({ length: config.tasksPerProject }, (_, i) => `t${i}`);
+}

package/src/index.ts

@@ -0,0 +1,31 @@
+export * from "./bench";
+export { installWorkspace } from "./bench/install";
+export { formatReport } from "./bench/report";
+export type { Stats } from "./bench/stats";
+export { computeStats, formatMs } from "./bench/stats";
+export * from "./config";
+export type { GenerateResult } from "./generate";
+export { generateWorkspace } from "./generate";
+export * from "./graph";
+export type {
+    RunSuiteOptions,
+    SuiteEvent,
+    SuiteResult,
+    SuiteScenarioResult,
+} from "./suite";
+export { runSuite } from "./suite";
+export * from "./suite/preset";
+export { formatSuiteMarkdown } from "./suite/report";
+export type {
+    GenerationContext,
+    RunInvocation,
+    ToolAdapter,
+    ToolContext,
+    WorkspaceWriter,
+} from "./tools";
+export {
+    assertSupportedVersion,
+    getAdapter,
+    getAdapters,
+    resolveToolVersions,
+} from "./tools";

package/src/suite/index.ts

@@ -0,0 +1,113 @@
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import { version } from "../../package.json";
+import { type BenchEvent, type BenchmarkResult, runBenchmark } from "../bench";
+import { installWorkspace } from "../bench/install";
+import type { HarnessConfig } from "../config";
+import { generateWorkspace } from "../generate";
+import { type RunOptions, resolveScenario, type SuiteConfig } from "./preset";
+
+export interface SuiteScenarioResult {
+    name: string;
+    description?: string | undefined;
+    config: HarnessConfig;
+    run: RunOptions;
+    result: BenchmarkResult;
+}
+
+export interface SuiteResult {
+    name: string;
+    description?: string | undefined;
+    generatedAt: string;
+    scenarios: SuiteScenarioResult[];
+    taskBenchVersion: string;
+}
+
+export type SuiteEvent =
+    | { kind: "scenario-start"; name: string; index: number; total: number }
+    | { kind: "scenario-done"; name: string; index: number; total: number }
+    | { kind: "bench"; name: string; event: BenchEvent };
+
+export interface RunSuiteOptions {
+    /** Base directory under which each scenario workspace is generated. */
+    workdir: string;
+    /** Run `bun install` in each generated workspace (default true). */
+    install?: boolean | undefined;
+    /** Keep generated workspaces on disk instead of removing them (default false). */
+    keep?: boolean | undefined;
+    /** Global run-option overrides applied to every scenario. */
+    overrides?: Partial<RunOptions> | undefined;
+    onEvent?: ((event: SuiteEvent) => void) | undefined;
+}
+
+/**
+ * Run every scenario in a suite: generate a workspace, install, benchmark, and
+ * collect the results. Workspaces are removed afterwards unless `keep` is set.
+ */
+export async function runSuite(
+    suite: SuiteConfig,
+    options: RunSuiteOptions,
+): Promise<SuiteResult> {
+    const emit = options.onEvent ?? (() => {});
+    const scenarios: SuiteScenarioResult[] = [];
+    const total = suite.scenarios.length;
+
+    for (let index = 0; index < total; index++) {
+        const scenario = suite.scenarios[index];
+        if (!scenario) continue;
+        const resolved = resolveScenario(suite, scenario);
+        const { config, run } = resolved;
+
+        // Global overrides win over per-scenario/defaults.
+        const tools = options.overrides?.tools ?? run.tools ?? config.tools;
+        // Keep generation and execution tool sets consistent.
+        config.tools = tools;
+
+        const dir = join(options.workdir, resolved.name);
+
+        emit({
+            kind: "scenario-start",
+            name: resolved.name,
+            index,
+            total,
+        });
+
+        await generateWorkspace(dir, config);
+        if (options.install !== false) {
+            await installWorkspace(dir, { quiet: true });
+        }
+
+        const result = await runBenchmark(dir, {
+            tools,
+            task: options.overrides?.task ?? run.task,
+            coldRuns: options.overrides?.coldRuns ?? run.coldRuns,
+            warmRuns: options.overrides?.warmRuns ?? run.warmRuns,
+            concurrency: options.overrides?.concurrency ?? run.concurrency,
+            daemon: options.overrides?.daemon ?? run.daemon,
+            onEvent: (event) =>
+                emit({ kind: "bench", name: resolved.name, event }),
+        });
+
+        scenarios.push({
+            name: resolved.name,
+            description: resolved.description,
+            config,
+            run,
+            result,
+        });
+
+        if (!options.keep) {
+            await rm(dir, { recursive: true, force: true });
+        }
+
+        emit({ kind: "scenario-done", name: resolved.name, index, total });
+    }
+
+    return {
+        name: suite.name,
+        description: suite.description,
+        generatedAt: new Date().toISOString(),
+        scenarios,
+        taskBenchVersion: version,
+    };
+}