@omni-oss/task-bench 0.0.0-beta.1

package/src/bench/index.ts

@@ -0,0 +1,323 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { cpus, tmpdir } from "node:os";
+import { join } from "node:path";
+import { performance } from "node:perf_hooks";
+import { execa } from "execa";
+import { HarnessConfigSchema, type Tool } from "../config";
+import { buildGraph, taskNames } from "../graph";
+import { getAdapter, resolveToolVersions, type ToolContext } from "../tools";
+import { computeStats, type Stats } from "./stats";
+
+export interface RunSample {
+    durationMs: number;
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+    /** Number of tasks that actually executed (0 == a full cache hit). */
+    executed: number;
+    ok: boolean;
+}
+
+export interface ScenarioResult {
+    runs: number;
+    failures: number;
+    stats: Stats;
+    /** Median number of tasks that actually executed across the runs. */
+    executedMedian: number;
+}
+
+export interface ToolResult {
+    tool: Tool;
+    task: string;
+    /** Size of the executed task graph (tasks run on a cold, uncached run). */
+    taskGraphSize: number;
+    cold: ScenarioResult;
+    warm: ScenarioResult;
+    error?: string;
+}
+
+export interface BenchmarkResult {
+    rootDir: string;
+    task: string;
+    projects: number;
+    tasksPerProject: number;
+    concurrency: number;
+    daemon: boolean;
+    /** Resolved version of each benchmarked tool (detected for omni). */
+    versions: Record<string, string | null>;
+    generatedAt: string;
+    tools: ToolResult[];
+}
+
+export type BenchEvent =
+    | { kind: "tool-start"; tool: Tool }
+    | {
+          kind: "scenario";
+          tool: Tool;
+          scenario: "cold" | "warm";
+          run: number;
+          total: number;
+          sample: RunSample;
+      }
+    | { kind: "tool-error"; tool: Tool; error: string }
+    | {
+          kind: "tool-unsuccessful";
+          tool: Tool;
+          sample: RunSample;
+      };
+
+export interface RunBenchmarkOptions {
+    tools?: Tool[] | undefined;
+    task?: string | undefined;
+    coldRuns?: number | undefined;
+    warmRuns?: number | undefined;
+    concurrency?: number | undefined;
+    /** Allow each tool's persistent daemon (default true). */
+    daemon?: boolean | undefined;
+    onEvent?: ((event: BenchEvent) => void) | undefined;
+}
+
+function median(values: number[]): number {
+    if (values.length === 0) return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0
+        ? ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2
+        : (sorted[mid] ?? 0);
+}
+
+function countLines(path: string): number {
+    try {
+        const text = readFileSync(path, "utf8");
+        if (text.length === 0) return 0;
+        // Each executed task appends exactly one newline-terminated line.
+        let count = 0;
+        for (let i = 0; i < text.length; i++) {
+            if (text[i] === "\n") count++;
+        }
+        return count;
+    } catch {
+        return 0;
+    }
+}
+
+async function timeRun(
+    file: string,
+    args: string[],
+    rootDir: string,
+    execLog: string,
+    extraEnv: Record<string, string>,
+): Promise<RunSample> {
+    // Reset the execution marker so we can count real executions for this run.
+    writeFileSync(execLog, "");
+    const start = performance.now();
+    const result = await execa(file, args, {
+        cwd: rootDir,
+        reject: false,
+        env: {
+            FORCE_COLOR: "0",
+            TURBO_TELEMETRY_DISABLED: "1",
+            DO_NOT_TRACK: "1",
+            NX_TUI: "false",
+            TASK_BENCH_EXEC_LOG: execLog,
+            ...extraEnv,
+        },
+    });
+
+    const durationMs = performance.now() - start;
+    const stdout = typeof result.stdout === "string" ? result.stdout : "";
+    const stderr = typeof result.stderr === "string" ? result.stderr : "";
+    return {
+        durationMs,
+        exitCode: result.exitCode ?? -1,
+        stdout,
+        stderr,
+        executed: countLines(execLog),
+        ok: result.exitCode === 0,
+    };
+}
+
+function scenarioFromSamples(samples: RunSample[]): ScenarioResult {
+    return {
+        runs: samples.length,
+        failures: samples.filter((s) => !s.ok).length,
+        stats: computeStats(samples.map((s) => s.durationMs)),
+        executedMedian: median(samples.map((s) => s.executed)),
+    };
+}
+
+/**
+ * Benchmark the enabled tools against an already-generated workspace.
+ *
+ * For each tool we measure two scenarios:
+ *   - cold: caches + outputs wiped before every run (discovery + full exec + cache write)
+ *   - warm: caches primed, so ideally every task is a cache hit
+ *           (isolates discovery + cache-restore overhead)
+ *
+ * A tool-agnostic execution counter (see the generated task runner) records how
+ * many tasks actually ran, so warm-run cache effectiveness is *verified* rather
+ * than assumed. Concurrency is pinned identically across all runners.
+ */
+export async function runBenchmark(
+    rootDir: string,
+    options: RunBenchmarkOptions = {},
+): Promise<BenchmarkResult> {
+    const raw = JSON.parse(
+        await readFile(join(rootDir, "bench.config.json"), "utf8"),
+    );
+    const config = HarnessConfigSchema.parse(raw);
+    const projects = buildGraph(config);
+    const tasks = taskNames(config);
+
+    const tools = options.tools ?? config.tools;
+    const task = options.task ?? tasks[tasks.length - 1] ?? "t0";
+    const coldRuns = options.coldRuns ?? 3;
+    const warmRuns = options.warmRuns ?? 5;
+    const concurrency = options.concurrency ?? Math.max(1, cpus().length);
+    const daemon = options.daemon ?? true;
+    const emit = options.onEvent ?? (() => {});
+    const execLog = join(
+        tmpdir(),
+        `task-bench-exec-${process.pid}-${Date.now()}.log`,
+    );
+
+    const ctx: ToolContext = {
+        rootDir,
+        projectDirs: projects.map((p) => p.dir),
+        concurrency,
+        daemon,
+    };
+
+    // Resolve (and re-validate) the version of each tool actually used.
+    const versionMap = await resolveToolVersions(config, rootDir, tools);
+    const versions: Record<string, string | null> = {};
+    for (const [tool, version] of versionMap) versions[tool] = version;
+
+    const results: ToolResult[] = [];
+
+    for (const tool of tools) {
+        emit({ kind: "tool-start", tool });
+        const adapter = getAdapter(tool);
+        const invocation = adapter.run(task, ctx);
+        const env = adapter.env(ctx);
+
+        try {
+            // In no-daemon mode make sure no stale daemon lingers first.
+            if (!daemon && adapter.hasDaemon) {
+                await adapter.stopDaemon(ctx);
+            }
+
+            // Cold scenario: a fresh start each run. Caches + outputs are wiped,
+            // and (in daemon mode) the daemon is torn down so cold includes its
+            // startup cost. Warm runs below deliberately keep the daemon alive.
+            const coldSamples: RunSample[] = [];
+            for (let run = 1; run <= coldRuns; run++) {
+                await adapter.clearCaches(ctx);
+                if (daemon && adapter.hasDaemon) {
+                    await adapter.stopDaemon(ctx);
+                }
+                const sample = await timeRun(
+                    invocation.file,
+                    invocation.args,
+                    rootDir,
+                    execLog,
+                    env,
+                );
+                coldSamples.push(sample);
+                emit({
+                    kind: "scenario",
+                    tool,
+                    scenario: "cold",
+                    run,
+                    total: coldRuns,
+                    sample,
+                });
+                if (sample.exitCode !== 0) {
+                    emit({
+                        kind: "tool-unsuccessful",
+                        tool,
+                        sample,
+                    });
+                }
+            }
+
+            // Warm scenario: prime once (unmeasured, also warms the daemon),
+            // then measure while caches + daemon stay hot.
+            await timeRun(
+                invocation.file,
+                invocation.args,
+                rootDir,
+                execLog,
+                env,
+            );
+            const warmSamples: RunSample[] = [];
+            for (let run = 1; run <= warmRuns; run++) {
+                const sample = await timeRun(
+                    invocation.file,
+                    invocation.args,
+                    rootDir,
+                    execLog,
+                    env,
+                );
+                warmSamples.push(sample);
+                emit({
+                    kind: "scenario",
+                    tool,
+                    scenario: "warm",
+                    run,
+                    total: warmRuns,
+                    sample,
+                });
+                if (sample.exitCode !== 0) {
+                    emit({
+                        kind: "tool-unsuccessful",
+                        tool,
+                        sample,
+                    });
+                }
+            }
+
+            const taskGraphSize = Math.max(
+                0,
+                ...coldSamples.map((s) => s.executed),
+                ...warmSamples.map((s) => s.executed),
+            );
+
+            results.push({
+                tool,
+                task,
+                taskGraphSize,
+                cold: scenarioFromSamples(coldSamples),
+                warm: scenarioFromSamples(warmSamples),
+            });
+        } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            emit({ kind: "tool-error", tool, error: message });
+            results.push({
+                tool,
+                task,
+                taskGraphSize: 0,
+                cold: scenarioFromSamples([]),
+                warm: scenarioFromSamples([]),
+                error: message,
+            });
+        } finally {
+            // Always clean up the daemon so it does not leak between tools or
+            // outlive the benchmark.
+            await adapter.stopDaemon(ctx);
+        }
+    }
+
+    return {
+        rootDir,
+        task,
+        projects: config.projects,
+        tasksPerProject: config.tasksPerProject,
+        concurrency,
+        daemon,
+        versions,
+        generatedAt: new Date().toISOString(),
+        tools: results,
+    };
+}

package/src/bench/install.ts

@@ -0,0 +1,12 @@
+import { execa } from "execa";
+
+/** Install dependencies in a generated workspace with bun. */
+export async function installWorkspace(
+    dir: string,
+    opts: { quiet?: boolean } = {},
+): Promise<void> {
+    await execa("bun", ["install"], {
+        cwd: dir,
+        stdio: opts.quiet ? "ignore" : "inherit",
+    });
+}

package/src/bench/report.ts

@@ -0,0 +1,142 @@
+import type { BenchmarkResult, ToolResult } from "./index";
+import { formatMs, type Stats } from "./stats";
+
+function pad(value: string, width: number): string {
+    return value.length >= width
+        ? value
+        : value + " ".repeat(width - value.length);
+}
+
+function statCell(stats: Stats, failures: number): string {
+    if (stats.samples.length === 0) return "—";
+    const base = `${formatMs(stats.median)} ±${formatMs(stats.stddev)}`;
+    return failures > 0 ? `${base} ⚠${failures}` : base;
+}
+
+function cacheHitPct(tool: ToolResult): number | null {
+    if (tool.taskGraphSize <= 0 || tool.warm.stats.samples.length === 0) {
+        return null;
+    }
+    const hits = tool.taskGraphSize - tool.warm.executedMedian;
+    return (hits / tool.taskGraphSize) * 100;
+}
+
+function cacheCell(tool: ToolResult): string {
+    const pct = cacheHitPct(tool);
+    if (pct === null) return "—";
+    return `${pct.toFixed(0)}%`;
+}
+
+/**
+ * Render a human-readable comparison table plus a short takeaway. The warm
+ * column is the key metric: with all tasks cached it approximates each tool's
+ * discovery + cache-restore overhead. The warm-cache-hit column verifies that
+ * assumption held (should be 100%).
+ */
+export function formatReport(result: BenchmarkResult): string {
+    const lines: string[] = [];
+    const graphSize = Math.max(0, ...result.tools.map((t) => t.taskGraphSize));
+    lines.push("");
+    lines.push(
+        `task-bench: ${result.projects} projects × ${result.tasksPerProject} tasks ` +
+            `(${graphSize} task-graph nodes), running "${result.task}" ` +
+            `at concurrency ${result.concurrency} ` +
+            `(daemons ${result.daemon ? "on" : "off"})`,
+    );
+    lines.push(formatVersions(result));
+    lines.push("");
+
+    const headers = [
+        "tool",
+        "cold (median)",
+        "warm (median)",
+        "warm cache-hit",
+        "notes",
+    ];
+    const rows = result.tools.map((t) => {
+        const notes = t.error
+            ? `error: ${(t.error.split("\n")[0] ?? "").slice(0, 40)}`
+            : "";
+        return [
+            t.tool,
+            statCell(t.cold.stats, t.cold.failures),
+            statCell(t.warm.stats, t.warm.failures),
+            cacheCell(t),
+            notes,
+        ];
+    });
+
+    const widths = headers.map((h, i) =>
+        Math.max(h.length, ...rows.map((r) => (r[i] ?? "").length)),
+    );
+
+    const renderRow = (cells: string[]) =>
+        `| ${cells.map((c, i) => pad(c, widths[i] ?? 0)).join(" | ")} |`;
+
+    lines.push(renderRow(headers));
+    lines.push(`| ${widths.map((w) => "-".repeat(w)).join(" | ")} |`);
+    for (const row of rows) lines.push(renderRow(row));
+    lines.push("");
+
+    // Warn if any warm scenario was not a full cache hit — the warm numbers
+    // are only meaningful as "overhead" when everything is cached.
+    const impure = result.tools.filter((t) => {
+        const pct = cacheHitPct(t);
+        return pct !== null && pct < 99.5;
+    });
+    for (const t of impure) {
+        lines.push(
+            `⚠ ${t.tool}: warm runs were not fully cached ` +
+                `(${cacheCell(t)} hit, ${t.warm.executedMedian}/${t.taskGraphSize} tasks re-ran) — ` +
+                `treat its warm number with caution.`,
+        );
+    }
+    if (impure.length) lines.push("");
+
+    // Fastest-warm takeaway (only among fully-cached, error-free tools).
+    const ranked = result.tools
+        .filter((t) => {
+            const pct = cacheHitPct(t);
+            return !t.error && pct !== null && pct >= 99.5;
+        })
+        .sort((a, b) => a.warm.stats.median - b.warm.stats.median);
+    const fastest = ranked[0];
+    const slowest = ranked[ranked.length - 1];
+    if (ranked.length > 1 && fastest && slowest) {
+        const factor = slowest.warm.stats.median / fastest.warm.stats.median;
+        lines.push(
+            `Warm-cache overhead: ${fastest.tool} is fastest ` +
+                `(${formatMs(fastest.warm.stats.median)}), ` +
+                `${factor.toFixed(2)}× faster than ${slowest.tool} ` +
+                `(${formatMs(slowest.warm.stats.median)}).\n`,
+        );
+    }
+
+    // Fastest-cold takeaway (discovery + full execution + cache write).
+    const coldRanked = result.tools
+        .filter((t) => !t.error && t.cold.stats.samples.length > 0)
+        .sort((a, b) => a.cold.stats.median - b.cold.stats.median);
+    const coldFastest = coldRanked[0];
+    const coldSlowest = coldRanked[coldRanked.length - 1];
+    if (coldRanked.length > 1 && coldFastest && coldSlowest) {
+        const factor =
+            coldSlowest.cold.stats.median / coldFastest.cold.stats.median;
+        lines.push(
+            `Cold-run overhead: ${coldFastest.tool} is fastest ` +
+                `(${formatMs(coldFastest.cold.stats.median)}), ` +
+                `${factor.toFixed(2)}× faster than ${coldSlowest.tool} ` +
+                `(${formatMs(coldSlowest.cold.stats.median)}).`,
+        );
+    }
+    if (ranked.length > 1 || coldRanked.length > 1) lines.push("");
+
+    return lines.join("\n");
+}
+
+/** A one-line summary of the resolved tool versions used. */
+function formatVersions(result: BenchmarkResult): string {
+    const parts = result.tools.map(
+        (t) => `\`${t.tool} ${result.versions[t.tool] ?? "?"}\``,
+    );
+    return `versions: ${parts.join(", ")}`;
+}

package/src/bench/stats.spec.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import { computeStats, formatMs } from "./stats";
+
+describe("computeStats", () => {
+    it("handles the empty case", () => {
+        const stats = computeStats([]);
+        expect(stats).toMatchObject({
+            min: 0,
+            max: 0,
+            mean: 0,
+            median: 0,
+            stddev: 0,
+        });
+    });
+
+    it("computes summary statistics", () => {
+        const stats = computeStats([10, 20, 30]);
+        expect(stats.min).toBe(10);
+        expect(stats.max).toBe(30);
+        expect(stats.mean).toBe(20);
+        expect(stats.median).toBe(20);
+        expect(stats.stddev).toBeCloseTo(Math.sqrt(200 / 3), 6);
+    });
+
+    it("averages the two middle values for an even count", () => {
+        expect(computeStats([1, 2, 3, 4]).median).toBe(2.5);
+    });
+});
+
+describe("formatMs", () => {
+    it("uses ms below a second and seconds above", () => {
+        expect(formatMs(250)).toBe("250ms");
+        expect(formatMs(1500)).toBe("1.50s");
+    });
+});

package/src/bench/stats.ts

@@ -0,0 +1,38 @@
+/** Summary statistics for a set of timing samples (all in milliseconds). */
+export interface Stats {
+    samples: number[];
+    min: number;
+    max: number;
+    mean: number;
+    median: number;
+    stddev: number;
+}
+
+export function computeStats(samples: number[]): Stats {
+    if (samples.length === 0) {
+        return { samples, min: 0, max: 0, mean: 0, median: 0, stddev: 0 };
+    }
+    const sorted = [...samples].sort((a, b) => a - b);
+    const sum = sorted.reduce((acc, n) => acc + n, 0);
+    const mean = sum / sorted.length;
+    const mid = Math.floor(sorted.length / 2);
+    const median =
+        sorted.length % 2 === 0
+            ? ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2
+            : (sorted[mid] ?? 0);
+    const variance =
+        sorted.reduce((acc, n) => acc + (n - mean) ** 2, 0) / sorted.length;
+    return {
+        samples,
+        min: sorted[0] ?? 0,
+        max: sorted[sorted.length - 1] ?? 0,
+        mean,
+        median,
+        stddev: Math.sqrt(variance),
+    };
+}
+
+export function formatMs(ms: number): string {
+    if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
+    return `${ms.toFixed(0)}ms`;
+}