@omni-oss/task-bench 0.0.0-beta.1

package/src/suite/preset.spec.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it } from "vitest";
+import {
+    deepMerge,
+    getPreset,
+    listPresets,
+    parseSuite,
+    resolveScenario,
+} from "./preset";
+
+describe("deepMerge", () => {
+    it("recursively merges plain objects and replaces scalars/arrays", () => {
+        expect(
+            deepMerge(
+                { a: 1, nested: { x: 1, y: 2 }, list: [1, 2] },
+                { a: 2, nested: { y: 3, z: 4 }, list: [9] },
+            ),
+        ).toEqual({ a: 2, nested: { x: 1, y: 3, z: 4 }, list: [9] });
+    });
+});
+
+describe("built-in presets", () => {
+    it("exposes the expected presets", () => {
+        expect(listPresets()).toEqual(
+            expect.arrayContaining([
+                "quick",
+                "shapes",
+                "scale",
+                "density",
+                "daemon",
+                "full",
+            ]),
+        );
+    });
+
+    it("throws a helpful error for an unknown preset", () => {
+        expect(() => getPreset("nope")).toThrow(/unknown preset/);
+    });
+
+    it("every built-in preset parses and resolves", () => {
+        for (const name of listPresets()) {
+            const suite = getPreset(name);
+            for (const scenario of suite.scenarios) {
+                const resolved = resolveScenario(suite, scenario);
+                expect(resolved.config.projects).toBeGreaterThan(0);
+            }
+        }
+    });
+});
+
+describe("resolveScenario", () => {
+    it("merges suite defaults under scenario overrides", () => {
+        const suite = getPreset("shapes");
+        const chain = suite.scenarios.find((s) => s.name === "shape-chain");
+        expect(chain).toBeDefined();
+        // biome-ignore lint/style/noNonNullAssertion: for testing
+        const resolved = resolveScenario(suite, chain!);
+        // From defaults:
+        expect(resolved.config.projects).toBe(120);
+        expect(resolved.run.concurrency).toBe(8);
+        // From the scenario override:
+        expect(resolved.config.dependency.strategy).toBe("chain");
+    });
+
+    it("carries per-scenario run overrides (daemon on/off)", () => {
+        const suite = getPreset("daemon");
+        const off = suite.scenarios.find((s) => s.name === "daemon-off");
+        // biome-ignore lint/style/noNonNullAssertion: for testing
+        expect(resolveScenario(suite, off!).run.daemon).toBe(false);
+    });
+});
+
+describe("parseSuite", () => {
+    it("validates a custom suite object", () => {
+        const suite = parseSuite({
+            name: "custom",
+            scenarios: [
+                { name: "a", config: { projects: 5 } },
+                { name: "b", config: { projects: 6, tasksPerProject: 2 } },
+            ],
+        });
+        expect(suite.scenarios).toHaveLength(2);
+        expect(
+            // biome-ignore lint/style/noNonNullAssertion: for testing
+            resolveScenario(suite, suite.scenarios[0]!).config.projects,
+        ).toBe(5);
+    });
+
+    it("rejects filesystem-unsafe scenario names", () => {
+        expect(() =>
+            parseSuite({
+                scenarios: [{ name: "bad/name", config: {} }],
+            }),
+        ).toThrow();
+    });
+});

package/src/suite/preset.ts

@@ -0,0 +1,253 @@
+import { z } from "zod";
+import {
+    type HarnessConfig,
+    type HarnessConfigInput,
+    resolveConfig,
+    ToolSchema,
+} from "../config";
+
+/** Per-scenario benchmark run options (serializable subset of RunBenchmarkOptions). */
+export const RunOptionsSchema = z
+    .object({
+        tools: z.array(ToolSchema).optional(),
+        task: z.string().optional(),
+        coldRuns: z.number().int().nonnegative().optional(),
+        warmRuns: z.number().int().nonnegative().optional(),
+        concurrency: z.number().int().positive().optional(),
+        daemon: z.boolean().optional(),
+    })
+    .default({});
+
+export type RunOptions = z.infer<typeof RunOptionsSchema>;
+
+/**
+ * A harness config fragment. Kept loose here (merged then validated by
+ * `resolveConfig`) so scenarios can inherit from suite-level defaults.
+ */
+const ConfigFragmentSchema = z.record(z.string(), z.unknown()).default({});
+
+export const ScenarioSchema = z.object({
+    name: z
+        .string()
+        .regex(
+            /^[a-zA-Z0-9._-]+$/,
+            "scenario name must be filesystem-safe (letters, digits, . _ -)",
+        ),
+    description: z.string().optional(),
+    config: ConfigFragmentSchema,
+    run: RunOptionsSchema,
+});
+
+export const SuiteSchema = z.object({
+    name: z.string().default("task-bench suite"),
+    description: z.string().optional(),
+    defaults: z
+        .object({
+            config: ConfigFragmentSchema,
+            run: RunOptionsSchema,
+        })
+        .prefault({}),
+    scenarios: z.array(ScenarioSchema).min(1),
+});
+
+export type SuiteConfig = z.infer<typeof SuiteSchema>;
+export type SuiteConfigInput = z.input<typeof SuiteSchema>;
+export type Scenario = z.infer<typeof ScenarioSchema>;
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/** Deep-merge plain objects; arrays and scalars from `override` replace `base`. */
+export function deepMerge(
+    base: Record<string, unknown>,
+    override: Record<string, unknown>,
+): Record<string, unknown> {
+    const out: Record<string, unknown> = { ...base };
+    for (const [key, value] of Object.entries(override)) {
+        const current = out[key];
+        out[key] =
+            isPlainObject(current) && isPlainObject(value)
+                ? deepMerge(current, value)
+                : value;
+    }
+    return out;
+}
+
+export interface ResolvedScenario {
+    name: string;
+    description?: string | undefined;
+    config: HarnessConfig;
+    run: RunOptions;
+}
+
+/** Merge a scenario over suite defaults and resolve/validate its config. */
+export function resolveScenario(
+    suite: SuiteConfig,
+    scenario: Scenario,
+): ResolvedScenario {
+    const mergedConfig = deepMerge(
+        suite.defaults.config,
+        scenario.config,
+    ) as HarnessConfigInput;
+    const run: RunOptions = { ...suite.defaults.run, ...scenario.run };
+    return {
+        name: scenario.name,
+        description: scenario.description,
+        config: resolveConfig(mergedConfig),
+        run,
+    };
+}
+
+// ---------------------------------------------------------------------------
+// Built-in presets
+// ---------------------------------------------------------------------------
+
+const DEPENDENCY_STRATEGIES_FOR_SWEEP = [
+    "isolated",
+    "chain",
+    "fan-out",
+    "layered",
+    "random",
+] as const;
+
+const shapes: SuiteConfigInput = {
+    name: "dependency-shape sweep",
+    description:
+        "How the dependency-graph shape affects discovery/scheduling overhead at a fixed scale.",
+    defaults: {
+        config: { projects: 120, tasksPerProject: 3 },
+        run: { concurrency: 8, coldRuns: 2, warmRuns: 3 },
+    },
+    scenarios: DEPENDENCY_STRATEGIES_FOR_SWEEP.map((strategy) => ({
+        name: `shape-${strategy}`,
+        config: { dependency: { strategy } },
+    })),
+};
+
+const scale: SuiteConfigInput = {
+    name: "scale sweep",
+    description: "How overhead grows with workspace size (layered graph).",
+    defaults: {
+        config: {
+            tasksPerProject: 3,
+            dependency: { strategy: "layered", layers: 8 },
+        },
+        run: { concurrency: 8, coldRuns: 2, warmRuns: 3 },
+    },
+    scenarios: [50, 150, 300, 600].map((projects) => ({
+        name: `scale-${projects}`,
+        config: { projects },
+    })),
+};
+
+const density: SuiteConfigInput = {
+    name: "task-density sweep",
+    description: "How the number of tasks per project affects overhead.",
+    defaults: {
+        config: { projects: 120, dependency: { strategy: "layered" } },
+        run: { concurrency: 8, coldRuns: 2, warmRuns: 3 },
+    },
+    scenarios: [2, 5, 10].map((tasksPerProject) => ({
+        name: `density-${tasksPerProject}`,
+        config: { tasksPerProject },
+    })),
+};
+
+const daemon: SuiteConfigInput = {
+    name: "daemon on vs off",
+    description:
+        "Whether each tool's persistent daemon changes warm/cold overhead.",
+    defaults: {
+        config: {
+            projects: 200,
+            tasksPerProject: 3,
+            dependency: { strategy: "layered" },
+        },
+        run: { concurrency: 8, coldRuns: 2, warmRuns: 3 },
+    },
+    scenarios: [
+        { name: "daemon-on", run: { daemon: true } },
+        { name: "daemon-off", run: { daemon: false } },
+    ],
+};
+
+const quick: SuiteConfigInput = {
+    name: "quick smoke suite",
+    description: "Tiny, fast sanity sweep across two shapes.",
+    defaults: {
+        config: { projects: 30, tasksPerProject: 2 },
+        run: { concurrency: 8, coldRuns: 1, warmRuns: 2 },
+    },
+    scenarios: [
+        {
+            name: "quick-isolated",
+            config: { dependency: { strategy: "isolated" } },
+        },
+        {
+            name: "quick-layered",
+            config: { dependency: { strategy: "layered" } },
+        },
+    ],
+};
+
+const full: SuiteConfigInput = {
+    name: "full suite",
+    description: "Shapes + scale + density in one run.",
+    defaults: { run: { concurrency: 8, coldRuns: 2, warmRuns: 3 } },
+    scenarios: [
+        ...DEPENDENCY_STRATEGIES_FOR_SWEEP.map((strategy) => ({
+            name: `shape-${strategy}`,
+            config: {
+                projects: 120,
+                tasksPerProject: 3,
+                dependency: { strategy },
+            },
+        })),
+        ...[50, 150, 300].map((projects) => ({
+            name: `scale-${projects}`,
+            config: {
+                projects,
+                tasksPerProject: 3,
+                dependency: { strategy: "layered" as const, layers: 8 },
+            },
+        })),
+        ...[2, 5, 10].map((tasksPerProject) => ({
+            name: `density-${tasksPerProject}`,
+            config: {
+                projects: 120,
+                tasksPerProject,
+                dependency: { strategy: "layered" as const },
+            },
+        })),
+    ],
+};
+
+export const BUILTIN_PRESETS: Record<string, SuiteConfigInput> = {
+    quick,
+    shapes,
+    scale,
+    density,
+    daemon,
+    full,
+};
+
+export function listPresets(): string[] {
+    return Object.keys(BUILTIN_PRESETS);
+}
+
+/** Resolve a built-in preset by name, or throw with the list of valid names. */
+export function getPreset(name: string): SuiteConfig {
+    const preset = BUILTIN_PRESETS[name];
+    if (!preset) {
+        throw new Error(
+            `unknown preset "${name}" (available: ${listPresets().join(", ")})`,
+        );
+    }
+    return SuiteSchema.parse(preset);
+}
+
+/** Parse and validate a suite preset object (e.g. loaded from a JSON file). */
+export function parseSuite(input: unknown): SuiteConfig {
+    return SuiteSchema.parse(input);
+}

package/src/suite/report.ts

@@ -0,0 +1,135 @@
+import { formatReport } from "../bench/report";
+import { formatMs } from "../bench/stats";
+import { TOOLS, type Tool } from "../config";
+import type { SuiteResult, SuiteScenarioResult } from "./index";
+
+/** Tools that appear in at least one scenario, in canonical order. */
+function toolsInSuite(suite: SuiteResult): Tool[] {
+    const present = new Set<Tool>();
+    for (const s of suite.scenarios) {
+        for (const t of s.result.tools) present.add(t.tool);
+    }
+    return TOOLS.filter((t) => present.has(t));
+}
+
+/** One-line summary of the resolved tool versions used across the suite. */
+function formatSuiteVersions(suite: SuiteResult, tools: Tool[]): string {
+    const parts = tools.map((tool) => {
+        const version = suite.scenarios
+            .map((s) => s.result.versions[tool])
+            .find((v) => v != null);
+        return `\`${tool} ${version ?? "?"}\``;
+    });
+    return `Tool versions:\n\n${parts.join("\n\n")}`;
+}
+
+function warmCell(scenario: SuiteScenarioResult, tool: Tool): string {
+    const t = scenario.result.tools.find((r) => r.tool === tool);
+    if (!t || t.error) return t?.error ? "err" : "—";
+    const hits =
+        t.taskGraphSize > 0
+            ? (t.taskGraphSize - t.warm.executedMedian) / t.taskGraphSize
+            : 1;
+    const value = formatMs(t.warm.stats.median);
+    return hits < 0.995 ? `${value}⚠` : value;
+}
+
+function coldCell(scenario: SuiteScenarioResult, tool: Tool): string {
+    const t = scenario.result.tools.find((r) => r.tool === tool);
+    if (!t || t.error) return t?.error ? "err" : "—";
+    return formatMs(t.cold.stats.median);
+}
+
+function table(headers: string[], rows: string[][]): string[] {
+    const widths = headers.map((h, i) =>
+        Math.max(h.length, ...rows.map((r) => (r[i] ?? "").length)),
+    );
+    const pad = (v: string, w: number) => v.padEnd(w);
+    const line = (cells: string[]) =>
+        `| ${cells.map((c, i) => pad(c, widths[i] ?? 0)).join(" | ")} |`;
+    return [
+        line(headers),
+        `| ${widths.map((w) => "-".repeat(w)).join(" | ")} |`,
+        ...rows.map(line),
+    ];
+}
+
+/**
+ * Render a full suite as Markdown: two summary matrices (warm + cold median per
+ * tool per scenario) followed by the detailed per-scenario report tables.
+ */
+export function formatSuiteMarkdown(suite: SuiteResult): string {
+    const tools = toolsInSuite(suite);
+    const lines: string[] = [];
+
+    lines.push(`# ${suite.name}`);
+    lines.push("");
+    lines.push(`TaskBench v${suite.taskBenchVersion}`);
+    lines.push("");
+    if (suite.description) {
+        lines.push(suite.description);
+        lines.push("");
+    }
+    lines.push(
+        `Generated ${suite.generatedAt} · ${suite.scenarios.length} scenario(s).`,
+    );
+    lines.push("");
+    lines.push(formatSuiteVersions(suite, tools));
+    lines.push("");
+    lines.push(
+        "`warm` = median wall time with a verified 100% cache hit " +
+            "(discovery + cache-restore overhead). `⚠` marks a scenario whose " +
+            "warm runs were not fully cached. Absolute times are " +
+            "hardware-dependent — read the ratios.",
+    );
+    lines.push("");
+
+    // Warm summary.
+    lines.push("## Summary — warm median");
+    lines.push("");
+    lines.push(
+        ...table(
+            ["scenario", "nodes", "conc", "daemon", ...tools],
+            suite.scenarios.map((s) => [
+                s.name,
+                String(s.result.tools[0]?.taskGraphSize ?? 0),
+                String(s.result.concurrency),
+                s.result.daemon ? "on" : "off",
+                ...tools.map((t) => warmCell(s, t)),
+            ]),
+        ),
+    );
+    lines.push("");
+
+    // Cold summary.
+    lines.push("## Summary — cold median");
+    lines.push("");
+    lines.push(
+        ...table(
+            ["scenario", "nodes", "conc", ...tools],
+            suite.scenarios.map((s) => [
+                s.name,
+                String(s.result.tools[0]?.taskGraphSize ?? 0),
+                String(s.result.concurrency),
+                ...tools.map((t) => coldCell(s, t)),
+            ]),
+        ),
+    );
+    lines.push("");
+
+    // Details.
+    lines.push("## Details");
+    lines.push("");
+    for (const s of suite.scenarios) {
+        const heading = s.description ? `${s.name} — ${s.description}` : s.name;
+        lines.push(`### ${heading}`);
+        lines.push(
+            `Config: ${s.config.projects} projects × ${s.config.tasksPerProject} tasks, ` +
+                `strategy \`${s.config.dependency.strategy}\`.`,
+        );
+        lines.push(formatReport(s.result).trimEnd());
+        lines.push("");
+    }
+
+    return lines.join("\n");
+}

package/src/tools/adapters.spec.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it } from "vitest";
+import { resolveConfig } from "../config";
+import { assertSupportedVersion, getAdapter, type ToolContext } from "./index";
+
+const ctx = (daemon: boolean): ToolContext => ({
+    rootDir: "/tmp/x",
+    projectDirs: ["packages/a"],
+    concurrency: 4,
+    daemon,
+});
+
+describe("tool adapters (runtime)", () => {
+    it("omni has no daemon and pins concurrency", () => {
+        const omni = getAdapter("omni");
+        expect(omni.hasDaemon).toBe(false);
+        expect(omni.run("t2", ctx(true)).args).toEqual([
+            "run",
+            "t2",
+            "-u",
+            "stream",
+            "-c",
+            "4",
+        ]);
+        expect(omni.env(ctx(true))).toEqual({});
+    });
+
+    it("turbo toggles the daemon via run flags", () => {
+        const turbo = getAdapter("turbo");
+        expect(turbo.hasDaemon).toBe(true);
+        expect(turbo.run("t2", ctx(true)).args).toContain("--daemon");
+        expect(turbo.run("t2", ctx(true)).args).toContain("--concurrency=4");
+        expect(turbo.run("t2", ctx(false)).args).toContain("--no-daemon");
+    });
+
+    it("nx toggles the daemon via NX_DAEMON and pins parallelism", () => {
+        const nx = getAdapter("nx");
+        expect(nx.hasDaemon).toBe(true);
+        expect(nx.run("t2", ctx(true)).args).toContain("--parallel=4");
+        expect(nx.env(ctx(true)).NX_DAEMON).toBe("true");
+        expect(nx.env(ctx(false)).NX_DAEMON).toBe("false");
+    });
+
+    it("moon has no daemon and runs the :task target", () => {
+        const moon = getAdapter("moon");
+        expect(moon.hasDaemon).toBe(false);
+        expect(moon.run("t2", ctx(true)).args).toEqual([
+            "run",
+            ":t2",
+            "--concurrency",
+            "4",
+        ]);
+    });
+});
+
+describe("tool adapters (versions & dependencies)", () => {
+    const config = resolveConfig();
+
+    it("declares supported version ranges and derives deps from config", () => {
+        expect(getAdapter("turbo").devDependencies(config)).toEqual({
+            turbo: config.versions.turbo,
+        });
+        expect(getAdapter("nx").devDependencies(config)).toEqual({
+            nx: config.versions.nx,
+        });
+        expect(getAdapter("moon").devDependencies(config)).toEqual({
+            "@moonrepo/cli": config.versions.moon,
+        });
+        // omni is external and installs nothing.
+        expect(getAdapter("omni").devDependencies(config)).toEqual({});
+        expect(getAdapter("omni").pinnedVersion(config)).toBeNull();
+    });
+
+    it("accepts the default pinned versions", () => {
+        for (const tool of ["turbo", "nx", "moon"] as const) {
+            const adapter = getAdapter(tool);
+            const version = adapter.pinnedVersion(config);
+            expect(version).not.toBeNull();
+            expect(() =>
+                assertSupportedVersion(adapter, version as string),
+            ).not.toThrow();
+        }
+    });
+
+    it("rejects an unsupported version", () => {
+        expect(() =>
+            assertSupportedVersion(getAdapter("turbo"), "1.5.0"),
+        ).toThrow(/not supported/);
+        expect(() =>
+            assertSupportedVersion(getAdapter("nx"), "19.0.0"),
+        ).toThrow(/not supported/);
+        expect(() =>
+            assertSupportedVersion(getAdapter("moon"), "1.0.0"),
+        ).toThrow(/not supported/);
+    });
+});

package/src/tools/config.spec.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from "vitest";
+import { parse as parseYaml } from "yaml";
+import { resolveConfig } from "../config";
+import { buildGraph, type ProjectNode } from "../graph";
+import { moonProjectConfig } from "./moon";
+import { nxProjectConfig, nxRootConfig } from "./nx";
+import { omniProjectConfig } from "./omni";
+import { turboRootConfig } from "./turbo";
+import { taskDependencies } from "./types";
+
+describe("taskDependencies", () => {
+    it("chains within a project and fans upstream by default", () => {
+        const config = resolveConfig();
+        expect(taskDependencies(config, 0)).toEqual(["^t0"]);
+        expect(taskDependencies(config, 2)).toEqual(["t1", "^t2"]);
+    });
+
+    it("respects disabled chaining / fan-out", () => {
+        const config = resolveConfig({
+            task: { chainWithinProject: false, fanUpstream: false },
+        });
+        expect(taskDependencies(config, 2)).toEqual([]);
+    });
+});
+
+describe("equivalence across runners", () => {
+    const config = resolveConfig({
+        projects: 6,
+        tasksPerProject: 2,
+        dependency: { strategy: "chain" },
+    });
+    const projects = buildGraph(config);
+    const project = projects[3] as ProjectNode;
+    const parent = projects[2] as ProjectNode;
+
+    it("omni encodes the same task deps and project deps", () => {
+        const doc = parseYaml(omniProjectConfig(config, project, projects));
+        expect(doc.name).toBe(project.name);
+        expect(doc.dependencies).toEqual([parent.name]);
+        expect(doc.tasks.t1.dependencies).toEqual(["t0", "^t1"]);
+        expect(doc.tasks.t1.cache.output.files).toEqual(["dist/t1.*"]);
+    });
+
+    it("turbo encodes matching dependsOn / outputs", () => {
+        const turbo = JSON.parse(turboRootConfig(config));
+        expect(turbo.tasks.t1.dependsOn).toEqual(["t0", "^t1"]);
+        expect(turbo.tasks.t1.outputs).toEqual(["dist/t1.*"]);
+        expect(turbo.globalPassThroughEnv).toContain("TASK_BENCH_EXEC_LOG");
+    });
+
+    it("nx encodes matching dependsOn / outputs and per-project targets", () => {
+        const nx = JSON.parse(nxRootConfig(config));
+        expect(nx.targetDefaults.t1.dependsOn).toEqual(["t0", "^t1"]);
+        expect(nx.targetDefaults.t1.outputs).toEqual([
+            "{projectRoot}/dist/t1.*",
+        ]);
+        expect(nx.targetDefaults.t1.cache).toBe(true);
+
+        const proj = JSON.parse(nxProjectConfig(config, project));
+        expect(proj.name).toBe(project.name);
+        expect(proj.targets.t1.executor).toBe("nx:run-commands");
+        expect(proj.targets.t1.options.command).toBe("node ./task.mjs t1");
+    });
+
+    it("moon encodes matching deps / outputs and project deps", () => {
+        const doc = parseYaml(moonProjectConfig(config, project, projects));
+        expect(doc.id).toBe(project.name);
+        expect(doc.dependsOn).toEqual([parent.name]);
+        expect(doc.tasks.t1.deps).toEqual(["~:t0", "^:t1"]);
+        expect(doc.tasks.t1.command).toBe("node ./task.mjs t1");
+        expect(doc.tasks.t1.outputs).toEqual(["dist/t1.*"]);
+    });
+});