@omni-oss/task-bench 0.0.0-beta.1

package/src/tools/index.ts

@@ -0,0 +1,76 @@
+import { satisfies } from "semver";
+import type { HarnessConfig, Tool } from "../config";
+import { moonAdapter } from "./moon";
+import { nxAdapter } from "./nx";
+import { omniAdapter } from "./omni";
+import { turboAdapter } from "./turbo";
+import type { ToolAdapter } from "./types";
+
+const ADAPTERS: Record<Tool, ToolAdapter> = {
+    omni: omniAdapter,
+    turbo: turboAdapter,
+    nx: nxAdapter,
+    moon: moonAdapter,
+};
+
+export function getAdapter(tool: Tool): ToolAdapter {
+    return ADAPTERS[tool];
+}
+
+export function getAdapters(tools: Tool[]): ToolAdapter[] {
+    return tools.map(getAdapter);
+}
+
+/** Throw if `version` does not satisfy any of the adapter's supported ranges. */
+export function assertSupportedVersion(
+    adapter: ToolAdapter,
+    version: string,
+): void {
+    const ok = adapter.supportedVersions.some((range) =>
+        satisfies(version, range, { includePrerelease: true, loose: true }),
+    );
+    if (!ok) {
+        throw new Error(
+            `${adapter.tool} version "${version}" is not supported by task-bench ` +
+                `(supported: ${adapter.supportedVersions.join(" || ")}). ` +
+                `Adjust versions.${adapter.tool} in your config or upgrade/downgrade the tool.`,
+        );
+    }
+}
+
+/**
+ * Resolve each enabled tool's version (pinned via config, or detected for
+ * external tools) and validate it against the adapter's supported ranges.
+ * Throws on the first unsupported version. Returns a map of tool -> version.
+ */
+export async function resolveToolVersions(
+    config: HarnessConfig,
+    rootDir: string,
+    tools: Tool[] = config.tools,
+): Promise<Map<Tool, string | null>> {
+    const versions = new Map<Tool, string | null>();
+    for (const tool of tools) {
+        const adapter = getAdapter(tool);
+        let version = adapter.pinnedVersion(config);
+        if (version === null && adapter.detectVersion) {
+            version = await adapter.detectVersion(rootDir);
+        }
+        if (version !== null) {
+            assertSupportedVersion(adapter, version);
+        }
+        versions.set(tool, version);
+    }
+    return versions;
+}
+
+export {
+    moonAdapter,
+    moonProjectConfig,
+    moonTaskDependencies,
+    moonToolchainConfig,
+    moonWorkspaceConfig,
+} from "./moon";
+export { nxAdapter, nxProjectConfig, nxRootConfig } from "./nx";
+export { omniAdapter, omniProjectConfig, omniWorkspaceConfig } from "./omni";
+export { turboAdapter, turboRootConfig } from "./turbo";
+export * from "./types";

package/src/tools/moon.ts

@@ -0,0 +1,106 @@
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import { stringify as stringifyYaml } from "yaml";
+import type { HarnessConfig } from "../config";
+import { type ProjectNode, taskNames } from "../graph";
+import {
+    dependencyNames,
+    type GenerationContext,
+    removeDist,
+    resolveBin,
+    type ToolAdapter,
+    type ToolContext,
+} from "./types";
+
+const WORKSPACE_SCHEMA = "https://moonrepo.dev/schemas/workspace.json";
+const TOOLCHAIN_SCHEMA = "https://moonrepo.dev/schemas/toolchain.json";
+const PROJECT_SCHEMA = "https://moonrepo.dev/schemas/project.json";
+
+/** moon dependency edges for task index `k`, using moon's target syntax. */
+export function moonTaskDependencies(
+    config: HarnessConfig,
+    k: number,
+): string[] {
+    const deps: string[] = [];
+    if (config.task.chainWithinProject && k > 0) deps.push(`~:t${k - 1}`);
+    if (config.task.fanUpstream) deps.push(`^:t${k}`);
+    return deps;
+}
+
+export function moonWorkspaceConfig(): string {
+    return stringifyYaml({
+        $schema: WORKSPACE_SCHEMA,
+        projects: ["packages/*"],
+    });
+}
+
+export function moonToolchainConfig(): string {
+    // Intentionally empty: no managed toolchain, so moon runs `node` from the
+    // system PATH (kept equivalent to the other runners).
+    return stringifyYaml({ $schema: TOOLCHAIN_SCHEMA });
+}
+
+export function moonProjectConfig(
+    config: HarnessConfig,
+    project: ProjectNode,
+    projects: ProjectNode[],
+): string {
+    const tasks: Record<string, unknown> = {};
+    taskNames(config).forEach((task, k) => {
+        const deps = moonTaskDependencies(config, k);
+        tasks[task] = {
+            command: `node ./task.mjs ${task}`,
+            ...(deps.length ? { deps } : {}),
+            inputs: ["package.json", "task.mjs", "src/**/*"],
+            outputs: [`dist/${task}.*`],
+        };
+    });
+
+    const doc: Record<string, unknown> = {
+        $schema: PROJECT_SCHEMA,
+        id: project.name,
+        layer: "library",
+        language: "javascript",
+        ...(project.dependencies.length
+            ? { dependsOn: dependencyNames(project, projects) }
+            : {}),
+        tasks,
+    };
+
+    return stringifyYaml(doc);
+}
+
+export const moonAdapter: ToolAdapter = {
+    tool: "moon",
+    hasDaemon: false,
+    supportedVersions: ["^2.0.0"],
+
+    pinnedVersion: (config) => config.versions.moon,
+    devDependencies: (config) => ({ "@moonrepo/cli": config.versions.moon }),
+    setup: async (ctx: GenerationContext) => {
+        await ctx.write(".moon/workspace.yml", moonWorkspaceConfig());
+        await ctx.write(".moon/toolchain.yml", moonToolchainConfig());
+        for (const project of ctx.projects) {
+            await ctx.write(
+                `${project.dir}/moon.yml`,
+                moonProjectConfig(ctx.config, project, ctx.projects),
+            );
+        }
+    },
+
+    run: (task, ctx) => ({
+        file: resolveBin(ctx.rootDir, "moon"),
+        args: ["run", `:${task}`, "--concurrency", String(ctx.concurrency)],
+    }),
+    env: () => ({}),
+    clearCaches: async (ctx: ToolContext) => {
+        await removeDist(ctx);
+        await rm(join(ctx.rootDir, ".moon", "cache"), {
+            recursive: true,
+            force: true,
+        });
+    },
+    stopDaemon: async () => {
+        // moon has no persistent daemon.
+    },
+};

package/src/tools/nx.ts

@@ -0,0 +1,106 @@
+import { execa } from "execa";
+import type { HarnessConfig } from "../config";
+import { type ProjectNode, taskNames } from "../graph";
+import {
+    type GenerationContext,
+    removeDist,
+    resolveBin,
+    type ToolAdapter,
+    type ToolContext,
+    taskDependencies,
+} from "./types";
+
+export function nxRootConfig(config: HarnessConfig): string {
+    const targetDefaults: Record<string, unknown> = {};
+    taskNames(config).forEach((task, k) => {
+        targetDefaults[task] = {
+            dependsOn: taskDependencies(config, k),
+            cache: true,
+            outputs: [`{projectRoot}/dist/${task}.*`],
+            inputs: [
+                "{projectRoot}/package.json",
+                "{projectRoot}/task.mjs",
+                "{projectRoot}/src/**/*",
+            ],
+        };
+    });
+    return `${JSON.stringify(
+        {
+            $schema: "./node_modules/nx/schemas/nx-schema.json",
+            targetDefaults,
+        },
+        null,
+        2,
+    )}\n`;
+}
+
+export function nxProjectConfig(
+    config: HarnessConfig,
+    project: ProjectNode,
+): string {
+    const targets: Record<string, unknown> = {};
+    for (const task of taskNames(config)) {
+        targets[task] = {
+            executor: "nx:run-commands",
+            options: { command: `node ./task.mjs ${task}`, cwd: project.dir },
+        };
+    }
+    return `${JSON.stringify(
+        {
+            name: project.name,
+            $schema: "../../node_modules/nx/schemas/project-schema.json",
+            targets,
+        },
+        null,
+        2,
+    )}\n`;
+}
+
+export const nxAdapter: ToolAdapter = {
+    tool: "nx",
+    hasDaemon: true,
+    supportedVersions: [">=21.0.0 <24.0.0"],
+
+    pinnedVersion: (config) => config.versions.nx,
+    devDependencies: (config) => ({ nx: config.versions.nx }),
+    setup: async (ctx: GenerationContext) => {
+        await ctx.write("nx.json", nxRootConfig(ctx.config));
+        for (const project of ctx.projects) {
+            await ctx.write(
+                `${project.dir}/project.json`,
+                nxProjectConfig(ctx.config, project),
+            );
+        }
+    },
+
+    run: (task, ctx) => ({
+        file: resolveBin(ctx.rootDir, "nx"),
+        args: ["run-many", "-t", task, `--parallel=${ctx.concurrency}`],
+    }),
+    env: (ctx) => ({
+        NX_DAEMON: ctx.daemon ? "true" : "false",
+        NX_TUI: "false",
+    }),
+    clearCaches: async (ctx: ToolContext) => {
+        await removeDist(ctx);
+        // nx 23 stores task results in a DB under `.nx/workspace-data` (used
+        // even in no-daemon mode), which `--onlyCache` does NOT clear. A full
+        // `nx reset` is the only reliable way to force a genuinely cold run.
+        // clearCaches is only invoked for cold runs, so tearing down the daemon
+        // too is acceptable (warm runs never call this).
+        await execa(resolveBin(ctx.rootDir, "nx"), ["reset"], {
+            cwd: ctx.rootDir,
+            reject: false,
+            stdio: "ignore",
+            env: { NX_DAEMON: ctx.daemon ? "true" : "false", NX_TUI: "false" },
+        });
+    },
+    stopDaemon: async (ctx: ToolContext) => {
+        await execa(resolveBin(ctx.rootDir, "nx"), ["reset"], {
+            cwd: ctx.rootDir,
+            reject: false,
+            stdio: "ignore",
+            env: { NX_DAEMON: "false" },
+        });
+    },
+};

package/src/tools/omni.ts

@@ -0,0 +1,96 @@
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import { execa } from "execa";
+import { stringify as stringifyYaml } from "yaml";
+import type { HarnessConfig } from "../config";
+import { type ProjectNode, taskNames } from "../graph";
+import {
+    dependencyNames,
+    type GenerationContext,
+    removeDist,
+    type ToolAdapter,
+    type ToolContext,
+    taskDependencies,
+} from "./types";
+
+const SCHEMA =
+    "# yaml-language-server: $schema=https://raw.githubusercontent.com/omni-oss/json-schemas/refs/heads/main/project.json";
+const WORKSPACE_SCHEMA =
+    "# yaml-language-server: $schema=https://raw.githubusercontent.com/omni-oss/json-schemas/refs/heads/main/workspace.json";
+
+export function omniWorkspaceConfig(): string {
+    const body = stringifyYaml({ ui: "stream", projects: ["packages/*"] });
+    return `${WORKSPACE_SCHEMA}\n${body}`;
+}
+
+export function omniProjectConfig(
+    config: HarnessConfig,
+    project: ProjectNode,
+    projects: ProjectNode[],
+): string {
+    const tasks: Record<string, unknown> = {};
+    taskNames(config).forEach((task, k) => {
+        const deps = taskDependencies(config, k);
+        tasks[task] = {
+            exec: `node ./task.mjs ${task}`,
+            ...(deps.length ? { dependencies: deps } : {}),
+            cache: { output: { files: [`dist/${task}.*`] } },
+        };
+    });
+
+    const doc: Record<string, unknown> = {
+        name: project.name,
+        ...(project.dependencies.length
+            ? { dependencies: dependencyNames(project, projects) }
+            : {}),
+        cache: { key: { files: ["package.json", "task.mjs", "src/**/*.js"] } },
+        tasks,
+    };
+
+    return `${SCHEMA}\n${stringifyYaml(doc)}`;
+}
+
+export const omniAdapter: ToolAdapter = {
+    tool: "omni",
+    hasDaemon: false,
+    // omni is a host-provided binary, not installed per-workspace.
+    supportedVersions: [">=0.16.0"],
+
+    pinnedVersion: () => null,
+    detectVersion: async (rootDir) => {
+        const result = await execa("omni", ["--version"], {
+            cwd: rootDir,
+            reject: false,
+        });
+        const match = `${result.stdout} ${result.stderr}`.match(
+            /(\d+\.\d+\.\d+[\w.-]*)/,
+        );
+        return match?.[1] ?? null;
+    },
+    devDependencies: () => ({}),
+    setup: async (ctx: GenerationContext) => {
+        await ctx.write("workspace.omni.yaml", omniWorkspaceConfig());
+        for (const project of ctx.projects) {
+            await ctx.write(
+                `${project.dir}/project.omni.yaml`,
+                omniProjectConfig(ctx.config, project, ctx.projects),
+            );
+        }
+    },
+
+    run: (task, ctx) => ({
+        file: "omni",
+        args: ["run", task, "-u", "stream", "-c", String(ctx.concurrency)],
+    }),
+    env: () => ({}),
+    clearCaches: async (ctx: ToolContext) => {
+        await removeDist(ctx);
+        await rm(join(ctx.rootDir, ".omni", "cache"), {
+            recursive: true,
+            force: true,
+        });
+    },
+    stopDaemon: async () => {
+        // omni has no persistent daemon.
+    },
+};

package/src/tools/turbo.ts

@@ -0,0 +1,78 @@
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import { execa } from "execa";
+import type { HarnessConfig } from "../config";
+import { taskNames } from "../graph";
+import {
+    type GenerationContext,
+    removeDist,
+    resolveBin,
+    type ToolAdapter,
+    type ToolContext,
+    taskDependencies,
+} from "./types";
+
+export function turboRootConfig(config: HarnessConfig): string {
+    const tasks: Record<string, unknown> = {};
+    taskNames(config).forEach((task, k) => {
+        tasks[task] = {
+            dependsOn: taskDependencies(config, k),
+            outputs: [`dist/${task}.*`],
+            inputs: ["package.json", "task.mjs", "src/**"],
+        };
+    });
+    return `${JSON.stringify(
+        {
+            $schema: "https://turbo.build/schema.json",
+            // Turbo 2.x defaults to strict env mode, which would strip the
+            // benchmark's execution-marker var. Pass it through (not hashed)
+            // so cache-hit verification works without perturbing cache keys.
+            globalPassThroughEnv: ["TASK_BENCH_EXEC_LOG"],
+            tasks,
+        },
+        null,
+        2,
+    )}\n`;
+}
+
+export const turboAdapter: ToolAdapter = {
+    tool: "turbo",
+    hasDaemon: true,
+    supportedVersions: ["^2.0.0"],
+
+    pinnedVersion: (config) => config.versions.turbo,
+    devDependencies: (config) => ({ turbo: config.versions.turbo }),
+    setup: async (ctx: GenerationContext) => {
+        await ctx.write("turbo.json", turboRootConfig(ctx.config));
+    },
+
+    run: (task, ctx) => ({
+        file: resolveBin(ctx.rootDir, "turbo"),
+        args: [
+            "run",
+            task,
+            "--log-order=stream",
+            `--concurrency=${ctx.concurrency}`,
+            ctx.daemon ? "--daemon" : "--no-daemon",
+        ],
+    }),
+    env: () => ({}),
+    clearCaches: async (ctx: ToolContext) => {
+        await removeDist(ctx);
+        await rm(join(ctx.rootDir, ".turbo"), {
+            recursive: true,
+            force: true,
+        });
+        await rm(join(ctx.rootDir, "node_modules", ".cache", "turbo"), {
+            recursive: true,
+            force: true,
+        });
+    },
+    stopDaemon: async (ctx: ToolContext) => {
+        await execa(resolveBin(ctx.rootDir, "turbo"), ["daemon", "stop"], {
+            cwd: ctx.rootDir,
+            reject: false,
+            stdio: "ignore",
+        });
+    },
+};

package/src/tools/types.ts

@@ -0,0 +1,116 @@
+import { existsSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import { join } from "node:path";
+import type { HarnessConfig, Tool } from "../config";
+import type { ProjectNode } from "../graph";
+
+/** Writes a workspace-relative file (creating parent dirs) and records it. */
+export type WorkspaceWriter = (
+    relPath: string,
+    contents: string,
+) => Promise<void>;
+
+/** Everything a tool needs to write its configuration after generation. */
+export interface GenerationContext {
+    /** Absolute path to the generated workspace root. */
+    rootDir: string;
+    config: HarnessConfig;
+    /** All generated projects, in index order. */
+    projects: ProjectNode[];
+    /** The resolved version of this tool (pinned or detected), if known. */
+    version: string | null;
+    /** Write a file into the workspace. */
+    write: WorkspaceWriter;
+}
+
+/** Runtime context for executing/benchmarking a tool. */
+export interface ToolContext {
+    /** Absolute path to the generated workspace root. */
+    rootDir: string;
+    /** Workspace-relative project directories, e.g. `packages/bench-p0001`. */
+    projectDirs: string[];
+    /** Concurrency applied identically to every runner. */
+    concurrency: number;
+    /** Whether the tool's persistent daemon (if any) is allowed. */
+    daemon: boolean;
+}
+
+export interface RunInvocation {
+    file: string;
+    args: string[];
+}
+
+/**
+ * A self-contained integration for one task runner. Each adapter owns:
+ *   - which tool versions it supports (`supportedVersions`),
+ *   - the npm dependencies it needs (`devDependencies`),
+ *   - how to write its own configuration (`setup`),
+ *   - and how to run / reset / clean it up at benchmark time.
+ * This keeps every tool decoupled from the generator and from each other.
+ */
+export interface ToolAdapter {
+    tool: Tool;
+    /** Whether this runner has a persistent daemon that can boost warm perf. */
+    hasDaemon: boolean;
+    /** Semver ranges of the tool version this adapter supports. */
+    supportedVersions: readonly string[];
+
+    /** Version pinned via config for installable tools; null for external ones. */
+    pinnedVersion(config: HarnessConfig): string | null;
+    /** Detect the installed version of an external/global tool (e.g. omni). */
+    detectVersion?(rootDir: string): Promise<string | null>;
+    /** npm devDependencies to add to the root package.json. */
+    devDependencies(config: HarnessConfig): Record<string, string>;
+    /** Derive and write this tool's config files from the generated projects. */
+    setup(ctx: GenerationContext): Promise<void>;
+
+    /** Command that runs `task` across every project in the workspace. */
+    run(task: string, ctx: ToolContext): RunInvocation;
+    /** Extra environment variables for each invocation (e.g. daemon toggles). */
+    env(ctx: ToolContext): Record<string, string>;
+    /** Remove caches and all task outputs. */
+    clearCaches(ctx: ToolContext): Promise<void>;
+    /** Stop the persistent daemon, if any. Used for cold runs and cleanup. */
+    stopDaemon(ctx: ToolContext): Promise<void>;
+}
+
+/** Resolve a locally-installed binary, falling back to the global name. */
+export function resolveBin(rootDir: string, name: string): string {
+    const local = join(rootDir, "node_modules", ".bin", name);
+    return existsSync(local) ? local : name;
+}
+
+/** Remove the `dist/` output directory of every project. */
+export async function removeDist(ctx: ToolContext): Promise<void> {
+    await Promise.all(
+        ctx.projectDirs.map((dir) =>
+            rm(join(ctx.rootDir, dir, "dist"), {
+                recursive: true,
+                force: true,
+            }),
+        ),
+    );
+}
+
+/**
+ * Dependency edges for task index `k`, shared by omni/turbo/nx so their graphs
+ * stay equivalent:
+ *   - within-project: `t{k-1}` (if enabled and k > 0)
+ *   - upstream:       `^t{k}`  (if enabled)
+ */
+export function taskDependencies(config: HarnessConfig, k: number): string[] {
+    const deps: string[] = [];
+    if (config.task.chainWithinProject && k > 0) deps.push(`t${k - 1}`);
+    if (config.task.fanUpstream) deps.push(`^t${k}`);
+    return deps;
+}
+
+/** Map upstream project indices to their names for a project's dependencies. */
+export function dependencyNames(
+    project: ProjectNode,
+    projects: ProjectNode[],
+): string[] {
+    return project.dependencies.map((i) => projects[i]?.name ?? "");
+}
+
+export type { Tool };

package/tsconfig.json

@@ -0,0 +1,4 @@
+{
+	"extends": ["@omni-oss/tsconfig/with-paths.json", "./tsconfig.project.json"],
+    "compilerOptions": {}
+}

package/tsconfig.project.json

@@ -0,0 +1,6 @@
+{
+    "compilerOptions": {
+        "outDir": "./dist",
+        "types": ["node"]
+    }
+}

package/tsconfig.types.json

@@ -0,0 +1,4 @@
+{
+    "extends": ["@omni-oss/tsconfig/types.json", "./tsconfig.project.json"],
+    "compilerOptions": {}
+}

package/vite.config.ts

@@ -0,0 +1,29 @@
+
+import { createConfig } from "@omni-oss/vite-config/script";
+
+
+import packageJson from "./package.json";
+
+export default createConfig({
+    package: packageJson,
+    
+    generateTypes: true,
+    externalizeDeps: true,
+    
+    overrides: {
+        build: {
+            lib: {
+                entry: {
+                    
+                    "task-bench-cli": "src/cli/index.ts",
+                    
+                    "index": "src/index.ts",
+                },
+                formats: ["es"],
+                fileName: (format, entryName) =>
+                    `${entryName || "task-bench"}.${format === "cjs" ? "cjs" : "mjs"}`,
+                name: "TaskBench",
+            },
+        },
+    },
+});

package/vitest.config.unit.ts

@@ -0,0 +1,13 @@
+import { mergeConfig, type UserWorkspaceConfig } from "vitest/config";
+import baseConfig from "./vite.config";
+import unitTestConfig from "@omni-oss/vitest-config/unit";
+
+export default mergeConfig(mergeConfig(baseConfig, unitTestConfig), {
+    test: {
+        testTimeout: 1000,
+        include: ["./src/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}"],
+        exclude: [
+            "./src/**/__tests__/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}",
+        ],
+    },
+} satisfies UserWorkspaceConfig);