@wingman-ai/gateway 0.4.2 → 0.4.4

package/dist/bench/adapters/wingmanCliAdapter.js

@@ -0,0 +1,66 @@
+import { existsSync } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
+import { runCommand } from "../process.js";
+import { parseWingmanJsonOutput } from "./helpers.js";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function resolveAgent(config, context) {
+    return context.task.adapterOverrides?.agent || config.agent;
+}
+function resolveArgs(config, context) {
+    const overrideArgs = context.task.adapterOverrides?.extraArgs || [];
+    const extraArgs = [
+        ...config.extraArgs || [],
+        ...overrideArgs
+    ];
+    const cliArgs = [
+        "agent",
+        "--local",
+        "--output=json",
+        "--agent",
+        resolveAgent(config, context),
+        ...extraArgs,
+        context.task.prompt
+    ];
+    return cliArgs;
+}
+class WingmanCliAdapter {
+    async invoke(context) {
+        const configuredCommand = this.config.cliPath || "./bin/wingman";
+        const cliEntryPath = isAbsolute(configuredCommand) ? configuredCommand : resolve(process.cwd(), configuredCommand);
+        const args = resolveArgs(this.config, context);
+        const runtimeCommand = existsSync(process.execPath) ? process.execPath : "bun";
+        const execution = await runCommand(runtimeCommand, [
+            cliEntryPath,
+            ...args
+        ], {
+            cwd: context.workingDirectory,
+            timeoutMs: context.timeoutMs,
+            env: this.config.env
+        });
+        const parsed = parseWingmanJsonOutput(execution.stdout);
+        return {
+            exitCode: execution.exitCode,
+            timedOut: execution.timedOut,
+            durationMs: execution.durationMs,
+            stdout: execution.stdout,
+            stderr: execution.stderr,
+            assistantText: parsed.assistantText,
+            errorMessage: parsed.errorMessage || (0 === execution.exitCode ? void 0 : execution.stderr.trim()),
+            tokens: parsed.tokenUsage
+        };
+    }
+    constructor(config){
+        _define_property(this, "config", void 0);
+        this.config = config;
+    }
+}
+export { WingmanCliAdapter };

package/dist/bench/cleanup.cjs

@@ -0,0 +1,122 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    cleanBenchArtifacts: ()=>cleanBenchArtifacts,
+    getBenchCleanupTargets: ()=>getBenchCleanupTargets
+});
+const promises_namespaceObject = require("node:fs/promises");
+const external_node_path_namespaceObject = require("node:path");
+async function listImmediateChildren(dirPath) {
+    try {
+        const entries = await (0, promises_namespaceObject.readdir)(dirPath, {
+            withFileTypes: true,
+            encoding: "utf8"
+        });
+        return entries.map((entry)=>(0, external_node_path_namespaceObject.join)(dirPath, entry.name));
+    } catch  {
+        return [];
+    }
+}
+async function collectPycacheDirs(rootPath) {
+    const discovered = [];
+    const queue = [
+        rootPath
+    ];
+    while(queue.length > 0){
+        const current = queue.shift();
+        if (!current) continue;
+        let entries;
+        try {
+            entries = await (0, promises_namespaceObject.readdir)(current, {
+                withFileTypes: true,
+                encoding: "utf8"
+            });
+        } catch  {
+            continue;
+        }
+        for (const entry of entries){
+            if (!entry.isDirectory()) continue;
+            const entryName = "string" == typeof entry.name ? entry.name : String(entry.name);
+            const child = (0, external_node_path_namespaceObject.join)(current, entryName);
+            if ("__pycache__" === entryName) {
+                discovered.push(child);
+                continue;
+            }
+            queue.push(child);
+        }
+    }
+    return discovered;
+}
+async function getBenchCleanupTargets(workspaceRoot) {
+    const root = (0, external_node_path_namespaceObject.resolve)(workspaceRoot);
+    const jobsDir = (0, external_node_path_namespaceObject.join)(root, "jobs");
+    const officialResultsDir = (0, external_node_path_namespaceObject.join)(root, "bench", "results", "official");
+    const wrapperResultsDir = (0, external_node_path_namespaceObject.join)(root, "bench", "results", "official-wrapper");
+    const benchDir = (0, external_node_path_namespaceObject.join)(root, "bench");
+    const [jobArtifacts, officialArtifacts, wrapperArtifacts, pycacheDirs] = await Promise.all([
+        listImmediateChildren(jobsDir),
+        listImmediateChildren(officialResultsDir),
+        listImmediateChildren(wrapperResultsDir),
+        collectPycacheDirs(benchDir)
+    ]);
+    return Array.from(new Set([
+        ...jobArtifacts,
+        ...officialArtifacts,
+        ...wrapperArtifacts,
+        ...pycacheDirs
+    ]));
+}
+async function cleanBenchArtifacts(workspaceRoot, options = {}) {
+    const targets = await getBenchCleanupTargets(workspaceRoot);
+    const removedPaths = [];
+    const missingPaths = [];
+    for (const target of targets)try {
+        if (options.dryRun) {
+            removedPaths.push(target);
+            continue;
+        }
+        await (0, promises_namespaceObject.rm)(target, {
+            recursive: true,
+            force: false
+        });
+        removedPaths.push(target);
+    } catch  {
+        missingPaths.push(target);
+    }
+    return {
+        removedPaths,
+        missingPaths
+    };
+}
+exports.cleanBenchArtifacts = __webpack_exports__.cleanBenchArtifacts;
+exports.getBenchCleanupTargets = __webpack_exports__.getBenchCleanupTargets;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "cleanBenchArtifacts",
+    "getBenchCleanupTargets"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/bench/cleanup.d.ts

@@ -0,0 +1,9 @@
+export interface BenchCleanupResult {
+    removedPaths: string[];
+    missingPaths: string[];
+}
+export interface BenchCleanupOptions {
+    dryRun?: boolean;
+}
+export declare function getBenchCleanupTargets(workspaceRoot: string): Promise<string[]>;
+export declare function cleanBenchArtifacts(workspaceRoot: string, options?: BenchCleanupOptions): Promise<BenchCleanupResult>;

package/dist/bench/cleanup.js

@@ -0,0 +1,85 @@
+import { readdir, rm } from "node:fs/promises";
+import { join, resolve } from "node:path";
+async function listImmediateChildren(dirPath) {
+    try {
+        const entries = await readdir(dirPath, {
+            withFileTypes: true,
+            encoding: "utf8"
+        });
+        return entries.map((entry)=>join(dirPath, entry.name));
+    } catch  {
+        return [];
+    }
+}
+async function collectPycacheDirs(rootPath) {
+    const discovered = [];
+    const queue = [
+        rootPath
+    ];
+    while(queue.length > 0){
+        const current = queue.shift();
+        if (!current) continue;
+        let entries;
+        try {
+            entries = await readdir(current, {
+                withFileTypes: true,
+                encoding: "utf8"
+            });
+        } catch  {
+            continue;
+        }
+        for (const entry of entries){
+            if (!entry.isDirectory()) continue;
+            const entryName = "string" == typeof entry.name ? entry.name : String(entry.name);
+            const child = join(current, entryName);
+            if ("__pycache__" === entryName) {
+                discovered.push(child);
+                continue;
+            }
+            queue.push(child);
+        }
+    }
+    return discovered;
+}
+async function getBenchCleanupTargets(workspaceRoot) {
+    const root = resolve(workspaceRoot);
+    const jobsDir = join(root, "jobs");
+    const officialResultsDir = join(root, "bench", "results", "official");
+    const wrapperResultsDir = join(root, "bench", "results", "official-wrapper");
+    const benchDir = join(root, "bench");
+    const [jobArtifacts, officialArtifacts, wrapperArtifacts, pycacheDirs] = await Promise.all([
+        listImmediateChildren(jobsDir),
+        listImmediateChildren(officialResultsDir),
+        listImmediateChildren(wrapperResultsDir),
+        collectPycacheDirs(benchDir)
+    ]);
+    return Array.from(new Set([
+        ...jobArtifacts,
+        ...officialArtifacts,
+        ...wrapperArtifacts,
+        ...pycacheDirs
+    ]));
+}
+async function cleanBenchArtifacts(workspaceRoot, options = {}) {
+    const targets = await getBenchCleanupTargets(workspaceRoot);
+    const removedPaths = [];
+    const missingPaths = [];
+    for (const target of targets)try {
+        if (options.dryRun) {
+            removedPaths.push(target);
+            continue;
+        }
+        await rm(target, {
+            recursive: true,
+            force: false
+        });
+        removedPaths.push(target);
+    } catch  {
+        missingPaths.push(target);
+    }
+    return {
+        removedPaths,
+        missingPaths
+    };
+}
+export { cleanBenchArtifacts, getBenchCleanupTargets };

package/dist/bench/config.cjs

@@ -0,0 +1,190 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    loadTerminalBenchConfig: ()=>loadTerminalBenchConfig
+});
+const promises_namespaceObject = require("node:fs/promises");
+const external_node_path_namespaceObject = require("node:path");
+const external_zod_namespaceObject = require("zod");
+const commandSchema = external_zod_namespaceObject.z.object({
+    command: external_zod_namespaceObject.z.string().min(1),
+    args: external_zod_namespaceObject.z.array(external_zod_namespaceObject.z.string()).optional(),
+    shell: external_zod_namespaceObject.z.boolean().optional(),
+    env: external_zod_namespaceObject.z.record(external_zod_namespaceObject.z.string(), external_zod_namespaceObject.z.string()).optional(),
+    allowFailure: external_zod_namespaceObject.z.boolean().optional()
+});
+const includesSchema = external_zod_namespaceObject.z.union([
+    external_zod_namespaceObject.z.string(),
+    external_zod_namespaceObject.z.array(external_zod_namespaceObject.z.string())
+]).transform((value)=>"string" == typeof value ? [
+        value
+    ] : value);
+const validatorSchema = external_zod_namespaceObject.z.union([
+    external_zod_namespaceObject.z.object({
+        type: external_zod_namespaceObject.z.literal("command"),
+        command: commandSchema,
+        expectedExitCode: external_zod_namespaceObject.z.number().int().optional()
+    }),
+    external_zod_namespaceObject.z.object({
+        type: external_zod_namespaceObject.z.literal("assistant_contains"),
+        includes: includesSchema
+    }),
+    external_zod_namespaceObject.z.object({
+        type: external_zod_namespaceObject.z.literal("file_contains"),
+        path: external_zod_namespaceObject.z.string().min(1),
+        includes: includesSchema
+    })
+]);
+const taskSchema = external_zod_namespaceObject.z.object({
+    id: external_zod_namespaceObject.z.string().min(1),
+    description: external_zod_namespaceObject.z.string().optional(),
+    prompt: external_zod_namespaceObject.z.string(),
+    workingDirectory: external_zod_namespaceObject.z.string().optional(),
+    timeoutMs: external_zod_namespaceObject.z.number().int().positive().optional(),
+    setup: external_zod_namespaceObject.z.array(commandSchema).optional(),
+    validator: validatorSchema,
+    metadata: external_zod_namespaceObject.z.record(external_zod_namespaceObject.z.string(), external_zod_namespaceObject.z.string()).optional(),
+    adapterOverrides: external_zod_namespaceObject.z.object({
+        agent: external_zod_namespaceObject.z.string().optional(),
+        extraArgs: external_zod_namespaceObject.z.array(external_zod_namespaceObject.z.string()).optional()
+    }).optional()
+});
+const taskFileSchema = external_zod_namespaceObject.z.object({
+    tasks: external_zod_namespaceObject.z.array(taskSchema).min(1)
+});
+const configSchema = external_zod_namespaceObject.z.object({
+    version: external_zod_namespaceObject.z.literal(1).optional(),
+    taskFile: external_zod_namespaceObject.z.string().min(1),
+    resultsDir: external_zod_namespaceObject.z.string().optional(),
+    run: external_zod_namespaceObject.z.object({
+        defaultTimeoutMs: external_zod_namespaceObject.z.number().int().positive().optional(),
+        continueOnFailure: external_zod_namespaceObject.z.boolean().optional()
+    }).optional(),
+    adapter: external_zod_namespaceObject.z.union([
+        external_zod_namespaceObject.z.object({
+            type: external_zod_namespaceObject.z.literal("wingman-cli"),
+            agent: external_zod_namespaceObject.z.string().min(1),
+            cliPath: external_zod_namespaceObject.z.string().optional(),
+            extraArgs: external_zod_namespaceObject.z.array(external_zod_namespaceObject.z.string()).optional(),
+            env: external_zod_namespaceObject.z.record(external_zod_namespaceObject.z.string(), external_zod_namespaceObject.z.string()).optional()
+        }),
+        external_zod_namespaceObject.z.object({
+            type: external_zod_namespaceObject.z.literal("command"),
+            command: commandSchema
+        })
+    ]),
+    scoring: external_zod_namespaceObject.z.object({
+        weights: external_zod_namespaceObject.z.object({
+            passRate: external_zod_namespaceObject.z.number().nonnegative().optional(),
+            reliability: external_zod_namespaceObject.z.number().nonnegative().optional(),
+            duration: external_zod_namespaceObject.z.number().nonnegative().optional(),
+            cost: external_zod_namespaceObject.z.number().nonnegative().optional()
+        }).optional(),
+        budgets: external_zod_namespaceObject.z.object({
+            targetAvgDurationMs: external_zod_namespaceObject.z.number().positive().optional(),
+            targetCostPerTaskUsd: external_zod_namespaceObject.z.number().nonnegative().optional()
+        }).optional(),
+        pricing: external_zod_namespaceObject.z.object({
+            inputPer1kTokensUsd: external_zod_namespaceObject.z.number().nonnegative().optional(),
+            outputPer1kTokensUsd: external_zod_namespaceObject.z.number().nonnegative().optional()
+        }).optional()
+    }).optional(),
+    qualityGate: external_zod_namespaceObject.z.object({
+        enabled: external_zod_namespaceObject.z.boolean().optional(),
+        baselineFile: external_zod_namespaceObject.z.string().optional(),
+        minPassRateDelta: external_zod_namespaceObject.z.number().optional(),
+        maxCostIncreaseRatio: external_zod_namespaceObject.z.number().nonnegative().optional(),
+        maxAvgDurationIncreaseRatio: external_zod_namespaceObject.z.number().nonnegative().optional()
+    }).optional(),
+    metadata: external_zod_namespaceObject.z.record(external_zod_namespaceObject.z.string(), external_zod_namespaceObject.z.string()).optional()
+});
+function normalizeValidator(validator) {
+    if ("assistant_contains" === validator.type) return {
+        ...validator,
+        includes: validator.includes.filter((entry)=>entry.trim().length > 0)
+    };
+    if ("file_contains" === validator.type) return {
+        ...validator,
+        includes: validator.includes.filter((entry)=>entry.trim().length > 0)
+    };
+    return validator;
+}
+async function loadTerminalBenchConfig(configPath) {
+    const resolvedConfigPath = (0, external_node_path_namespaceObject.resolve)(configPath);
+    const configFile = JSON.parse(await (0, promises_namespaceObject.readFile)(resolvedConfigPath, "utf-8"));
+    const parsedConfig = configSchema.parse(configFile);
+    const configDir = (0, external_node_path_namespaceObject.dirname)(resolvedConfigPath);
+    const taskFilePath = (0, external_node_path_namespaceObject.resolve)(configDir, parsedConfig.taskFile);
+    const tasksFile = JSON.parse(await (0, promises_namespaceObject.readFile)(taskFilePath, "utf-8"));
+    const parsedTasks = taskFileSchema.parse(tasksFile);
+    const tasks = parsedTasks.tasks.map((task)=>({
+            ...task,
+            validator: normalizeValidator(task.validator)
+        }));
+    return {
+        version: 1,
+        configPath: resolvedConfigPath,
+        taskFilePath,
+        resultsDir: (0, external_node_path_namespaceObject.resolve)(configDir, parsedConfig.resultsDir || "bench/results"),
+        run: {
+            defaultTimeoutMs: parsedConfig.run?.defaultTimeoutMs || 300000,
+            continueOnFailure: parsedConfig.run?.continueOnFailure ?? true
+        },
+        adapter: parsedConfig.adapter,
+        tasks,
+        scoring: {
+            weights: {
+                passRate: parsedConfig.scoring?.weights?.passRate ?? 0.7,
+                reliability: parsedConfig.scoring?.weights?.reliability ?? 0.15,
+                duration: parsedConfig.scoring?.weights?.duration ?? 0.1,
+                cost: parsedConfig.scoring?.weights?.cost ?? 0.05
+            },
+            budgets: {
+                targetAvgDurationMs: parsedConfig.scoring?.budgets?.targetAvgDurationMs,
+                targetCostPerTaskUsd: parsedConfig.scoring?.budgets?.targetCostPerTaskUsd
+            },
+            pricing: {
+                inputPer1kTokensUsd: parsedConfig.scoring?.pricing?.inputPer1kTokensUsd ?? 0,
+                outputPer1kTokensUsd: parsedConfig.scoring?.pricing?.outputPer1kTokensUsd ?? 0
+            }
+        },
+        qualityGate: {
+            enabled: parsedConfig.qualityGate?.enabled ?? false,
+            baselineFile: parsedConfig.qualityGate?.baselineFile ? (0, external_node_path_namespaceObject.resolve)(configDir, parsedConfig.qualityGate.baselineFile) : void 0,
+            minPassRateDelta: parsedConfig.qualityGate?.minPassRateDelta ?? -0.03,
+            maxCostIncreaseRatio: parsedConfig.qualityGate?.maxCostIncreaseRatio ?? 0.15,
+            maxAvgDurationIncreaseRatio: parsedConfig.qualityGate?.maxAvgDurationIncreaseRatio ?? 0.2
+        },
+        metadata: parsedConfig.metadata || {}
+    };
+}
+exports.loadTerminalBenchConfig = __webpack_exports__.loadTerminalBenchConfig;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "loadTerminalBenchConfig"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/bench/config.d.ts

@@ -0,0 +1,2 @@
+import type { TerminalBenchResolvedConfig } from "./types.js";
+export declare function loadTerminalBenchConfig(configPath: string): Promise<TerminalBenchResolvedConfig>;

package/dist/bench/config.js

@@ -0,0 +1,156 @@
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { z } from "zod";
+const commandSchema = z.object({
+    command: z.string().min(1),
+    args: z.array(z.string()).optional(),
+    shell: z.boolean().optional(),
+    env: z.record(z.string(), z.string()).optional(),
+    allowFailure: z.boolean().optional()
+});
+const includesSchema = z.union([
+    z.string(),
+    z.array(z.string())
+]).transform((value)=>"string" == typeof value ? [
+        value
+    ] : value);
+const validatorSchema = z.union([
+    z.object({
+        type: z.literal("command"),
+        command: commandSchema,
+        expectedExitCode: z.number().int().optional()
+    }),
+    z.object({
+        type: z.literal("assistant_contains"),
+        includes: includesSchema
+    }),
+    z.object({
+        type: z.literal("file_contains"),
+        path: z.string().min(1),
+        includes: includesSchema
+    })
+]);
+const taskSchema = z.object({
+    id: z.string().min(1),
+    description: z.string().optional(),
+    prompt: z.string(),
+    workingDirectory: z.string().optional(),
+    timeoutMs: z.number().int().positive().optional(),
+    setup: z.array(commandSchema).optional(),
+    validator: validatorSchema,
+    metadata: z.record(z.string(), z.string()).optional(),
+    adapterOverrides: z.object({
+        agent: z.string().optional(),
+        extraArgs: z.array(z.string()).optional()
+    }).optional()
+});
+const taskFileSchema = z.object({
+    tasks: z.array(taskSchema).min(1)
+});
+const configSchema = z.object({
+    version: z.literal(1).optional(),
+    taskFile: z.string().min(1),
+    resultsDir: z.string().optional(),
+    run: z.object({
+        defaultTimeoutMs: z.number().int().positive().optional(),
+        continueOnFailure: z.boolean().optional()
+    }).optional(),
+    adapter: z.union([
+        z.object({
+            type: z.literal("wingman-cli"),
+            agent: z.string().min(1),
+            cliPath: z.string().optional(),
+            extraArgs: z.array(z.string()).optional(),
+            env: z.record(z.string(), z.string()).optional()
+        }),
+        z.object({
+            type: z.literal("command"),
+            command: commandSchema
+        })
+    ]),
+    scoring: z.object({
+        weights: z.object({
+            passRate: z.number().nonnegative().optional(),
+            reliability: z.number().nonnegative().optional(),
+            duration: z.number().nonnegative().optional(),
+            cost: z.number().nonnegative().optional()
+        }).optional(),
+        budgets: z.object({
+            targetAvgDurationMs: z.number().positive().optional(),
+            targetCostPerTaskUsd: z.number().nonnegative().optional()
+        }).optional(),
+        pricing: z.object({
+            inputPer1kTokensUsd: z.number().nonnegative().optional(),
+            outputPer1kTokensUsd: z.number().nonnegative().optional()
+        }).optional()
+    }).optional(),
+    qualityGate: z.object({
+        enabled: z.boolean().optional(),
+        baselineFile: z.string().optional(),
+        minPassRateDelta: z.number().optional(),
+        maxCostIncreaseRatio: z.number().nonnegative().optional(),
+        maxAvgDurationIncreaseRatio: z.number().nonnegative().optional()
+    }).optional(),
+    metadata: z.record(z.string(), z.string()).optional()
+});
+function normalizeValidator(validator) {
+    if ("assistant_contains" === validator.type) return {
+        ...validator,
+        includes: validator.includes.filter((entry)=>entry.trim().length > 0)
+    };
+    if ("file_contains" === validator.type) return {
+        ...validator,
+        includes: validator.includes.filter((entry)=>entry.trim().length > 0)
+    };
+    return validator;
+}
+async function loadTerminalBenchConfig(configPath) {
+    const resolvedConfigPath = resolve(configPath);
+    const configFile = JSON.parse(await readFile(resolvedConfigPath, "utf-8"));
+    const parsedConfig = configSchema.parse(configFile);
+    const configDir = dirname(resolvedConfigPath);
+    const taskFilePath = resolve(configDir, parsedConfig.taskFile);
+    const tasksFile = JSON.parse(await readFile(taskFilePath, "utf-8"));
+    const parsedTasks = taskFileSchema.parse(tasksFile);
+    const tasks = parsedTasks.tasks.map((task)=>({
+            ...task,
+            validator: normalizeValidator(task.validator)
+        }));
+    return {
+        version: 1,
+        configPath: resolvedConfigPath,
+        taskFilePath,
+        resultsDir: resolve(configDir, parsedConfig.resultsDir || "bench/results"),
+        run: {
+            defaultTimeoutMs: parsedConfig.run?.defaultTimeoutMs || 300000,
+            continueOnFailure: parsedConfig.run?.continueOnFailure ?? true
+        },
+        adapter: parsedConfig.adapter,
+        tasks,
+        scoring: {
+            weights: {
+                passRate: parsedConfig.scoring?.weights?.passRate ?? 0.7,
+                reliability: parsedConfig.scoring?.weights?.reliability ?? 0.15,
+                duration: parsedConfig.scoring?.weights?.duration ?? 0.1,
+                cost: parsedConfig.scoring?.weights?.cost ?? 0.05
+            },
+            budgets: {
+                targetAvgDurationMs: parsedConfig.scoring?.budgets?.targetAvgDurationMs,
+                targetCostPerTaskUsd: parsedConfig.scoring?.budgets?.targetCostPerTaskUsd
+            },
+            pricing: {
+                inputPer1kTokensUsd: parsedConfig.scoring?.pricing?.inputPer1kTokensUsd ?? 0,
+                outputPer1kTokensUsd: parsedConfig.scoring?.pricing?.outputPer1kTokensUsd ?? 0
+            }
+        },
+        qualityGate: {
+            enabled: parsedConfig.qualityGate?.enabled ?? false,
+            baselineFile: parsedConfig.qualityGate?.baselineFile ? resolve(configDir, parsedConfig.qualityGate.baselineFile) : void 0,
+            minPassRateDelta: parsedConfig.qualityGate?.minPassRateDelta ?? -0.03,
+            maxCostIncreaseRatio: parsedConfig.qualityGate?.maxCostIncreaseRatio ?? 0.15,
+            maxAvgDurationIncreaseRatio: parsedConfig.qualityGate?.maxAvgDurationIncreaseRatio ?? 0.2
+        },
+        metadata: parsedConfig.metadata || {}
+    };
+}
+export { loadTerminalBenchConfig };

package/dist/bench/index.cjs

@@ -0,0 +1,43 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    runTerminalBench: ()=>external_runner_cjs_namespaceObject.runTerminalBench,
+    runTerminalBenchWithConfig: ()=>external_runner_cjs_namespaceObject.runTerminalBenchWithConfig,
+    loadTerminalBenchConfig: ()=>external_config_cjs_namespaceObject.loadTerminalBenchConfig
+});
+const external_config_cjs_namespaceObject = require("./config.cjs");
+const external_runner_cjs_namespaceObject = require("./runner.cjs");
+exports.loadTerminalBenchConfig = __webpack_exports__.loadTerminalBenchConfig;
+exports.runTerminalBench = __webpack_exports__.runTerminalBench;
+exports.runTerminalBenchWithConfig = __webpack_exports__.runTerminalBenchWithConfig;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "loadTerminalBenchConfig",
+    "runTerminalBench",
+    "runTerminalBenchWithConfig"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/bench/index.d.ts

@@ -0,0 +1,3 @@
+export { loadTerminalBenchConfig } from "./config.js";
+export { runTerminalBench, runTerminalBenchWithConfig } from "./runner.js";
+export type { TaskRunResult, TerminalBenchResolvedConfig, TerminalBenchSummary, } from "./types.js";