agent-regression-lab 0.1.0

package/dist/agent/openaiResponsesAdapter.js

@@ -0,0 +1,155 @@
+import OpenAI from "openai";
+class OpenAIResponsesSession {
+    client;
+    model;
+    input;
+    previousResponseId;
+    pendingToolCall;
+    toolNameMap;
+    providerTools;
+    constructor(client, model, input) {
+        this.client = client;
+        this.model = model;
+        this.input = input;
+        this.providerTools = input.availableTools.map((tool) => ({
+            internalName: tool.name,
+            providerName: toProviderToolName(tool.name),
+            tool,
+        }));
+        this.toolNameMap = new Map(this.providerTools.map((entry) => [entry.providerName, entry.internalName]));
+    }
+    async next(event) {
+        try {
+            const requestBody = {
+                model: this.model,
+                instructions: this.input.instructions,
+                input: this.buildInput(event),
+                tools: this.providerTools.map((entry) => toOpenAITool(entry.providerName, entry.tool)),
+                previous_response_id: this.previousResponseId,
+                parallel_tool_calls: false,
+            };
+            const response = await this.client.responses.create(requestBody);
+            this.previousResponseId = response.id;
+            const output = Array.isArray(response.output) ? response.output : [];
+            const functionCall = output.find((item) => item?.type === "function_call");
+            if (functionCall) {
+                const providerToolName = String(functionCall.name);
+                const internalToolName = this.toolNameMap.get(providerToolName);
+                if (!internalToolName) {
+                    return {
+                        type: "error",
+                        message: `OpenAI requested unknown provider tool '${providerToolName}'.`,
+                    };
+                }
+                this.pendingToolCall = {
+                    callId: String(functionCall.call_id),
+                    providerToolName,
+                };
+                return {
+                    type: "tool_call",
+                    toolName: internalToolName,
+                    input: safeJsonParse(String(functionCall.arguments ?? "{}")),
+                    metadata: {
+                        responseId: response.id,
+                        providerToolName,
+                        message: `OpenAI requested tool ${internalToolName}.`,
+                    },
+                };
+            }
+            const finalOutput = typeof response.output_text === "string" ? response.output_text : "";
+            if (finalOutput) {
+                return {
+                    type: "final",
+                    output: finalOutput,
+                    metadata: {
+                        responseId: response.id,
+                        usage: response.usage ?? undefined,
+                    },
+                };
+            }
+            return {
+                type: "error",
+                message: "OpenAI response did not include a function call or final output.",
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            return { type: "error", message };
+        }
+    }
+    buildInput(event) {
+        if (event.type === "run_started") {
+            return [
+                {
+                    role: "user",
+                    content: [
+                        {
+                            type: "input_text",
+                            text: buildInitialPrompt(this.input),
+                        },
+                    ],
+                },
+            ];
+        }
+        if (event.type === "tool_result") {
+            if (!this.pendingToolCall) {
+                throw new Error("Received tool result without a pending provider tool call.");
+            }
+            const output = typeof event.result === "string" ? event.result : JSON.stringify(event.result);
+            const payload = [
+                {
+                    type: "function_call_output",
+                    call_id: this.pendingToolCall.callId,
+                    output,
+                },
+            ];
+            this.pendingToolCall = undefined;
+            return payload;
+        }
+        throw new Error(event.message);
+    }
+}
+export class OpenAIResponsesAgentAdapter {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    async startRun(input) {
+        if (!this.options.apiKey && !this.options.client) {
+            throw new Error("OPENAI_API_KEY is required for provider=openai.");
+        }
+        const model = typeof input.metadata?.model === "string" && input.metadata.model.length > 0 ? input.metadata.model : "gpt-4o-mini";
+        const client = this.options.client ?? new OpenAI({ apiKey: this.options.apiKey });
+        return new OpenAIResponsesSession(client, model, input);
+    }
+}
+function toOpenAITool(providerName, tool) {
+    return {
+        type: "function",
+        name: providerName,
+        description: tool.description ?? "",
+        parameters: tool.inputSchema ?? {
+            type: "object",
+            additionalProperties: true,
+            properties: {},
+        },
+    };
+}
+function buildInitialPrompt(input) {
+    const context = JSON.stringify(input.context, null, 2);
+    const tools = input.availableTools
+        .map((tool) => `- ${toProviderToolName(tool.name)}: ${tool.description ?? "No description"}`)
+        .join("\n");
+    return `Task:\n${input.instructions}\n\nContext:\n${context}\n\nAvailable tools:\n${tools}\n\nUse tools when needed and provide a final answer when the task is complete.`;
+}
+function toProviderToolName(internalName) {
+    return internalName.replace(/[^a-zA-Z0-9_-]/g, "_");
+}
+function safeJsonParse(value) {
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return {};
+    }
+}

package/dist/config.js

@@ -0,0 +1,123 @@
+import { statSync, readFileSync } from "node:fs";
+import { resolve, relative, sep } from "node:path";
+import { parse } from "yaml";
+const CONFIG_PATH = resolve("agentlab.config.yaml");
+export function loadAgentLabConfig() {
+    if (!exists(CONFIG_PATH)) {
+        return {};
+    }
+    const raw = readFileSync(CONFIG_PATH, "utf8");
+    const parsed = parse(raw);
+    validateConfig(parsed);
+    return parsed;
+}
+function validateConfig(value) {
+    if (!isObject(value)) {
+        throw new Error("agentlab.config.yaml must contain a YAML object.");
+    }
+    if (value.tools !== undefined) {
+        if (!Array.isArray(value.tools)) {
+            throw new Error("agentlab.config.yaml field 'tools' must be an array.");
+        }
+        const names = new Set();
+        for (const tool of value.tools) {
+            validateToolRegistration(tool);
+            if (names.has(tool.name)) {
+                throw new Error(`agentlab.config.yaml defines duplicate tool '${tool.name}'.`);
+            }
+            names.add(tool.name);
+        }
+    }
+    if (value.agents !== undefined) {
+        if (!Array.isArray(value.agents)) {
+            throw new Error("agentlab.config.yaml field 'agents' must be an array.");
+        }
+        const names = new Set();
+        for (const agent of value.agents) {
+            validateAgentRegistration(agent);
+            if (names.has(agent.name)) {
+                throw new Error(`agentlab.config.yaml defines duplicate agent '${agent.name}'.`);
+            }
+            names.add(agent.name);
+        }
+    }
+}
+function validateToolRegistration(value) {
+    if (!isObject(value)) {
+        throw new Error("Each tool registration in agentlab.config.yaml must be an object.");
+    }
+    if (typeof value.name !== "string" || value.name.length === 0) {
+        throw new Error("Each tool registration must define a non-empty 'name'.");
+    }
+    if (typeof value.modulePath !== "string" || value.modulePath.length === 0) {
+        throw new Error(`Tool '${value.name}' must define a non-empty 'modulePath'.`);
+    }
+    if (typeof value.exportName !== "string" || value.exportName.length === 0) {
+        throw new Error(`Tool '${value.name}' must define a non-empty 'exportName'.`);
+    }
+    if (typeof value.description !== "string" || value.description.length === 0) {
+        throw new Error(`Tool '${value.name}' must define a non-empty 'description'.`);
+    }
+    if (!isObject(value.inputSchema)) {
+        throw new Error(`Tool '${value.name}' must define an object 'inputSchema'.`);
+    }
+    const resolved = resolve(value.modulePath);
+    const root = `${process.cwd()}${sep}`;
+    if (!(resolved === process.cwd() || resolved.startsWith(root))) {
+        throw new Error(`Tool '${value.name}' modulePath must stay within the repo.`);
+    }
+    if (!exists(resolved)) {
+        throw new Error(`Tool '${value.name}' references missing module '${relative(process.cwd(), resolved)}'.`);
+    }
+}
+function validateAgentRegistration(value) {
+    if (!isObject(value)) {
+        throw new Error("Each agent registration in agentlab.config.yaml must be an object.");
+    }
+    if (typeof value.name !== "string" || value.name.length === 0) {
+        throw new Error("Each agent registration must define a non-empty 'name'.");
+    }
+    if (value.provider !== "mock" && value.provider !== "openai" && value.provider !== "external_process") {
+        throw new Error(`Agent '${value.name}' uses unsupported provider '${String(value.provider)}'.`);
+    }
+    if (value.label !== undefined && (typeof value.label !== "string" || value.label.length === 0)) {
+        throw new Error(`Agent '${value.name}' must define a non-empty 'label' when provided.`);
+    }
+    if (value.provider === "openai" && value.model !== undefined && (typeof value.model !== "string" || value.model.length === 0)) {
+        throw new Error(`Agent '${value.name}' must define a non-empty 'model' when provided.`);
+    }
+    if (value.provider === "external_process") {
+        if (typeof value.command !== "string" || value.command.length === 0) {
+            throw new Error(`Agent '${value.name}' must define a non-empty 'command'.`);
+        }
+        if (value.args !== undefined) {
+            if (!Array.isArray(value.args) || value.args.some((arg) => typeof arg !== "string")) {
+                throw new Error(`Agent '${value.name}' field 'args' must be an array of strings.`);
+            }
+        }
+        if (value.envAllowlist !== undefined) {
+            if (!Array.isArray(value.envAllowlist) || value.envAllowlist.some((key) => typeof key !== "string" || key.length === 0)) {
+                throw new Error(`Agent '${value.name}' field 'envAllowlist' must be an array of non-empty strings.`);
+            }
+        }
+    }
+}
+export function getAgentRegistration(name) {
+    const match = loadAgentLabConfig().agents?.find((agent) => agent.name === name);
+    if (!match) {
+        throw new Error(`agentlab.config.yaml does not define agent '${name}'.`);
+    }
+    return match;
+}
+function exists(path) {
+    try {
+        statSync(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/dist/evaluators.js

@@ -0,0 +1,109 @@
+export function evaluateScenario(bundle, evaluators) {
+    return evaluators.map((evaluator) => evaluateOne(evaluator, bundle));
+}
+function evaluateOne(evaluator, bundle) {
+    switch (evaluator.type) {
+        case "forbidden_tool":
+            return evaluateForbiddenTool(evaluator, bundle.toolCalls);
+        case "tool_call_assertion":
+            return evaluateToolCallAssertion(evaluator, bundle.toolCalls);
+        case "final_answer_contains":
+            return evaluateFinalAnswerContains(evaluator, bundle.run.finalOutput);
+        case "exact_final_answer":
+            return evaluateExactFinalAnswer(evaluator, bundle.run.finalOutput);
+        case "step_count_max":
+            return evaluateStepCountMax(evaluator, bundle.run.totalSteps);
+        default:
+            return {
+                evaluatorId: evaluator.id,
+                evaluatorType: evaluator.type,
+                mode: evaluator.mode,
+                status: "fail",
+                weight: evaluator.weight,
+                message: `Unsupported evaluator type '${evaluator.type}'.`,
+            };
+    }
+}
+function evaluateForbiddenTool(evaluator, toolCalls) {
+    const forbidden = Array.isArray(evaluator.config.tools) ? evaluator.config.tools.map(String) : [];
+    const used = toolCalls.find((call) => forbidden.includes(call.toolName));
+    return {
+        evaluatorId: evaluator.id,
+        evaluatorType: evaluator.type,
+        mode: evaluator.mode,
+        status: used ? "fail" : "pass",
+        weight: evaluator.weight,
+        rawScore: used ? 0 : 1,
+        message: used ? `Forbidden tool '${used.toolName}' was used.` : "No forbidden tools were used.",
+    };
+}
+function evaluateToolCallAssertion(evaluator, toolCalls) {
+    const tool = String(evaluator.config.tool ?? "");
+    const match = isObject(evaluator.config.match) ? evaluator.config.match : {};
+    const call = toolCalls.find((candidate) => candidate.toolName === tool && matches(candidate.input, match));
+    return {
+        evaluatorId: evaluator.id,
+        evaluatorType: evaluator.type,
+        mode: evaluator.mode,
+        status: call ? "pass" : "fail",
+        weight: evaluator.weight,
+        rawScore: call ? 1 : 0,
+        message: call ? `Observed expected tool call for '${tool}'.` : `Expected tool call for '${tool}' was not found.`,
+        details: { expected: match },
+    };
+}
+function evaluateFinalAnswerContains(evaluator, finalOutput) {
+    const required = Array.isArray(evaluator.config.required_substrings)
+        ? evaluator.config.required_substrings.map(String)
+        : [];
+    const normalizedOutput = normalizeText(finalOutput);
+    const missing = required.filter((candidate) => !normalizedOutput.includes(normalizeText(candidate)));
+    const passed = missing.length === 0;
+    return {
+        evaluatorId: evaluator.id,
+        evaluatorType: evaluator.type,
+        mode: evaluator.mode,
+        status: passed ? "pass" : "fail",
+        weight: evaluator.weight,
+        rawScore: passed ? required.length : required.length - missing.length,
+        message: passed ? "Final answer contains all required substrings." : `Missing required substrings: ${missing.join(", ")}.`,
+    };
+}
+function normalizeText(value) {
+    return value.toLowerCase().replace(/\s+/g, " ").trim();
+}
+function evaluateExactFinalAnswer(evaluator, finalOutput) {
+    const expected = String(evaluator.config.expected ?? "");
+    const passed = finalOutput.trim() === expected.trim();
+    return {
+        evaluatorId: evaluator.id,
+        evaluatorType: evaluator.type,
+        mode: evaluator.mode,
+        status: passed ? "pass" : "fail",
+        weight: evaluator.weight,
+        rawScore: passed ? 1 : 0,
+        message: passed ? "Final answer matched exactly." : "Final answer did not match expected output.",
+    };
+}
+function evaluateStepCountMax(evaluator, stepCount) {
+    const max = Number(evaluator.config.max_steps ?? 0);
+    const passed = stepCount <= max;
+    return {
+        evaluatorId: evaluator.id,
+        evaluatorType: evaluator.type,
+        mode: evaluator.mode,
+        status: passed ? "pass" : "fail",
+        weight: evaluator.weight,
+        rawScore: passed ? 1 : 0,
+        message: passed ? `Step count ${stepCount} is within max ${max}.` : `Step count ${stepCount} exceeds max ${max}.`,
+    };
+}
+function matches(input, match) {
+    if (!isObject(input)) {
+        return false;
+    }
+    return Object.entries(match).every(([key, value]) => input[key] === value);
+}
+function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/dist/index.js

@@ -0,0 +1,296 @@
+#!/usr/bin/env node
+import { createAgentFactory } from "./agent/factory.js";
+import { getAgentRegistration } from "./config.js";
+import { getRunErrorDetail } from "./runOutput.js";
+async function main() {
+    const [, , command, ...args] = process.argv;
+    switch (command) {
+        case "help":
+        case "--help":
+        case "-h":
+            printUsage();
+            return;
+        case "version":
+        case "--version":
+        case "-v":
+            printVersion();
+            return;
+        case "list":
+            await handleList(args);
+            return;
+        case "run":
+            await handleRun(args);
+            return;
+        case "show":
+            await handleShow(args);
+            return;
+        case "compare":
+            await handleCompare(args);
+            return;
+        case "ui":
+            await handleUi();
+            return;
+        default:
+            printUsage();
+    }
+}
+function printUsage() {
+    console.log(`Usage:
+  agentlab list scenarios
+  agentlab run <scenario-id> [--agent <name>] [--provider mock|openai|external_process] [--model <model>] [--agent-label <label>]
+  agentlab run --suite <suite-id> [--agent <name>] [--provider mock|openai|external_process] [--model <model>] [--agent-label <label>]
+  agentlab show <run-id>
+  agentlab compare <baseline-run-id> <candidate-run-id>
+  agentlab ui
+  agentlab help
+  agentlab version`);
+}
+function printVersion() {
+    console.log("0.1.0");
+}
+async function handleList(args) {
+    if (args[0] !== "scenarios") {
+        printUsage();
+        return;
+    }
+    const { listScenarios } = await import("./scenarios.js");
+    for (const scenario of listScenarios()) {
+        console.log(`${scenario.id}\t${scenario.suite}\t${scenario.difficulty ?? "-"}\t${scenario.description ?? ""}`);
+    }
+}
+async function handleRun(args) {
+    const parsed = parseRunArgs(args);
+    const runtimeConfig = validateRuntimeConfig(parsed.runtimeConfig);
+    const { loadScenariosBySuite } = await import("./scenarios.js");
+    if (parsed.suite) {
+        const suite = parsed.suite;
+        if (!suite) {
+            throw new Error("Missing suite id.");
+        }
+        const scenarios = loadScenariosBySuite(suite);
+        if (scenarios.length === 0) {
+            throw new Error(`No scenarios found for suite '${suite}'.`);
+        }
+        const runs = [];
+        for (const scenario of scenarios) {
+            runs.push(await executeOne(scenario.definition.id, runtimeConfig));
+        }
+        const passed = runs.filter((bundle) => bundle.run.status === "pass").length;
+        const failed = runs.filter((bundle) => bundle.run.status === "fail").length;
+        const errored = runs.filter((bundle) => bundle.run.status === "error").length;
+        const avgScore = Math.round(runs.reduce((sum, bundle) => sum + bundle.run.score, 0) / runs.length);
+        console.log(`Suite: ${suite}`);
+        console.log(`Passed: ${passed}/${runs.length}`);
+        console.log(`Failed: ${failed}/${runs.length}`);
+        console.log(`Errored: ${errored}/${runs.length}`);
+        console.log(`Average score: ${avgScore}`);
+        return;
+    }
+    const scenarioId = parsed.scenarioId;
+    if (!scenarioId) {
+        throw new Error("Missing scenario id.");
+    }
+    await executeOne(scenarioId, runtimeConfig);
+}
+async function executeOne(scenarioId, runtimeConfig) {
+    const [{ Storage }, { loadToolRegistry, loadToolSpecs }, { loadScenarioById }, { runScenario }] = await Promise.all([
+        import("./storage.js"),
+        import("./tools.js"),
+        import("./scenarios.js"),
+        import("./runner.js"),
+    ]);
+    const storage = new Storage();
+    const toolSpecs = await loadToolSpecs();
+    const toolRegistry = await loadToolRegistry();
+    const loaded = loadScenarioById(scenarioId);
+    storage.upsertScenario({
+        id: loaded.definition.id,
+        name: loaded.definition.name,
+        suite: loaded.definition.suite,
+        difficulty: loaded.definition.difficulty,
+        description: loaded.definition.description,
+    }, loaded.definition, loaded.filePath, loaded.fileHash);
+    const factory = createAgentFactory(runtimeConfig);
+    const agentVersion = factory.createVersion(runtimeConfig);
+    storage.upsertAgentVersion(agentVersion);
+    const bundle = await runScenario({
+        agentAdapter: factory.createAdapter(),
+        agentVersion,
+        scenario: loaded.definition,
+        scenarioFileHash: loaded.fileHash,
+        toolSpecs,
+        tools: toolRegistry,
+    });
+    bundle.agentVersion = agentVersion;
+    storage.saveRun(bundle);
+    printRunSummary(bundle);
+    return bundle;
+}
+async function handleUi() {
+    const { startUiServer } = await import("./ui/server.js");
+    await startUiServer();
+}
+function printRunSummary(bundle) {
+    console.log(`Run: ${bundle.run.id}`);
+    console.log(`Scenario: ${bundle.run.scenarioId}`);
+    console.log(`Status: ${bundle.run.status.toUpperCase()}`);
+    console.log(`Score: ${bundle.run.score}/100`);
+    console.log(`Agent: ${bundle.agentVersion?.label ?? bundle.run.agentVersionId}`);
+    if (bundle.agentVersion?.provider) {
+        console.log(`Provider: ${bundle.agentVersion.provider}`);
+    }
+    if (bundle.agentVersion?.modelId) {
+        console.log(`Model: ${bundle.agentVersion.modelId}`);
+    }
+    if (bundle.agentVersion?.command) {
+        console.log(`Command: ${bundle.agentVersion.command} ${(bundle.agentVersion.args ?? []).join(" ")}`.trim());
+    }
+    console.log(`Runtime: ${bundle.run.durationMs}ms`);
+    if (bundle.run.status !== "pass") {
+        console.log(`Reason: ${bundle.run.terminationReason}`);
+        const errorDetail = getRunErrorDetail(bundle);
+        if (errorDetail) {
+            console.log(`Error: ${errorDetail}`);
+        }
+    }
+}
+async function handleShow(args) {
+    const runId = args[0];
+    if (!runId) {
+        throw new Error("Missing run id.");
+    }
+    const { Storage } = await import("./storage.js");
+    const storage = new Storage();
+    const bundle = storage.getRun(runId);
+    if (!bundle) {
+        throw new Error(`Run '${runId}' not found.`);
+    }
+    console.log(`Run: ${bundle.run.id}`);
+    console.log(`Scenario: ${bundle.run.scenarioId}`);
+    console.log(`Status: ${bundle.run.status.toUpperCase()}`);
+    console.log(`Score: ${bundle.run.score}/100`);
+    if (bundle.agentVersion) {
+        console.log(`Provider: ${bundle.agentVersion.provider ?? "unknown"}`);
+        console.log(`Model: ${bundle.agentVersion.modelId ?? "unknown"}`);
+        if (bundle.agentVersion.command) {
+            console.log(`Command: ${bundle.agentVersion.command} ${(bundle.agentVersion.args ?? []).join(" ")}`.trim());
+        }
+    }
+    console.log(`Termination: ${bundle.run.terminationReason}`);
+    const errorDetail = getRunErrorDetail(bundle);
+    if (errorDetail) {
+        console.log(`Error: ${errorDetail}`);
+    }
+    console.log(`Final output: ${bundle.run.finalOutput}`);
+    console.log("Evaluators:");
+    for (const result of bundle.evaluatorResults) {
+        console.log(`- ${result.evaluatorId}: ${result.status.toUpperCase()} - ${result.message}`);
+    }
+}
+async function handleCompare(args) {
+    const [baselineRunId, candidateRunId] = args;
+    if (!baselineRunId || !candidateRunId) {
+        throw new Error("Missing baseline or candidate run id.");
+    }
+    const { Storage } = await import("./storage.js");
+    const storage = new Storage();
+    const comparison = storage.compareRuns(baselineRunId, candidateRunId);
+    console.log(`Scenario: ${comparison.baseline.run.scenarioId}`);
+    console.log(`Baseline: ${comparison.baseline.run.id} (${comparison.baseline.run.status.toUpperCase()} ${comparison.baseline.run.score}/100)`);
+    console.log(`Candidate: ${comparison.candidate.run.id} (${comparison.candidate.run.status.toUpperCase()} ${comparison.candidate.run.score}/100)`);
+    console.log("Changes:");
+    if (comparison.notes.length === 0) {
+        console.log("- No material changes.");
+    }
+    else {
+        for (const note of comparison.notes) {
+            console.log(`- ${note}`);
+        }
+    }
+    if (comparison.evaluatorDiffs.length > 0) {
+        console.log("Evaluator diffs:");
+        for (const diff of comparison.evaluatorDiffs) {
+            console.log(`- ${diff.note}`);
+        }
+    }
+    if (comparison.toolDiffs.length > 0) {
+        console.log("Tool diffs:");
+        for (const diff of comparison.toolDiffs) {
+            console.log(`- ${diff.note}`);
+        }
+    }
+}
+function parseRunArgs(args) {
+    const runtimeConfig = { provider: "mock" };
+    let scenarioId;
+    let suite;
+    for (let index = 0; index < args.length; index += 1) {
+        const arg = args[index];
+        if (arg === "--suite") {
+            suite = args[index + 1];
+            index += 1;
+            continue;
+        }
+        if (arg === "--provider") {
+            const provider = args[index + 1];
+            if (provider !== "mock" && provider !== "openai" && provider !== "external_process") {
+                throw new Error(`Unsupported provider '${String(provider)}'.`);
+            }
+            runtimeConfig.provider = provider;
+            index += 1;
+            continue;
+        }
+        if (arg === "--agent") {
+            runtimeConfig.agentName = args[index + 1];
+            index += 1;
+            continue;
+        }
+        if (arg === "--model") {
+            runtimeConfig.model = args[index + 1];
+            index += 1;
+            continue;
+        }
+        if (arg === "--agent-label") {
+            runtimeConfig.label = args[index + 1];
+            index += 1;
+            continue;
+        }
+        if (!scenarioId) {
+            scenarioId = arg;
+            continue;
+        }
+        throw new Error(`Unexpected argument '${arg}'.`);
+    }
+    return { scenarioId, suite, runtimeConfig };
+}
+function validateRuntimeConfig(config) {
+    if (config.agentName) {
+        const registration = getAgentRegistration(config.agentName);
+        config.provider = registration.provider;
+        config.model = config.model ?? registration.model;
+        config.label = config.label ?? registration.label ?? registration.name;
+        config.command = registration.command;
+        config.args = registration.args;
+        config.envAllowlist = registration.envAllowlist;
+    }
+    if (config.provider === "openai") {
+        if (!process.env.OPENAI_API_KEY) {
+            throw new Error("OPENAI_API_KEY is required when --provider openai is used.");
+        }
+        config.model = config.model ?? "gpt-4o-mini";
+    }
+    if (config.provider === "mock") {
+        config.label = config.label ?? config.agentName ?? "mock-support-agent-v1";
+    }
+    if (config.provider === "external_process") {
+        if (!config.command) {
+            throw new Error("External process agents require a configured command.");
+        }
+        config.label = config.label ?? config.agentName ?? "external-process-agent";
+    }
+    return config;
+}
+main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+});

package/dist/lib/fs.js

@@ -0,0 +1,8 @@
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+export function ensureDir(path) {
+    mkdirSync(path, { recursive: true });
+}
+export function ensureParentDir(path) {
+    ensureDir(dirname(path));
+}